package.cpp

// Name:        package.cpp
// Purpose:     wxPackage, wxPackageArray
// Author:      Julian Smart, Francesco Montorsi, Daniel J. Lauk
// Modified by:
// Created:     2004-11-07
// RCS-ID:      $Id: package.cpp,v 1.24 2007/02/01 20:28:16 frm Exp $
// Copyright:   (c) Julian Smart, Francesco Montorsi
// Licence:     wxWidgets licence


// For compilers that support precompilation, includes "wx/wx.h".
#include "wx/wxprec.h"

#ifdef __BORLANDC__
#pragma hdrstop
#endif

#ifndef WX_PRECOMP
    #include "wx/wx.h"
#endif

// includes
#include <wx/fileconf.h>
#include <wx/filename.h>
#include <wx/ptr_scpd.h>
#include <wx/dir.h>

// streams
#include <wx/zipstrm.h>
#include <wx/wfstream.h>
#include <wx/tarstrm.h>
#include <wx/mstream.h>
#include <wx/txtstrm.h>
#include <wx/zstream.h>

#include <wx/protocol/ftp.h>

#include "wxp/package.h"
#include "wxp/wxp.h"
#include "wxp/packagewxp.h"
#include "wxp/wxbuild.h"

#include <memory>       // for auto_ptr
using namespace std;

// static
wxArrayString wxPackage::s_arrLocalRepo;
wxArrayString wxPackage::s_arrRemoteRepo;

// definitions

wxString wxPackageStatusName[] =
{
    _("Remote"),
    _("Downloaded"),
    _("Decompressed"),
    _("Built"),
    _("Installed")
};


// RTTI
IMPLEMENT_DYNAMIC_CLASS(wxPackage, wxObject)

// wx array implementations
#include <wx/arrimpl.cpp> // this is a magic incantation which must be done!
WX_DEFINE_OBJARRAY(wxPackageArrayHelper);

// global objects
wxPackage wxEmptyPackage;
wxPackageArray wxEmptyPackageArray;



// ----------------------------------------------------------------------------
// string2enum helpers
// ----------------------------------------------------------------------------

wxIMPLEMENT_STRING2ENUM(PackageStatus, wxPS_MAX)



// ----------------------------------------------------------------------------
// wxPackage
// ----------------------------------------------------------------------------

bool wxPackage::IsOk() const
{
    if (!wxPackageInfo::IsOk())
        return false;

    // can check anything of m_compiler ?

    switch (m_status)
    {
    case wxPS_INSTALLED:
        if (!wxFileName::DirExists(m_strInstallationPath))
            return false;
        // don't break !!

    case wxPS_DECOMPRESSED:
        if (m_strWXPPath.IsEmpty() || !wxFileName(m_strWXPPath).IsRelative())
            return false;
        if (!wxFileName::DirExists(m_strDecompressionPath))
            return false;
        // don't break !!

    case wxPS_DOWNLOADED:
        if (!wxFileName::FileExists(m_strDownloadPath))
            return false;
        // don't break !!

    case wxPS_REMOTE:
        break;

    default:
        return true;
    }

    return true;
}

void wxPackage::DoUpdateSubstitutionHashMap(wxPackageSubstituteInfoContext ctx)
{
    wxPackageInfo::DoUpdateSubstitutionHashMap(ctx);

    wxStringHashMap &keywords = m_hashmapSubstitution[ctx];

#define ADD_KEYWORD(name, value)   \
    keywords[wxT(name)] = value

    ADD_KEYWORD("packageroot", m_strDecompressionPath);

    switch (m_status)
    {
    case wxPS_REMOTE:
        if (!GetDownloadLink().IsEmpty())
            ADD_KEYWORD("location", GetDownloadLink());
        else
            ADD_KEYWORD("location", wxT("not available"));
        break;

    case wxPS_DOWNLOADED:
        ADD_KEYWORD("location", m_strDownloadPath);
        break;
    case wxPS_DECOMPRESSED:
    case wxPS_BUILT:
    case wxPS_INSTALLED:
        ADD_KEYWORD("location", m_strDecompressionPath);
        break;

    default:
        ADD_KEYWORD("location", wxT("not available"));
    }

    // merge the compiler's substitution hashmap with the "keywords" hashmap
    wxStringHashMap compiler = 
        m_compiler.GetSubstitutionHashMap(m_buildSystemType);
    wxMergeHashMap(keywords, compiler);
}

wxPackageCommand wxPackage::GetCmd(const wxPackageCommand &cmd,
                                   wxPackageBuildSystemStage stage,
                                   long flags,
                                   const wxPackageCondition &cond) const
{
    // substitute basic info:
    wxPackageCommand ret(wxPackageInfo::GetCmd(cmd, stage, flags, cond));

    // substitute even more info
    wxString subst = SubstituteInfo(ret.GetCommand());
    ret.SetCommand(subst);

#ifdef __WXDEBUG__
    // at this stage typically all $() keys should have been substituted:
    if (subst.Contains(wxWPM_SUBSTITUTION_START_MARKER))
        wxLogDebug(wxT("wxPackage::GetCmd - found unknown keywords in the command: [%s]"),
                   subst.c_str());
#endif

    return ret;
}

void wxPackage::SetRemoteStatus(const wxString &downloadpath)
{
    m_status = wxPS_REMOTE;

    // this is the path where the downloaded package will be placed
    if (!downloadpath.IsEmpty())
    {
        wxFileName fn(downloadpath);
        wxASSERT(!fn.IsDir() && fn.IsAbsolute());
        m_strDownloadPath = fn.GetFullPath();
    }
    else
        m_strDownloadPath = wxEmptyString;

    UpdatePackageInfo();
}

void wxPackage::SetDownloadedStatus(const wxString &downloadpath)
{
    wxFileName dp(downloadpath);

    // downloadpath should be an ABSOLUTE fully qualified filename: path + name + ext
    wxASSERT(!dp.IsDir() && dp.IsAbsolute());

    m_status = wxPS_DOWNLOADED;
    m_strDownloadPath = dp.GetFullPath();

    UpdatePackageInfo();
}

void wxPackage::SetDecompressedStatus(const wxString &decompresspath)
{
    wxFileName dp = wxFileName::DirName(decompresspath);

    // decompresspath should be an ABSOLUTE directory name
    wxASSERT(dp.IsDir() && dp.IsAbsolute());

    m_status = wxPS_DECOMPRESSED;
    m_strDecompressionPath = dp.GetPath();

    UpdatePackageInfo();
}

void wxPackage::SetBuiltStatus()
{
    m_status = wxPS_BUILT;
    UpdatePackageInfo();
}

void wxPackage::SetInstalledStatus(const wxString &installpath)
{
    m_status = wxPS_INSTALLED;
    m_strInstallationPath = installpath;

    UpdatePackageInfo();
}

void wxPackage::UpdatePackageInfo()
{
    if (m_status >= wxPS_DECOMPRESSED)
    {
        // make local references as absolute
        m_refLogo.ExpandLocalRef(m_strDecompressionPath);
        for (size_t i=0; i < m_arrScreenshots.GetCount(); i++)
            m_arrScreenshots[i].ExpandLocalRef(m_strDecompressionPath);
        for (size_t i=0; i < m_arrDocumentation.GetCount(); i++)
            m_arrDocumentation[i].ExpandLocalRef(m_strDecompressionPath);

        UpdateCompressedSize();
        UpdateMD5();
    }
    else if (m_status >= wxPS_DOWNLOADED)
    {
        // the package has not been decompressed yet but we still can make
        // the browsing faster loading references to the logo,screenshots and
        // documentation from the WXZ instead of from WWW
        m_refLogo.ExpandLocalRefToWXZRef(m_strDownloadPath);
        for (size_t i=0; i < m_arrScreenshots.GetCount(); i++)
            m_arrScreenshots[i].ExpandLocalRefToWXZRef(m_strDownloadPath);
        for (size_t i=0; i < m_arrDocumentation.GetCount(); i++)
            m_arrDocumentation[i].ExpandLocalRefToWXZRef(m_strDownloadPath);

        UpdateCompressedSize();
        UpdateMD5();
    }
    else if (m_status == wxPS_REMOTE)
    {
        // force the use of href instead of localref
        m_refLogo.RemoveLocalRef();
        for (size_t i=0; i < m_arrScreenshots.GetCount(); i++)
            m_arrScreenshots[i].RemoveLocalRef();
        for (size_t i=0; i < m_arrDocumentation.GetCount(); i++)
            m_arrDocumentation[i].RemoveLocalRef();
    }
}

bool wxPackage::LoadStatusInfo(wxConfigBase *cfg, const wxString &path)
{
    if (!cfg->Read(path + wxT("/Status"), (long *)&m_status))
        return false;

    // wxPS_REMOTE is a valid state for a package even if it means
    // that there are no status info for it.
    // A package's status may have been saved on the user's system
    // as wxPS_REMOTE when e.g. the package has been first installed
    // and then uninstalled and finally removed from the system...
    if (m_status == wxPS_REMOTE)
    {
        // let's see however if the package do exist
        if (!m_strDownloadPath.IsEmpty() &&
            wxFileName::FileExists(m_strDownloadPath))
            m_status = wxPS_DOWNLOADED;
    }

    if (m_status != wxPS_DOWNLOADED &&
        m_status != wxPS_DECOMPRESSED &&
        m_status != wxPS_BUILT &&
        m_status != wxPS_INSTALLED)
        return false;

    // read all paths
    wxString str;
    if (!cfg->Read(path + wxT("/DownloadPath"), &str))
        return false;

    // if the status of the package has been fixed previously, don't
    // overwrite the, maybe correct, m_strDownloadPath
    if (!str.IsEmpty())
        m_strDownloadPath = str;

    if (!cfg->Read(path + wxT("/WXPPath"), &m_strWXPPath))
        return false;
    if (m_status >= wxPS_DECOMPRESSED &&
        !cfg->Read(path + wxT("/DecompressionPath"), &m_strDecompressionPath))
        return false;
    if (m_status >= wxPS_INSTALLED &&
        !cfg->Read(path + wxT("/InstallationPath"), &m_strInstallationPath))
        return false;

    // check them
    if (m_status >= wxPS_DOWNLOADED && !wxFileName::FileExists(m_strDownloadPath))
    {
        wxLogWarning(_("The package %s %s is marked as downloaded but\n\t%s\ndoes not exist. Resetting it to the not-downloaded-yet package status."),
                     GetName().c_str(), GetVersion().GetAsString().c_str(), m_strDownloadPath.c_str());
        m_status = wxPS_REMOTE;
        m_strDownloadPath = wxEmptyString;
    }

    if (m_status >= wxPS_DECOMPRESSED)
    {
        if (!wxFileName::DirExists(m_strDecompressionPath))
        {
            wxLogWarning(_("The package %s %s is marked as decompressed but the folder\n\t%s\ndoes not exist. Resetting it to the just-downloaded package status."),
                        GetName().c_str(), GetVersion().GetAsString().c_str(), m_strDecompressionPath.c_str());
            m_status = wxPS_DOWNLOADED;
            m_strDecompressionPath = wxEmptyString;
        }
        else if (!wxFileName::FileExists(GetAbsWXPPath()))
        {
            wxLogWarning(_("The package %s %s is marked as decompressed in the folder\n\t%s\nbut the file\n\t%s\ndoes not exist. Resetting it to the just-downloaded package status."),
                        GetName().c_str(), GetVersion().GetAsString().c_str(), m_strDecompressionPath.c_str(),
                        GetAbsWXPPath().c_str());
            m_status = wxPS_DOWNLOADED;
            m_strDecompressionPath = wxEmptyString;
        }
        else
        {
            /*wxLogNull nul;

            // try to load the WXP of this package
            wxPackageXMLDescriptor wxp;
            if (wxp.Load(GetAbsWXPPath()))
                *this = wxp.GetPackageInfo();*/
        }
    }

    // now that our status has changed
    // (and that we gathered new info about this package) we can eventually
    // update some of our fields.
    UpdatePackageInfo();

    return true;
}

bool wxPackage::SaveStatusInfo(wxConfigBase *cfg, const wxString &path) const
{
    cfg->Write(path + wxT("/Status"), m_status);

    cfg->Write(path + wxT("/DownloadPath"), m_strDownloadPath);
    cfg->Write(path + wxT("/DecompressionPath"), m_strDecompressionPath);
    cfg->Write(path + wxT("/InstallationPath"), m_strInstallationPath);
    cfg->Write(path + wxT("/WXPPath"), m_strWXPPath);

    return true;
}

wxULongLong wxPackage::GetDecompressedSize() const
{
    wxCHECK_MSG(m_status >= wxPS_DOWNLOADED, 0, wxT("Invalid package status"));
    wxULongLong ret;

    // compute the size of the decompressed package summing ZIP entries sizes
    auto_ptr<wxArchiveInputStream> arc( GetInputStream(GetDownloadPath()) );
    auto_ptr<wxArchiveEntry> entry;

    if (arc.get() == NULL)
        return false;

    while (entry.reset(arc->GetNextEntry()), entry.get() != NULL)
        ret += entry->GetSize();

    return ret;
}

wxWidgetsBuildArray wxPackage::GetSupportedWxBuilds(bool docompilercheck) const
{
    wxWidgetsBuildArray ret;
    for (size_t i=0; i < wxWidgetsBuildArray::s_arrWxBuilds.GetCount(); i++)
    {
        const wxWidgetsBuild &b = wxWidgetsBuildArray::s_arrWxBuilds[i];

        // does the port+version match any of our supported ports ?
        if (!SupportsWxPort(b.GetPortId(), b.GetVersion()))
            continue;       // skip this wxbuild

        if (docompilercheck)
            if (b.GetCompilerUsed() != m_compiler.GetSelFormat())
                continue;

        // all checks successful
        ret.Add(b);
    }

    return ret;
}

bool wxPackage::GetRecursiveDependencies(wxPackageDependencyArray *ret, 
                                         const wxPackageArray &arr,
                                         wxPackageDependencyType type, 
                                         wxPackageDependencyArray *notfound) const
{
    static wxPackageIdArray s_temp;
    s_temp.Clear();

    if (!DoGetRecursiveDependencies(ret, arr, type, notfound, &s_temp))
    {
        // false means that there were unresolved dependencies
        if (notfound)
        {
            // because of the organization of wxPackage::DoGetRecursiveDependencies,
            // even if these deps are missing, they've been added to 'ret'.
            // We need to remove them !
            for (size_t i=0; i<notfound->GetCount(); i++)
            {
                int n = ret->Index(notfound->Item(i));
                wxASSERT(n != wxNOT_FOUND);
                ret->RemoveAt(n);
            }
        }

        return false;
    }

    return true;
}

bool wxPackage::DoGetRecursiveDependencies(wxPackageDependencyArray *ret, 
                                           const wxPackageArray &arr,
                                           wxPackageDependencyType type, 
                                           wxPackageDependencyArray *notfound, 
                                           wxPackageIdArray *alreadyprocessed) const
{
    // some of given pointers cannot be NULL !!
    wxASSERT(ret && alreadyprocessed);

    if (alreadyprocessed->Index(*this) != wxNOT_FOUND)
    {
        // we've found a circular dependency !
        // but this is not a problem ;)
        return true;
    }

    // add to the given list of dependencies our dependencies of given type
    wxPackageDependencyArray temp;
    const wxPackageDependencyArray *toadd;
    if (type == wxPDT_INVALID)
    {
        toadd = &m_arrDependencies;
    }
    else
    {
        // NB: we create a temp array otherwise the macro WX_APPEND_ARRAY
        //     would generate lots of calls to GetDependenciesOfType !!
        temp = m_arrDependencies.GetDependenciesOfType(type);
        toadd = &temp;
    }

    for (size_t i=0; i < toadd->GetCount(); i++)
    {
        int n = ret->Index(toadd->Item(i));
        if (n != wxNOT_FOUND)
        {
            ret->Item(n).AddOwnerId(*this);
            continue;       // do not add duplicates!!
        }

        // add our i-th dependency to the array
        wxASSERT(toadd->Item(i).GetOwnersId().Index(*this) != wxNOT_FOUND);
        ret->Add(toadd->Item(i));
    }

    // add ourselves to the list of already processed packages
    // this is very important in order to avoid being trapped in
    // an infinite loop in case of circular dependencies !
    alreadyprocessed->Add(*this);

    // add all our dependencies recursively
    return m_arrDependencies.DoGetRecursiveDependencies(ret, arr, type, notfound, alreadyprocessed);
}

wxArchiveInputStream *wxPackage::GetInputStream(const wxString &wxzpath) const
{
    wxFileName fn(wxzpath);
    if (!fn.IsAbsolute())
        fn.MakeAbsolute();
    wxString filename = fn.GetFullPath();

    // generic code to read any type of archive supported by the
    // wxArchive* classes taken from wxDocs:

    auto_ptr<wxInputStream> in(new wxFFileInputStream(filename));

    // if the file ends with .wxz it may be any of the compressed formats
    // supported by wxPackage
    if (fn.GetExt().CmpNoCase(wxT("wxz")) == 0)
    {
        // try to detect the format type reading e.g. the first 
        wxByte signature[12];
        if (!in->Read(signature, 10*sizeof(wxByte)))
            return NULL;

        // seek back to the start
        in->SeekI(0, wxFromStart);

        struct
        {
            wxByte magic[4];
            size_t nByteLen;
            wxPackageCompressionMode compression;
        } magics[2] = 
        {
            {
                { 0x50, 0x4B, 0x03, 0x04 },
                4,
                wxPCM_ZIP
            },
            {
                { 0x1F, 0x8B, 0x08 },
                3,
                wxPCM_TAR_GZ
            }
        };

        wxPackageCompressionMode detected = wxPCM_INVALID;
        for (size_t i=0; i < WXSIZEOF(magics); i++)
        {
            bool match = true;
            for (size_t j=0; j < magics[i].nByteLen; j++)
                match &= (signature[j] == magics[i].magic[j]);

            if (match)
            {
                detected = magics[i].compression;
                break;
            }
        }

        switch (detected)
        {
        case wxPCM_ZIP:
            // should be a ZIP archive
            return new wxZipInputStream(in.release());  


        case wxPCM_TAR_GZ:
            {
                // should be a .tar.gz archive
                wxZlibInputStream *filter = new wxZlibInputStream(in.release());
                return new wxTarInputStream(filter);
            }

        case wxPCM_INVALID:
        default:
            wxLogError(wxT("The archive '%s' is of unknown type."), wxzpath.c_str());
            return NULL;       // unknown archive type
        }
    }

    // look for a filter handler, e.g. for '.gz'
    const wxFilterClassFactory *fcf;
    fcf = wxFilterClassFactory::Find(filename, wxSTREAM_FILEEXT);
    if (fcf)
    {
        in.reset(fcf->NewStream(in.release()));

        // pop the extension, so if it was '.tar.gz' it is now just '.tar'
        filename = fcf->PopExtension(filename);
    }
    //else: ignore, no filter required for this archive

    // look for a archive handler, e.g. for '.zip' or '.tar'
    const wxArchiveClassFactory *acf;
    acf = wxArchiveClassFactory::Find(filename, wxSTREAM_FILEEXT);
    if (!acf)
    {
        wxLogError(wxT("The archive '%s' is of unknown type."), wxzpath.c_str());
        return NULL;       // unknown archive type
    }

    return acf->NewStream(in.release());
}

bool wxPackage::LoadCompressedPackage(const wxString &wxzpath)
{
    auto_ptr<wxArchiveInputStream> arc( GetInputStream(wxzpath) );
    auto_ptr<wxArchiveEntry> entry;

    if (arc.get() == NULL)
        return false;

    // scan this ZIP in search of the WXP it should contain
    while (entry.reset(arc->GetNextEntry()), entry.get() != NULL)
    {
        wxString name = entry->GetName();
        if (name.EndsWith(wxT(".wxp")))
        {
            wxPackageXMLDescriptor wxp;
            if (!wxp.Load(*arc))
                return false;       // something failed!

            // load the package info
            *this = wxp.GetPackageInfo();
            if (!IsOk())
            {
                wxLogError(wxT("The WXP contained in '%s' is invalid!"),
                           wxzpath.c_str());
                return false;
            }

            // we must provide an absolute path as download path
            wxFileName fn(wxzpath);
            if (!fn.IsAbsolute())
                fn.MakeAbsolute();

            // this is a local package and we know where its WXP file is...
            SetDownloadedStatus(fn.GetFullPath());
            SetRelativeWXPPath(name);

            // ok, WXP loaded successfully from this WXZ
            return true;
        }
    }

    // WXP not found in given archive!
    return false;
}

bool wxPackage::Decompress(const wxString &dir, wxProgressHandler *progress)
{
    wxASSERT(wxFileName(dir).IsAbsolute());

    // check that the output dir already exists
    wxString path(dir);
    if (path.Last() == wxFileName::GetPathSeparator())
        path.RemoveLast();
    if (!wxFileName::DirExists(path))
    {
        wxLogError(wxT("The decompression folder '%s' does not exist!"), dir.c_str());
        return false;
    }

    wxULongLong decompressedbytes = 0,
                totalbytes = GetDecompressedSize();

    auto_ptr<wxArchiveInputStream> arc( GetInputStream(GetDownloadPath()) );
    auto_ptr<wxArchiveEntry> entry;

    if (arc.get() == NULL)
        return false;

    while (entry.reset(arc->GetNextEntry()), entry.get() != NULL)
    {
        wxFileName fn(path + wxFileName::GetPathSeparator() + entry->GetName());

        if (entry->IsDir())
        {
            // create this folder if it does not exist
            if (!fn.DirExists() && !fn.Mkdir(0777, wxPATH_MKDIR_FULL))
            {
                wxLogError(_("Cannot create folder '%s' - maybe because of a permission problem"),
                           fn.GetFullPath().c_str());
                return false;
            }
            else
                continue;
        }

        // open the output file
        wxString outpath = fn.GetFullPath();
        wxFileName basepath(outpath);
        if (!basepath.DirExists() && !basepath.Mkdir(0777, wxPATH_MKDIR_FULL))
        {
            wxLogError(_("Cannot create folder '%s' - maybe because of a permission problem"),
                        basepath.GetPath().c_str());
            return false;
        }

        wxFFileOutputStream out(outpath);

        // and decompress data inside it
        out.Write(*arc);
        if (out.GetSize() != entry->GetSize())
        {
            wxLogError(_("An error occurred when decompressing the '%s' file.\nIt should be %d bytes long but the size of the decompressed file is %d bytes. Please check the archive's integrity."),
                       fn.GetFullPath().c_str(), entry->GetSize(), out.GetSize());
            return false;
        }

        decompressedbytes += entry->GetSize();

        // advance the progress handler: remember that we must pass
        // a value between [0;1000] to it!
        if (progress)
            progress->OnUpdate((int)(1000 * ( decompressedbytes.ToDouble() / totalbytes.ToDouble() )),
                               entry->GetName());
    }

    SetDecompressedStatus(path);
    return true;
}

// helper class for wxPackage::GetIncludedFiles
class wxPackageDirTraverser : public wxDirTraverser
{
public:
    wxPackageDirTraverser(const wxString &prefix,
                          const wxArrayString &filesToExclude,
                          const wxArrayString &dirsToExclude,
                          const wxArrayString &patternsToExclude,
                          wxULongLong *totalsize)
        : m_prefix(prefix), 
          m_pTotalSize(totalsize)
    { 
        // we need to compare dirs/files/patterns using native path separator!
        ConvertToNativeSeparator(filesToExclude, m_arrFilesExcluded);
        ConvertToNativeSeparator(dirsToExclude, m_arrDirsExcluded);
        ConvertToNativeSeparator(patternsToExclude, m_arrPatternsExcluded);
    }

    void ConvertToNativeSeparator(const wxArrayString &source, wxArrayString &arr)
    {
        const wxString sep = wxString(wxFileName::GetPathSeparator(), 1);
        for (size_t i=0; i<source.GetCount(); i++)
        {
            wxString str = source[i];
            str.Replace(wxT("/"), sep);
            arr.Add(str);
        }
    }

    virtual wxDirTraverseResult OnFile(const wxString& filename)
    {
        wxASSERT(filename.StartsWith(m_prefix));
        wxString name = filename.Mid(m_prefix.Len());

        // remove the separator, if present
        if (name[0] == wxFileName::GetPathSeparator())
            name = name.Mid(1);

        if (m_arrFilesExcluded.Index(name) != wxNOT_FOUND)
            return wxDIR_CONTINUE;      // skip this file

        for (size_t i=0; i<m_arrPatternsExcluded.GetCount(); i++)
            if (wxMatchWild(m_arrPatternsExcluded[i], name, false /* dot_special */))
                return wxDIR_CONTINUE;      // skip this file

        // this file must be included
        m_files.Add(filename);

        if (m_pTotalSize)
            *m_pTotalSize += wxFileName::GetSize(filename);

        return wxDIR_CONTINUE;
    }

    virtual wxDirTraverseResult OnDir(const wxString& dirname)
    {
        wxASSERT(dirname.StartsWith(m_prefix));
        wxString name = dirname.Mid(m_prefix.Len());

        // remove the separator, if present
        if (name[0] == wxFileName::GetPathSeparator())
            name = name.Mid(1);

        if (m_arrDirsExcluded.Index(name) != wxNOT_FOUND)
            return wxDIR_IGNORE;      // skip this directory

        // recurse into this directory
        return wxDIR_CONTINUE;
    }

    wxArrayString GetFiles() const
        { return m_files; }

protected:
    wxString m_prefix;
    wxArrayString m_arrFilesExcluded, 
                  m_arrDirsExcluded, 
                  m_arrPatternsExcluded;

    wxULongLong *m_pTotalSize;

    // the list filled in by this class
    wxArrayString m_files;
};

wxArrayString wxPackage::GetIncludedFiles(wxULongLong *totalsize) const
{
    // we need the decompression path in order to proceed
    wxCHECK_MSG(m_status >= wxPS_DECOMPRESSED, 0, wxT("Invalid package status"));
    wxString prefix = GetDecompressionPath().BeforeLast(wxFileName::GetPathSeparator());

    wxPackageDirTraverser 
        traverser(prefix,
                  m_arrExcluded[wxPEM_FILE],
                  m_arrExcluded[wxPEM_DIRECTORY],
                  m_arrExcluded[wxPEM_PATTERN],
                  totalsize);

    // NOTE: we want the decompression path here and not the "prefix" !
    wxDir dir(GetDecompressionPath());

    if (!dir.IsOpened())
    {
        wxArrayString empty;
        return empty;
    }

    // search included files
    dir.Traverse(traverser);
    return traverser.GetFiles();
}

// helper class for wxPackage::Compress
class wxPackageDirTraverserArchiver : public wxDirTraverser
{
public:
    wxPackageDirTraverserArchiver(const wxString &prefix,
                                  const wxArrayString &filelist,
                                  wxArchiveOutputStream &z, 
                                  wxProgressHandler *h,
                                  wxULongLong totalsize)
        : m_prefix(prefix), m_fileList(filelist), m_arch(z), 
          m_progress(h), m_totalSize(totalsize)
    { 
        m_bAllCompressed=true; 
        m_compressedBytes=0; 
    }

    virtual wxDirTraverseResult OnFile(const wxString& filename)
    {
        if (m_fileList.Index(filename) == wxNOT_FOUND)
        {
            //wxLogDebug(wxT("Skipping file '%s'"), filename.c_str());
            return wxDIR_CONTINUE;
        }

        wxASSERT(filename.StartsWith(m_prefix));
        wxString name = filename.Mid(m_prefix.Len());

        // remove the separator, if present
        if (name[0] == wxFileName::GetPathSeparator())
            name = name.Mid(1);

        // read the file
        wxFFileInputStream in(filename);

        // create a new entry in the archive
        // since when creating a .tar.gz we are tarring data to a non-seekable
        // stream (i.e. the gzipper stream) we need to provude the data size
        // in advance, together with its timestamp:
        wxFileName fn(filename);
        if (!m_arch.PutNextEntry(name, fn.GetModificationTime(), in.GetSize()))
        {
            m_bAllCompressed = false;       // continue ignoring this error
            return wxDIR_CONTINUE;
        }

        // compress it using the output stream (which should be
        // a wxFilterOutputStream which pass data to the archive stream)
        if (m_arch.Write(in).GetLastError() != wxSTREAM_NO_ERROR)
            m_bAllCompressed = false;       // continue ignoring this error
        else
            m_compressedBytes += in.GetSize();

        // advance the progress handler: remember that we must pass
        // a value between [0;1000] to it!
        if (m_progress)
        {
            unsigned long progress =
                (int)(1000 * ( m_compressedBytes.ToDouble() / m_totalSize.ToDouble() ));

            if (progress > 1000)
                // something wrong: m_totalSize was not computed correctly;
                // do not assert however - just clip 'progress'
                progress = 1000;

            m_progress->OnUpdate(progress, filename);
        }

        return wxDIR_CONTINUE;
    }

    virtual wxDirTraverseResult OnDir(const wxString& WXUNUSED(dirname))
    {
        // no need to create an entry for a directory!
        return wxDIR_CONTINUE;
    }

    virtual wxDirTraverseResult OnOpenError(const wxString& dirname)
    {
        wxLogWarning(wxT("Couldn't compress '%s' directory: cannot open it"), 
                     dirname.c_str());
        m_bAllCompressed = false;
        return wxDIR_IGNORE;
    }

    bool AllCompressed() const
        { return m_bAllCompressed; }

protected:
    wxString m_prefix;
    const wxArrayString &m_fileList;

    wxArchiveOutputStream &m_arch;
    wxProgressHandler *m_progress;

    wxULongLong m_totalSize;
    wxULongLong m_compressedBytes;
    bool m_bAllCompressed;
};

bool wxPackage::Compress(const wxString &wxz, wxProgressHandler *progress)
{
    wxCHECK_MSG(m_status >= wxPS_DECOMPRESSED, 0, wxT("Invalid package status"));
    wxString file = wxz;

    // create a filename if none is provided
    if (file.IsEmpty())
        file = SubstituteInfo(GetOutputFileName());

    // open the output file
    wxFFileOutputStream out(file, wxT("wb"));
    if (!out.IsOk())
    {
        wxLogError(wxT("Couldn't create the output file '%s'."), wxz.c_str());
        return false;
    }

    // create the compressor stream
    wxArchiveOutputStream *archiveos = NULL;
    wxOutputStream *os = NULL;
    switch (GetCompressionMode())
    {
    case wxPCM_ZIP:
        archiveos = new wxZipOutputStream(out, wxPKG_ZIP_COMPRESSION_LEVEL);
        os = archiveos;
        break;

    case wxPCM_TAR_GZ:
        if (!wxZlibOutputStream::CanHandleGZip())
        {
            wxLogError(wxT("Cannot create tar.gz archives!"));
            return false;
        }

        // data need first to be tarred and then gzipped
        os = new wxZlibOutputStream(out, wxPKG_TARGZ_COMPRESSION_LEVEL, wxZLIB_GZIP);
        archiveos = new wxTarOutputStream(*os);
        break;
    }

    if (!archiveos->IsOk())
    {
        wxLogError(wxT("Couldn't create the compressor output stream."));
        return false;
    }

    // get the list of files to compress
    wxULongLong totalSz;
    wxArrayString filelist(GetIncludedFiles(&totalSz));
    if (filelist.GetCount() == 0)
    {
        wxLogError(wxT("No files to include!"));
        return false;
    }

    // the prefix is the parent's folder of the project:
    wxFileName prefix = wxFileName::DirName(GetDecompressionPath());
    prefix.RemoveLastDir();

    // init directory traverser
    wxPackageDirTraverserArchiver 
        traverser(prefix.GetPath(), filelist, *archiveos, 
                  progress, totalSz.ToULong());
    wxDir dir(GetDecompressionPath());

    if (!dir.IsOpened())
    {
        wxLogError(wxT("Cannot open the directory '%s'."),
                   GetDecompressionPath().c_str());
        return false;
    }

    // do compression
    dir.Traverse(traverser);

    // clean up
    if (!os->IsOk())
    {        
        wxLogError(wxT("Cannot write to the output stream"));
        return false;
    }

    switch (GetCompressionMode())
    {
    case wxPCM_ZIP:
        wxDELETE(archiveos);
        break;

    case wxPCM_TAR_GZ:
        // the order is important!
        wxDELETE(archiveos);
        wxDELETE(os);
        break;
    }

    return traverser.AllCompressed() &&
           out.IsOk();
}

bool wxPackage::UploadWithFTP(const wxString &u, const wxString &pass, 
                              const wxString &s, const wxString &p)
{
    wxCHECK_MSG(m_status >= wxPS_DOWNLOADED, 0, wxT("Invalid package status"));
    wxString user(u), password(pass), server(s), path(p);
    wxURI dest(GetUploadDestination());
    wxFTP ftp;

    if (user.IsEmpty())
        user = dest.GetUser();
    if (password.IsEmpty())
        password = dest.GetPassword();
    if (server.IsEmpty())
        server = dest.GetServer();
    if (path.IsEmpty())
        path = dest.GetPath();

    if (!user.IsEmpty())
    {
        ftp.SetUser(user);
        ftp.SetPassword(password);
    }
    // else: anonymous FTP login

    if ( !ftp.Connect(server) )
    {
        wxLogError(wxT("Couldn't connect to '%s'."), server.c_str());
        return false;
    }

    if ( !ftp.ChDir(path) )
    {
        wxLogError(wxT("Couldn't change the directory to '%s'."), path.c_str());
        return false;
    }

    // create destination filename
    wxString destfile = path;
    if (destfile.Last() != wxT('/'))
        destfile += wxT('/');
    destfile += wxFileName(GetDownloadPath()).GetFullName();

    wxOutputStream *out = ftp.GetOutputStream(destfile);
    if ( out )
    {
        wxFileInputStream source(GetDownloadPath());
        if (!IsOk())
        {
            wxLogError(wxT("Couldn't open the source file '%s'."), GetDownloadPath().c_str());
            wxDELETE(out);
            return false;
        }

        out->Write(source);

        if (out->GetLastError() != wxSTREAM_NO_ERROR)
        {
            wxLogError(wxT("Couldn't write to the destination file '%s'."), destfile.c_str());
            wxDELETE(out);
            return false;
        }

        wxDELETE(out);
    }

    ftp.Close();
    return true;
}

/* static */
bool wxPackage::LoadGlobals(wxConfigBase *p, const wxString &path)
{
    int num;
    wxString str;

    // local repository paths
    if (!p->Read(path + wxT("/LocalRepo/Num"), &num))
        return false;
    for (int i=0; i < num; i++)
    {
        if (p->Read(path + wxString::Format(wxT("/LocalRepo/Path%d"), i), &str))
        {
            if (wxFileName::IsDirWritable(str))
                s_arrLocalRepo.Add(str);
            else
                wxLogWarning(_("Discarded the '%s' local repository path as it's not writable."),
                             str.c_str());
        }
    }

    // remote repository paths
    if (!p->Read(path + wxT("/RemoteRepo/Num"), &num))
        return false;
    for (int i=0; i < num; i++)
        if (p->Read(path + wxString::Format(wxT("/RemoteRepo/Path%d"), i), &str))
            s_arrRemoteRepo.Add(str);

    return s_arrLocalRepo.GetCount() > 0 &&
           s_arrRemoteRepo.GetCount() > 0;
}

/* static */
void wxPackage::SaveGlobals(wxConfigBase *p, const wxString &path)
{
    // save local repo urls
    p->DeleteGroup(path + wxT("/LocalRepo"));
    for (size_t i=0; i < s_arrLocalRepo.GetCount(); i++)
        p->Write(path + wxString::Format(wxT("/LocalRepo/Path%d"), i),
                 s_arrLocalRepo[i]);
    p->Write(path + wxT("/LocalRepo/Num"), (long)s_arrLocalRepo.GetCount());

    // save remote repo urls
    p->DeleteGroup(path + wxT("/RemoteRepo"));
    for (size_t i=0; i < s_arrRemoteRepo.GetCount(); i++)
        p->Write(path + wxString::Format(wxT("/RemoteRepo/Path%d"), i),
                 s_arrRemoteRepo[i]);
    p->Write(path + wxT("/RemoteRepo/Num"), (long)s_arrRemoteRepo.GetCount());
}



// ----------------------------------------------------------------------------
// wxPackageArray
// ----------------------------------------------------------------------------

bool wxPackageArray::LoadPackagesStatusInfo(wxConfigBase *cfg, const wxString &path)
{
    size_t failedcount=0;

    // read all groups with the same IDs of our packages
    for (size_t i=0; i<GetCount(); i++)
    {
        wxString id = Item(i).GetName() + wxT(" ") + Item(i).GetVersion();

        if (cfg->HasGroup(path + wxT("/Status/") + id))
        {
            if (!Item(i).LoadStatusInfo(cfg, path + wxT("/Status/") + id))
                failedcount++;
        }
    }

    return failedcount == 0;
}

bool wxPackageArray::SavePackagesStatusInfo(wxConfigBase *cfg, const wxString &path) const
{
    size_t failedcount=0;

    // save all package status info
    cfg->DeleteGroup(path + wxT("/Status"));
    for (size_t i=0; i<GetCount(); i++)
    {
        wxString id = Item(i).GetName() + wxT(" ") + Item(i).GetVersion();
        if (!Item(i).SaveStatusInfo(cfg, path + wxT("/Status/") + id))
        {
            failedcount++;
            wxLogError(wxT("Could not save status info for package '%s'."), id.c_str());
        }
    }

    return failedcount == 0;
}

bool wxPackageArray::LoadCompressedPackagesFrom(const wxString &path)
{
    if (!wxFileName::DirExists(path))
        return false;

    wxArrayString wxplist;
    size_t failedcount=0;

    const wxString extensions[] = { wxT("*.wxz"), wxT("*.zip"), wxT("*.tar.gz") };

    for (size_t j=0; j < WXSIZEOF(extensions); j++)
    {
        wxplist.Clear();
        wxDir::GetAllFiles(path, &wxplist, extensions[j], wxDIR_FILES);
        for (size_t i=0; i<wxplist.GetCount(); i++)
        {
            wxPackage pkg;
            if (!pkg.LoadCompressedPackage(wxplist[i]))
            {
                failedcount++;
                wxLogError(
                    wxT("Could not load the '%s' package; it's not a valid wxWidgets package."),
                    wxplist[i].c_str());
            }
            else
            {
                Add(pkg);
            }
        }
    }

    return GetCount() > 0 ||
           failedcount == 0;
}

bool wxPackageArray::LoadCompressedPackagesFromLocalRepo()
{
    bool success = true;

    for (size_t i=0; i<wxPackage::s_arrLocalRepo.GetCount(); i++)
        success &= LoadCompressedPackagesFrom(wxPackage::s_arrLocalRepo[i]);

    return success;
}

size_t wxPackageArray::RemoveDuplicates()
{
    size_t removed=0;

    for (size_t i=0; i<GetCount(); i++)
    {
        for (size_t j=i+1; j<GetCount(); j++)
        {
            // packages are uniquely identified by name & version:
            // two packages with both these properties identic are the same package !
            if (Item(i).HasSameIdOf(Item(j)))
            {
                RemoveAt(j);
                removed++;

                // check this item again
                j--;
            }
        }
    }

    return removed;
}

bool wxPackageArray::UpdatePackage(const wxPackage &pkg)
{
    int n = IndexById(pkg);
    if (n == wxNOT_FOUND)
        return false;

    //ItemById(pkg) = pkg;
    RemoveAt(n);
    Insert(pkg, n);
    return true;
}

bool wxPackageArray::GetRecursiveDependencies(wxPackageDependencyArray *ret, 
                                              const wxPackageArray &arr,
                                              wxPackageDependencyType type, 
                                              wxPackageDependencyArray *notfound) const
{
    bool success = true;
    for (size_t i=0; i<GetCount(); i++)
        success &= Item(i).GetRecursiveDependencies(ret, arr, type, notfound);

    return success;
}

wxPackageArray wxPackageArray::GetPackagesWhichBelongTo(wxPackageCategory cat) const
{
    wxPackageArray ret;
    for (size_t i=0; i<GetCount(); i++)
        if (Item(i).GetPrimaryCategory() == cat ||
            Item(i).GetSecondaryCategory() == cat)
            ret.Add(Item(i));
    return ret;
}

void wxPackageArray::InvalidateCachedSubstHashMap(wxPackageSubstituteInfoContext ctx)
{
    for (size_t i=0; i<GetCount(); i++)
        Item(i).InvalidateCachedSubstHashMap(ctx);
}

---