text: var fs = require('fs');
  var path = require('path');
  var semver = require('semver');
  var _ = require('underscore');
  var packageClient = require('./package-client.js');
  var archinfo = require('./archinfo.js');
  var Unipackage = require('./unipackage.js').Unipackage;
  var compiler = require('./compiler.js');
  var buildmessage = require('./buildmessage.js');
  var tropohouse = require('./tropohouse.js');
  var watch = require('./watch.js');
  var files = require('./files.js');
  var utils = require('./utils.js');
  
  var baseCatalog = exports;
  
  var catalog = require('./catalog.js');
  
  // This is a basic catalog class. It accesses basic catalog data by looking
  // through the catalog's collections.
  //
  // YOU MUST SET self.initialized = true BEFORE USING THIS CATALOG. In fact, the
  // protolog is not even intended to be used by itself -- there is a server
  // catalog and a constraint solving catalog, which inherit from it.
  baseCatalog.BaseCatalog = function () {
    var self = this;
  
    // Package server data. Mostly arrays of objects.
    self.packages = null;
    self.versions = null;  // package name -> version -> object
    self.builds = null;
  
    // We use the initialization design pattern because it makes it easier to use
    // both of our catalogs as singletons.
    self.initialized = false;
  
  };
  
  _.extend(baseCatalog.BaseCatalog.prototype, {
    // Set all the collections to their initial values, which are mostly
    // blank. This does not set self.initialized -- do that manually in the child
    // class when applicable.
    reset: function () {
      var self = this;
  
      // Initialize everything to its default version.
      self.packages = [];
      self.versions = {};
      self.builds = [];
    },
  
    // Throw if the catalog's self.initialized value has not been set to true.
    _requireInitialized: function () {
      var self = this;
  
      if (! self.initialized)
        throw new Error("catalog not initialized yet?");
    },
  
    // serverPackageData is a description of the packages available from
    // the package server, as returned by
    // packageClient.loadPackageData. Add all of those packages to the
    // catalog without checking for duplicates.
    _insertServerPackages: function (serverPackageData) {
      var self = this;
  
      var collections = serverPackageData.collections;
  
      if (!collections)
        return;
  
      _.each(
        ['packages', 'builds'],
        function (field) {
          self[field].push.apply(self[field], collections[field]);
        });
  
      // Convert versions from array format to nested object format.
      _.each(collections.versions, function (record) {
        if (!_.has(self.versions, record.packageName)) {
          self.versions[record.packageName] = {};
        }
        self.versions[record.packageName][record.version] = record;
      });
    },
  
    // Accessor methods below. The primary function of both catalogs is to provide
    // data about the existence of various packages, versions, etc, which it does
    // through these methods.
  
    // We have a pattern, where we try to retrieve something and if we fail, call
    // refresh to check if there is new data. That makes sense to me, so let's do
    // that all the time. (In the future, we should continue some rate-limiting
    // on refresh, but for now, most of the time, refreshing on an unknown <stuff>
    // will just cause us to crash if it doesn't exist, so we are just delaying
    // the inevitable rather than slowing down normal operations)
    _recordOrRefresh: function (recordFinder) {
      var self = this;
      buildmessage.assertInCapture();
      var record = recordFinder();
      // If we cannot find it maybe refresh.
      if (!record && self._refreshingIsProductive()) {
        if (! catalog.official.refreshInProgress()) {
          catalog.official.refresh();
        }
        record = recordFinder();
      }
      // If we still cannot find it, give the user a null.
      if (!record) {
        return null;
      }
      return record;
    },
  
    _refreshingIsProductive: function () {
      // Refreshing is productive for catalog.official and catalog.complete only.
      return false;
    },
  
    // Return an array with the names of all of the packages that we
    // know about, in no particular order.
    getAllPackageNames: function () {
      var self = this;
      self._requireInitialized();
  
      return _.pluck(self.packages, 'name');
    },
  
    // Returns general (non-version-specific) information about a
    // package, or null if there is no such package.
    getPackage: function (name, options) {
      var self = this;
      buildmessage.assertInCapture();
      self._requireInitialized();
      options = options || {};
  
      var get = function () {
        return _.findWhere(self.packages, { name: name });
      };
  
      return options.noRefresh ? get() : self._recordOrRefresh(get);
    },
  
    // Given a package, returns an array of the versions available for
    // this package (for any architecture), sorted from oldest to newest
    // (according to the version string, not according to their
    // publication date). Returns the empty array if the package doesn't
    // exist or doesn't have any versions.
    getSortedVersions: function (name) {
      var self = this;
      self._requireInitialized();
      if (!_.has(self.versions, name)) {
        return [];
      }
      var ret = _.keys(self.versions[name]);
      ret.sort(semver.compare);
      return ret;
    },
  
    // Return information about a particular version of a package, or
    // null if there is no such package or version.
    getVersion: function (name, version) {
      var self = this;
      self._requireInitialized();
      buildmessage.assertInCapture();
  
      var lookupVersion = function () {
        return _.has(self.versions, name) &&
          _.has(self.versions[name], version) &&
          self.versions[name][version];
      };
  
      // The catalog doesn't understand buildID versions and doesn't know about
      // them. Depending on when we build them, we can refer to local packages as
      // 1.0.0+local or 1.0.0+[buildId]. Luckily, we know which packages are
      // local, so just look those up by their local version instead.
      // XXX ideally we'd only have isLocalPackage in the complete catalog and
      //     have CompleteCatalog override getVersion, but other things want
      //     to call isLocalPackage, eg maybeDownloadPackageForArchitectures
      //     which has the official package when running make-bootstrap-tarballs
      if (self.isLocalPackage(name)) {
        version = self._getLocalVersion(version);
        // No need to refresh here: if we can't find the local version, refreshing
        // isn't going to help!
        return lookupVersion() || null;
      }
  
      return self._recordOrRefresh(lookupVersion);
    },
  
    // Overridden by CompleteCatalog.
    // XXX this is kinda sketchy, maybe callers should only call this
    //     on the CompleteCatalog?
    isLocalPackage: function () {
      return false;
    },
  
    // As getVersion, but returns info on the latest version of the
    // package, or null if the package doesn't exist or has no versions.
    // It does not include prereleases (with dashes in the version);
    getLatestMainlineVersion: function (name) {
      var self = this;
      self._requireInitialized();
      buildmessage.assertInCapture();
  
      var versions = self.getSortedVersions(name);
      versions.reverse();
      var latest = _.find(versions, function (version) {
        return !/-/.test(version);
      });
      if (!latest)
        return null;
      return self.getVersion(name, latest);
    },
  
    // If this package has any builds at this version, return an array of builds
    // which cover all of the required arches, or null if it is impossible to
    // cover them all (or if the version does not exist).
    getBuildsForArches: function (name, version, arches) {
      var self = this;
      self._requireInitialized();
      buildmessage.assertInCapture();
  
      var versionInfo = self.getVersion(name, version);
      if (! versionInfo)
        return null;
  
      // XXX if we have a choice between os and os.mac, this returns a random one.
      //     so in practice we don't really support "maybe-platform-specific"
      //     packages
  
      // Even though getVersion already has its own _recordOrRefresh, we need this
      // one, in case our local cache says "version exists but only for the wrong
      // arch" and the right arch has been recently published.
      // XXX should ensure at most one refresh
      return self._recordOrRefresh(function () {
        var allBuilds = _.where(self.builds, { versionId: versionInfo._id });
        var solution = null;
        utils.generateSubsetsOfIncreasingSize(allBuilds, function (buildSubset) {
          // This build subset works if for all the arches we need, at least one
          // build in the subset satisfies it. It is guaranteed to be minimal,
          // because we look at subsets in increasing order of size.
          var satisfied = _.all(arches, function (neededArch) {
            return _.any(buildSubset, function (build) {
              var buildArches = build.buildArchitectures.split('+');
              return !!archinfo.mostSpecificMatch(neededArch, buildArches);
            });
          });
          if (satisfied) {
            solution = buildSubset;
            return true;  // stop the iteration
          }
        });
        return solution;  // might be null!
      });
    },
  
    // Unlike the previous, this looks for a build which *precisely* matches the
    // given buildArchitectures string. Also, it takes a versionRecord rather than
    // name/version.
    getBuildWithPreciseBuildArchitectures: function (versionRecord,
                                                     buildArchitectures) {
      var self = this;
      buildmessage.assertInCapture();
      self._requireInitialized();
  
      return self._recordOrRefresh(function () {
        return _.findWhere(self.builds,
                           { versionId: versionRecord._id,
                             buildArchitectures: buildArchitectures });
      });
    },
  
    getAllBuilds: function (name, version) {
      var self = this;
      self._requireInitialized();
      buildmessage.assertInCapture();
  
      var versionRecord = self.getVersion(name, version);
      if (!versionRecord)
        return null;
  
      return _.where(self.builds, { versionId: versionRecord._id });
    },
  
    // Reload catalog data to account for new information if needed.
    refresh: function () {
      throw new Error("no such thing as a base refresh");
    }
  });
  