text: // cloner.cpp - copy a database (export/import basically)
  
  /**
  *    Copyright (C) 2008 10gen Inc.
  *
  *    This program is free software: you can redistribute it and/or  modify
  *    it under the terms of the GNU Affero General Public License, version 3,
  *    as published by the Free Software Foundation.
  *
  *    This program is distributed in the hope that it will be useful,
  *    but WITHOUT ANY WARRANTY; without even the implied warranty of
  *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  *    GNU Affero General Public License for more details.
  *
  *    You should have received a copy of the GNU Affero General Public License
  *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  *
  *    As a special exception, the copyright holders give permission to link the
  *    code of portions of this program with the OpenSSL library under certain
  *    conditions as described in each individual source file and distribute
  *    linked combinations including the program with the OpenSSL library. You
  *    must comply with the GNU Affero General Public License in all respects for
  *    all of the code used other than as permitted herein. If you modify file(s)
  *    with this exception, you may extend this exception to your version of the
  *    file(s), but you are not obligated to do so. If you do not wish to do so,
  *    delete this exception statement from your version. If you delete this
  *    exception statement from all source files in the program, then also delete
  *    it in the license file.
  */
  
  #define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kStorage
  
  #include "mongo/platform/basic.h"
  
  #include "mongo/db/cloner.h"
  
  #include <boost/scoped_ptr.hpp>
  
  #include "mongo/base/status.h"
  #include "mongo/bson/util/builder.h"
  #include "mongo/client/dbclientinterface.h"
  #include "mongo/db/auth/authorization_manager.h"
  #include "mongo/db/auth/authorization_manager_global.h"
  #include "mongo/db/auth/internal_user_auth.h"
  #include "mongo/db/catalog/collection.h"
  #include "mongo/db/catalog/database_holder.h"
  #include "mongo/db/catalog/index_create.h"
  #include "mongo/db/commands.h"
  #include "mongo/db/commands/copydb.h"
  #include "mongo/db/commands/rename_collection.h"
  #include "mongo/db/dbdirectclient.h"
  #include "mongo/db/dbhelpers.h"
  #include "mongo/db/global_environment_experiment.h"
  #include "mongo/db/index_builder.h"
  #include "mongo/db/jsobj.h"
  #include "mongo/db/namespace_string.h"
  #include "mongo/db/op_observer.h"
  #include "mongo/db/repl/isself.h"
  #include "mongo/db/repl/replication_coordinator_global.h"
  #include "mongo/db/server_parameters.h"
  #include "mongo/db/storage_options.h"
  #include "mongo/util/assert_util.h"
  #include "mongo/util/log.h"
  #include "mongo/util/mongoutils/str.h"
  
  namespace mongo {
  
      using boost::scoped_ptr;
      using std::auto_ptr;
      using std::list;
      using std::set;
      using std::endl;
      using std::string;
      using std::vector;
  
      MONGO_EXPORT_SERVER_PARAMETER(skipCorruptDocumentsWhenCloning, bool, false);
  
      BSONElement getErrField(const BSONObj& o);
  
      /* for index info object:
           { "name" : "name_1" , "ns" : "foo.index3" , "key" :  { "name" : 1.0 } }
         we need to fix up the value in the "ns" parameter so that the name prefix is correct on a
         copy to a new name.
      */
      BSONObj fixindex(const string& newDbName, BSONObj o) {
          BSONObjBuilder b;
          BSONObjIterator i(o);
          while ( i.moreWithEOO() ) {
              BSONElement e = i.next();
              if ( e.eoo() )
                  break;
  
              // for now, skip the "v" field so that v:0 indexes will be upgraded to v:1
              if ( string("v") == e.fieldName() ) {
                  continue;
              }
  
              if ( string("ns") == e.fieldName() ) {
                  uassert( 10024 , "bad ns field for index during dbcopy", e.type() == String);
                  const char *p = strchr(e.valuestr(), '.');
                  uassert( 10025 , "bad ns field for index during dbcopy [2]", p);
                  string newname = newDbName + p;
                  b.append("ns", newname);
              }
              else {
                  b.append(e);
              }
          }
  
          BSONObj res= b.obj();
  
          return res;
      }
  
      Cloner::Cloner() { }
  
      struct Cloner::Fun {
          Fun(OperationContext* txn, const string& dbName)
              :lastLog(0),
               txn(txn),
               _dbName(dbName)
          {}
  
          void operator()( DBClientCursorBatchIterator &i ) {
              invariant(from_collection.coll() != "system.indexes");
  
              // XXX: can probably take dblock instead
              scoped_ptr<ScopedTransaction> scopedXact(new ScopedTransaction(txn, MODE_X));
              scoped_ptr<Lock::GlobalWrite> globalWriteLock(new Lock::GlobalWrite(txn->lockState()));
              uassert(ErrorCodes::NotMaster,
                      str::stream() << "Not primary while cloning collection " << from_collection.ns()
                                    << " to " << to_collection.ns(),
                      !logForRepl ||
                      repl::getGlobalReplicationCoordinator()->canAcceptWritesForDatabase(_dbName));
  
              // Make sure database still exists after we resume from the temp release
              Database* db = dbHolder().openDb(txn, _dbName);
  
              bool createdCollection = false;
              Collection* collection = NULL;
  
              collection = db->getCollection( to_collection );
              if ( !collection ) {
                  massert( 17321,
                           str::stream()
                           << "collection dropped during clone ["
                           << to_collection.ns() << "]",
                           !createdCollection );
                  WriteUnitOfWork wunit(txn);
                  collection = db->createCollection(txn, to_collection.ns());
                  verify(collection);
  
                  if (logForRepl) {
                      getGlobalEnvironment()->getOpObserver()->onCreateCollection(
                              txn,
                              to_collection,
                              CollectionOptions());
                  }
                  wunit.commit();
              }
  
              while( i.moreInCurrentBatch() ) {
                  if ( numSeen % 128 == 127 ) {
                      time_t now = time(0);
                      if( now - lastLog >= 60 ) {
                          // report progress
                          if( lastLog )
                              log() << "clone " << to_collection << ' ' << numSeen << endl;
                          lastLog = now;
                      }
  
                      if (_mayBeInterrupted) {
                          txn->checkForInterrupt();
                      }
  
                      if (_mayYield) {
                          scopedXact.reset();
                          globalWriteLock.reset();
  
                          txn->getCurOp()->yielded();
  
                          scopedXact.reset(new ScopedTransaction(txn, MODE_X));
                          globalWriteLock.reset(new Lock::GlobalWrite(txn->lockState()));
  
                          // Check if everything is still all right.
                          if (logForRepl) {
                              uassert(28592,
                                      str::stream() << "Cannot write to db: " << _dbName
                                                    << " after yielding",
                                      repl::getGlobalReplicationCoordinator()->
                                          canAcceptWritesForDatabase(_dbName));
                          }
  
                          // TODO: SERVER-16598 abort if original db or collection is gone.
                          db = dbHolder().get(txn, _dbName);
                          uassert(28593,
                                  str::stream() << "Database " << _dbName
                                                << " dropped while cloning",
                                  db != NULL);
  
                          collection = db->getCollection(to_collection);
                          uassert(28594,
                                  str::stream() << "Collection " << to_collection.ns()
                                                << " dropped while cloning",
                                  collection != NULL);
                      }
                  }
  
                  BSONObj tmp = i.nextSafe();
  
                  /* assure object is valid.  note this will slow us down a little. */
                  const Status status = validateBSON(tmp.objdata(), tmp.objsize());
                  if (!status.isOK()) {
                      str::stream ss;
                      ss << "Cloner: found corrupt document in " << from_collection.toString()
                            << ": " << status.reason();
                      if (skipCorruptDocumentsWhenCloning) {
                          warning() << ss.ss.str() << "; skipping";
                          continue;
                      }
                      msgasserted(28531, ss);
                  }
  
                  ++numSeen;
                  WriteUnitOfWork wunit(txn);
  
                  BSONObj js = tmp;
  
                  StatusWith<RecordId> loc = collection->insertDocument( txn, js, true );
                  if ( !loc.isOK() ) {
                      error() << "error: exception cloning object in " << from_collection
                              << ' ' << loc.getStatus() << " obj:" << js;
                  }
                  uassertStatusOK( loc.getStatus() );
                  if (logForRepl)
                      getGlobalEnvironment()->getOpObserver()->onInsert(txn, to_collection.ns(), js);
  
                  wunit.commit();
  
                  RARELY if ( time( 0 ) - saveLast > 60 ) {
                      log() << numSeen << " objects cloned so far from collection " << from_collection;
                      saveLast = time( 0 );
                  }
              }
          }
  
          time_t lastLog;
          OperationContext* txn;
          const string _dbName;
  
          int64_t numSeen;
          NamespaceString from_collection;
          NamespaceString to_collection;
          time_t saveLast;
          bool logForRepl;
          bool _mayYield;
          bool _mayBeInterrupted;
      };
  
      /* copy the specified collection
      */
      void Cloner::copy(OperationContext* txn,
                        const string& toDBName,
                        const NamespaceString& from_collection,
                        const NamespaceString& to_collection,
                        bool logForRepl,
                        bool masterSameProcess,
                        bool slaveOk,
                        bool mayYield,
                        bool mayBeInterrupted,
                        Query query) {
          LOG(2) << "\t\tcloning collection " << from_collection << " to " << to_collection << " on " << _conn->getServerAddress() << " with filter " << query.toString() << endl;
  
          Fun f(txn, toDBName);
          f.numSeen = 0;
          f.from_collection = from_collection;
          f.to_collection = to_collection;
          f.saveLast = time( 0 );
          f.logForRepl = logForRepl;
          f._mayYield = mayYield;
          f._mayBeInterrupted = mayBeInterrupted;
  
          int options = QueryOption_NoCursorTimeout | ( slaveOk ? QueryOption_SlaveOk : 0 );
          {
              Lock::TempRelease tempRelease(txn->lockState());
              _conn->query(stdx::function<void(DBClientCursorBatchIterator &)>(f), from_collection,
                           query, 0, options);
          }
  
          uassert(ErrorCodes::NotMaster,
                  str::stream() << "Not primary while cloning collection " << from_collection.ns()
                                << " to " << to_collection.ns() << " with filter "
                                << query.toString(),
                  !logForRepl ||
                  repl::getGlobalReplicationCoordinator()->canAcceptWritesForDatabase(toDBName));
      }
  
      void Cloner::copyIndexes(OperationContext* txn,
                               const string& toDBName,
                               const NamespaceString& from_collection,
                               const NamespaceString& to_collection,
                               bool logForRepl,
                               bool masterSameProcess,
                               bool slaveOk,
                               bool mayYield,
                               bool mayBeInterrupted) {
  
          LOG(2) << "\t\t copyIndexes " << from_collection << " to " << to_collection
                 << " on " << _conn->getServerAddress();
  
          vector<BSONObj> indexesToBuild;
  
          {
              Lock::TempRelease tempRelease(txn->lockState());
              list<BSONObj> sourceIndexes = _conn->getIndexSpecs( from_collection,
                                                                  slaveOk ? QueryOption_SlaveOk : 0 );
              for (list<BSONObj>::const_iterator it = sourceIndexes.begin();
                      it != sourceIndexes.end(); ++it) {
                  indexesToBuild.push_back(fixindex(to_collection.db().toString(), *it));
              }
          }
  
          uassert(ErrorCodes::NotMaster,
                  str::stream() << "Not primary while copying indexes from " << from_collection.ns()
                                << " to " << to_collection.ns() << " (Cloner)",
                  !logForRepl ||
                  repl::getGlobalReplicationCoordinator()->canAcceptWritesForDatabase(toDBName));
  
  
          if (indexesToBuild.empty())
              return;
  
          // We are under lock here again, so reload the database in case it may have disappeared
          // during the temp release
          Database* db = dbHolder().openDb(txn, toDBName);
  
          Collection* collection = db->getCollection( to_collection );
          if ( !collection ) {
              WriteUnitOfWork wunit(txn);
              collection = db->createCollection( txn, to_collection.ns() );
              invariant(collection);
              if (logForRepl) {
                  getGlobalEnvironment()->getOpObserver()->onCreateCollection(txn,
                                                                              to_collection,
                                                                              CollectionOptions());
              }
              wunit.commit();
          }
  
          // TODO pass the MultiIndexBlock when inserting into the collection rather than building the
          // indexes after the fact. This depends on holding a lock on the collection the whole time
          // from creation to completion without yielding to ensure the index and the collection
          // matches. It also wouldn't work on non-empty collections so we would need both
          // implementations anyway as long as that is supported.
          MultiIndexBlock indexer(txn, collection);
          if (mayBeInterrupted)
              indexer.allowInterruption();
  
          indexer.removeExistingIndexes(&indexesToBuild);
          if (indexesToBuild.empty())
              return;
  
          uassertStatusOK(indexer.init(indexesToBuild));
          uassertStatusOK(indexer.insertAllDocumentsInCollection());
  
          WriteUnitOfWork wunit(txn);
          indexer.commit();
          if (logForRepl) {
              const string targetSystemIndexesCollectionName =
                  to_collection.getSystemIndexesCollection();
              const char* createIndexNs = targetSystemIndexesCollectionName.c_str();
              for (vector<BSONObj>::const_iterator it = indexesToBuild.begin();
                      it != indexesToBuild.end(); ++it) {
                  getGlobalEnvironment()->getOpObserver()->onInsert(txn, createIndexNs, *it);
              }
          }
          wunit.commit();
      }
  
      bool Cloner::copyCollection(OperationContext* txn,
                                  const string& ns,
                                  const BSONObj& query,
                                  string& errmsg,
                                  bool mayYield,
                                  bool mayBeInterrupted,
                                  bool shouldCopyIndexes,
                                  bool logForRepl) {
  
          const NamespaceString nss(ns);
          const string dbname = nss.db().toString();
  
          ScopedTransaction transaction(txn, MODE_IX);
          Lock::DBLock dbWrite(txn->lockState(), dbname, MODE_X);
  
          uassert(ErrorCodes::NotMaster,
                  str::stream() << "Not primary while copying collection " << ns << " (Cloner)",
                  !logForRepl ||
                  repl::getGlobalReplicationCoordinator()->canAcceptWritesForDatabase(dbname));
  
          Database* db = dbHolder().openDb(txn, dbname);
  
          // config
          BSONObj filter = BSON("name" << nss.coll().toString());
          list<BSONObj> collList = _conn->getCollectionInfos( dbname, filter);
          if (!collList.empty()) {
              invariant(collList.size() <= 1);
              BSONObj col = collList.front();
              if (col["options"].isABSONObj()) {
                  WriteUnitOfWork wunit(txn);
                  Status status = userCreateNS(txn, db, ns, col["options"].Obj(), logForRepl, 0);
                  if ( !status.isOK() ) {
                      errmsg = status.toString();
                      return false;
                  }
                  wunit.commit();
              }
          }
  
          // main data
          copy(txn, dbname,
               nss, nss,
               logForRepl, false, true, mayYield, mayBeInterrupted,
               Query(query).snapshot());
  
          /* TODO : copyIndexes bool does not seem to be implemented! */
          if(!shouldCopyIndexes) {
              log() << "ERROR copy collection shouldCopyIndexes not implemented? " << ns << endl;
          }
  
          // indexes
          copyIndexes(txn, dbname,
                      NamespaceString(ns), NamespaceString(ns),
                      logForRepl, false, true, mayYield,
                      mayBeInterrupted);
  
          return true;
      }
  
      bool Cloner::go(OperationContext* txn,
                      const std::string& toDBName,
                      const string& masterHost,
                      const CloneOptions& opts,
                      set<string>* clonedColls,
                      string& errmsg,
                      int* errCode) {
  
          if ( errCode ) {
              *errCode = 0;
          }
          massert( 10289 ,  "useReplAuth is not written to replication log", !opts.useReplAuth || !opts.logForRepl );
  
          const ConnectionString cs = ConnectionString::parse(masterHost, errmsg);
          if (!cs.isValid()) {
              if (errCode)
                  *errCode = ErrorCodes::FailedToParse;
              return false;
          }
  
          bool masterSameProcess = false;
          std::vector<HostAndPort> csServers = cs.getServers();
          for (std::vector<HostAndPort>::const_iterator iter = csServers.begin();
               iter != csServers.end(); ++iter) {
  
              if (!repl::isSelf(*iter))
                  continue;
  
              masterSameProcess = true;
              break;
          }
  
          if (masterSameProcess) {
              if (opts.fromDB == toDBName) {
                  // guard against an "infinite" loop
                  /* if you are replicating, the local.sources config may be wrong if you get this */
                  errmsg = "can't clone from self (localhost).";
                  return false;
              }
          }
  
          {
              // setup connection
              if (_conn.get()) {
                  // nothing to do
              }
              else if ( !masterSameProcess ) {
                  auto_ptr<DBClientBase> con( cs.connect( errmsg ));
                  if ( !con.get() )
                      return false;
                  if (getGlobalAuthorizationManager()->isAuthEnabled() &&
                      !authenticateInternalUser(con.get())) {
  
                      return false;
                  }
  
                  _conn = con;
              }
              else {
                  _conn.reset(new DBDirectClient(txn));
              }
          }
  
          // Gather the list of collections to clone
          list<BSONObj> toClone;
          if (clonedColls) {
              clonedColls->clear();
          }
  
          {
              // getCollectionInfos may make a remote call, which may block indefinitely, so release
              // the global lock that we are entering with.
              Lock::TempRelease tempRelease(txn->lockState());
  
              list<BSONObj> raw = _conn->getCollectionInfos( opts.fromDB );
              for ( list<BSONObj>::iterator it = raw.begin(); it != raw.end(); ++it ) {
                  BSONObj collection = *it;
  
                  LOG(2) << "\t cloner got " << collection << endl;
  
                  BSONElement collectionOptions = collection["options"];
                  if ( collectionOptions.isABSONObj() ) {
                      Status parseOptionsStatus = CollectionOptions().parse(collectionOptions.Obj());
                      if ( !parseOptionsStatus.isOK() ) {
                          errmsg = str::stream() << "invalid collection options: " << collection
                                                 << ", reason: " << parseOptionsStatus.reason();
                          return false;
                      }
                  }
  
                  BSONElement e = collection.getField("name");
                  if ( e.eoo() ) {
                      string s = "bad collection object " + collection.toString();
                      massert( 10290 , s.c_str(), false);
                  }
                  verify( !e.eoo() );
                  verify( e.type() == String );
  
                  const NamespaceString ns(opts.fromDB, e.valuestr());
  
                  if( ns.isSystem() ) {
                      /* system.users and s.js is cloned -- but nothing else from system.
                       * system.indexes is handled specially at the end*/
                      if( legalClientSystemNS( ns.ns() , true ) == 0 ) {
                          LOG(2) << "\t\t not cloning because system collection" << endl;
                          continue;
                      }
                  }
                  if( !ns.isNormal() ) {
                      LOG(2) << "\t\t not cloning because has $ ";
                      continue;
                  }
  
                  if( opts.collsToIgnore.find( ns.ns() ) != opts.collsToIgnore.end() ){
                      LOG(2) << "\t\t ignoring collection " << ns;
                      continue;
                  }
                  else {
                      LOG(2) << "\t\t not ignoring collection " << ns;
                  }
  
                  if (clonedColls) {
                      clonedColls->insert(ns.ns());
                  }
  
                  toClone.push_back( collection.getOwned() );
              }
          }
  
          uassert(ErrorCodes::NotMaster,
                  str::stream() << "Not primary while cloning database " << opts.fromDB
                                << " (after getting list of collections to clone)",
                  !opts.logForRepl ||
                  repl::getGlobalReplicationCoordinator()->canAcceptWritesForDatabase(toDBName));
  
          if ( opts.syncData ) {
              for ( list<BSONObj>::iterator i=toClone.begin(); i != toClone.end(); i++ ) {
                  BSONObj collection = *i;
                  LOG(2) << "  really will clone: " << collection << endl;
                  const char* collectionName = collection["name"].valuestr();
                  BSONObj options = collection.getObjectField("options");
  
                  const NamespaceString from_name(opts.fromDB, collectionName);
                  const NamespaceString to_name(toDBName, collectionName);
  
                  Database* db = dbHolder().openDb(txn, toDBName);
  
                  {
                      WriteUnitOfWork wunit(txn);
  
                      // we defer building id index for performance - building it in batch is much
                      // faster
                      Status createStatus = userCreateNS(txn,
                                                         db,
                                                         to_name.ns(),
                                                         options,
                                                         opts.logForRepl,
                                                         false);
                      if ( !createStatus.isOK() ) {
                          errmsg = str::stream() << "failed to create collection \""
                                                 << to_name.ns() << "\": "
                                                 << createStatus.reason();
                          return false;
                      }
  
                      wunit.commit();
                  }
  
                  LOG(1) << "\t\t cloning " << from_name << " -> " << to_name << endl;
                  Query q;
                  if( opts.snapshot )
                      q.snapshot();
  
                  copy(txn,
                       toDBName,
                       from_name,
                       to_name,
                       opts.logForRepl,
                       masterSameProcess,
                       opts.slaveOk,
                       opts.mayYield,
                       opts.mayBeInterrupted,
                       q);
  
                  // Copy releases the lock, so we need to re-load the database. This should
                  // probably throw if the database has changed in between, but for now preserve
                  // the existing behaviour.
                  db = dbHolder().get(txn, toDBName);
                  uassert(18645,
                          str::stream() << "database " << toDBName << " dropped during clone",
                          db);
  
                  Collection* c = db->getCollection( to_name );
                  if ( c && !c->getIndexCatalog()->haveIdIndex( txn ) ) {
                      // We need to drop objects with duplicate _ids because we didn't do a true
                      // snapshot and this is before applying oplog operations that occur during the
                      // initial sync.
                      set<RecordId> dups;
  
                      MultiIndexBlock indexer(txn, c);
                      if (opts.mayBeInterrupted)
                          indexer.allowInterruption();
  
                      uassertStatusOK(indexer.init(c->getIndexCatalog()->getDefaultIdIndexSpec()));
                      uassertStatusOK(indexer.insertAllDocumentsInCollection(&dups));
  
                      // This must be done before we commit the indexer. See the comment about
                      // dupsAllowed in IndexCatalog::_unindexRecord and SERVER-17487.
                      for (set<RecordId>::const_iterator it = dups.begin(); it != dups.end(); ++it) {
                          WriteUnitOfWork wunit(txn);
                          BSONObj id;
  
                          c->deleteDocument(txn, *it, true, true, opts.logForRepl ? &id : NULL);
                          if (opts.logForRepl)
                              getGlobalEnvironment()->getOpObserver()->onDelete(txn,
                                                                                c->ns().ns(),
                                                                                id);
                          wunit.commit();
                      }
  
                      if (!dups.empty()) {
                          log() << "index build dropped: " << dups.size() << " dups";
                      }
  
                      WriteUnitOfWork wunit(txn);
                      indexer.commit();
                      if (opts.logForRepl) {
                          getGlobalEnvironment()->getOpObserver()->onCreateIndex(
                                  txn,
                                  c->ns().getSystemIndexesCollection().c_str(),
                                  c->getIndexCatalog()->getDefaultIdIndexSpec());
                      }
                      wunit.commit();
                  }
              }
          }
  
          // now build the secondary indexes
          if ( opts.syncIndexes ) {
              for ( list<BSONObj>::iterator i=toClone.begin(); i != toClone.end(); i++ ) {
                  BSONObj collection = *i;
                  log() << "copying indexes for: " << collection;
  
                  const char* collectionName = collection["name"].valuestr();
  
                  NamespaceString from_name( opts.fromDB, collectionName );
                  NamespaceString to_name( toDBName, collectionName );
  
                  copyIndexes(txn,
                              toDBName,
                              from_name,
                              to_name,
                              opts.logForRepl,
                              masterSameProcess,
                              opts.slaveOk,
                              opts.mayYield,
                              opts.mayBeInterrupted );
              }
          }
  
          return true;
      }
  
  } // namespace mongo
  