text: /**
   * @file sync.cpp
   * @brief Class for synchronizing local and remote trees
   *
   * (c) 2013-2014 by Mega Limited, Auckland, New Zealand
   *
   * This file is part of the MEGA SDK - Client Access Engine.
   *
   * Applications using the MEGA API must present a valid application key
   * and comply with the the rules set forth in the Terms of Service.
   *
   * The MEGA SDK is distributed in the hope that it will be useful,
   * but WITHOUT ANY WARRANTY; without even the implied warranty of
   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
   *
   * @copyright Simplified (2-clause) BSD License.
   *
   * You should have received a copy of the license along with this
   * program.
   */
  #include <cctype>
  #include <type_traits>
  #include <unordered_set>
  
  #include "mega.h"
  
  #ifdef ENABLE_SYNC
  #include "mega/sync.h"
  #include "mega/megaapp.h"
  #include "mega/transfer.h"
  #include "mega/megaclient.h"
  #include "mega/base64.h"
  #include "mega/heartbeats.h"
  
  namespace mega {
  
  const int Sync::SCANNING_DELAY_DS = 5;
  const int Sync::EXTRA_SCANNING_DELAY_DS = 150;
  const int Sync::FILE_UPDATE_DELAY_DS = 30;
  const int Sync::FILE_UPDATE_MAX_DELAY_SECS = 60;
  const dstime Sync::RECENT_VERSION_INTERVAL_SECS = 10800;
  
  namespace {
  
  // Need this to store `LightFileFingerprint` by-value in `FingerprintSet`
  struct LightFileFingerprintComparator
  {
      bool operator()(const LightFileFingerprint& lhs, const LightFileFingerprint& rhs) const
      {
          return LightFileFingerprintCmp{}(&lhs, &rhs);
      }
  };
  
  // Represents a file/folder for use in assigning fs IDs
  struct FsFile
  {
      handle fsid;
      LocalPath path;
  };
  
  // Caches fingerprints
  class FingerprintCache
  {
  public:
      using FingerprintSet = std::set<LightFileFingerprint, LightFileFingerprintComparator>;
  
      // Adds a new fingerprint
      template<typename T, typename = typename std::enable_if<std::is_same<LightFileFingerprint, typename std::decay<T>::type>::value>::type>
      const LightFileFingerprint* add(T&& ffp)
      {
           const auto insertPair = mFingerprints.insert(std::forward<T>(ffp));
           return &*insertPair.first;
      }
  
      // Returns the set of all fingerprints
      const FingerprintSet& all() const
      {
          return mFingerprints;
      }
  
  private:
      FingerprintSet mFingerprints;
  };
  
  using FingerprintLocalNodeMap = std::multimap<const LightFileFingerprint*, LocalNode*, LightFileFingerprintCmp>;
  using FingerprintFileMap = std::multimap<const LightFileFingerprint*, FsFile, LightFileFingerprintCmp>;
  
  // Collects all syncable filesystem paths in the given folder under `localpath`
  set<LocalPath> collectAllPathsInFolder(Sync& sync, MegaApp& app, FileSystemAccess& fsaccess, LocalPath& localpath,
                                      LocalPath& localdebris)
  {
      auto fa = fsaccess.newfileaccess(false);
      if (!fa->fopen(localpath, true, false))
      {
          LOG_err << "Unable to open path: " << localpath.toPath(fsaccess);
          return {};
      }
      if (fa->mIsSymLink)
      {
          LOG_debug << "Ignoring symlink: " << localpath.toPath(fsaccess);
          return {};
      }
      assert(fa->type == FOLDERNODE);
  
      auto da = std::unique_ptr<DirAccess>{fsaccess.newdiraccess()};
      if (!da->dopen(&localpath, fa.get(), false))
      {
          LOG_err << "Unable to open directory: " << localpath.toPath(fsaccess);
          return {};
      }
  
      set<LocalPath> paths; // has to be a std::set to enforce same sorting as `children` of `LocalNode`
  
      LocalPath localname;
      while (da->dnext(localpath, localname, false))
      {
          ScopedLengthRestore restoreLength(localpath);
          localpath.appendWithSeparator(localname, false);
  
          // check if this record is to be ignored
          const auto name = localname.toName(fsaccess, sync.mFilesystemType);
          if (app.sync_syncable(&sync, name.c_str(), localpath))
          {
              // skip the sync's debris folder
              if (!localdebris.isContainingPathOf(localpath))
              {
                  paths.insert(localpath);
              }
          }
      }
  
      return paths;
  }
  
  // Combines another fingerprint into `ffp`
  void hashCombineFingerprint(LightFileFingerprint& ffp, const LightFileFingerprint& other)
  {
      hashCombine(ffp.size, other.size);
      hashCombine(ffp.mtime, other.mtime);
  }
  
  // Combines the fingerprints of all file nodes in the given map
  bool combinedFingerprint(LightFileFingerprint& ffp, const localnode_map& nodeMap)
  {
      bool success = false;
      for (const auto& nodePair : nodeMap)
      {
          const LocalNode& l = *nodePair.second;
          if (l.type == FILENODE)
          {
              LightFileFingerprint lFfp;
              lFfp.genfingerprint(l.size, l.mtime);
              hashCombineFingerprint(ffp, lFfp);
              success = true;
          }
      }
      return success;
  }
  
  // Combines the fingerprints of all files in the given paths
  bool combinedFingerprint(LightFileFingerprint& ffp, FileSystemAccess& fsaccess, const set<LocalPath>& paths)
  {
      bool success = false;
      for (auto& path : paths)
      {
          auto fa = fsaccess.newfileaccess(false);
          auto pathArg = path; // todo: sort out const
          if (!fa->fopen(pathArg, true, false))
          {
              LOG_err << "Unable to open path: " << path.toPath(fsaccess);
              success = false;
              break;
          }
          if (fa->mIsSymLink)
          {
              LOG_debug << "Ignoring symlink: " << path.toPath(fsaccess);
              continue;
          }
          if (fa->type == FILENODE)
          {
              LightFileFingerprint faFfp;
              faFfp.genfingerprint(fa->size, fa->mtime);
              hashCombineFingerprint(ffp, faFfp);
              success = true;
          }
      }
      return success;
  }
  
  // Computes the fingerprint of the given `l` (file or folder) and stores it in `ffp`
  bool computeFingerprint(LightFileFingerprint& ffp, const LocalNode& l)
  {
      if (l.type == FILENODE)
      {
          ffp.genfingerprint(l.size, l.mtime);
          return true;
      }
      else if (l.type == FOLDERNODE)
      {
          return combinedFingerprint(ffp, l.children);
      }
      else
      {
          assert(false && "Invalid node type");
          return false;
      }
  }
  
  // Computes the fingerprint of the given `fa` (file or folder) and stores it in `ffp`
  bool computeFingerprint(LightFileFingerprint& ffp, FileSystemAccess& fsaccess,
                          FileAccess& fa, LocalPath& path, const set<LocalPath>& paths)
  {
      if (fa.type == FILENODE)
      {
          assert(paths.empty());
          ffp.genfingerprint(fa.size, fa.mtime);
          return true;
      }
      else if (fa.type == FOLDERNODE)
      {
          return combinedFingerprint(ffp, fsaccess, paths);
      }
      else
      {
          assert(false && "Invalid node type");
          return false;
      }
  }
  
  // Collects all `LocalNode`s by storing them in `localnodes`, keyed by LightFileFingerprint.
  // Invalidates the fs IDs of all local nodes.
  // Stores all fingerprints in `fingerprints` for later reference.
  void collectAllLocalNodes(FingerprintCache& fingerprints, FingerprintLocalNodeMap& localnodes,
                            LocalNode& l, handlelocalnode_map& fsidnodes)
  {
      // invalidate fsid of `l`
      l.fsid = mega::UNDEF;
      if (l.fsid_it != fsidnodes.end())
      {
          fsidnodes.erase(l.fsid_it);
          l.fsid_it = fsidnodes.end();
      }
      // collect fingerprint
      LightFileFingerprint ffp;
      if (computeFingerprint(ffp, l))
      {
          const auto ffpPtr = fingerprints.add(std::move(ffp));
          localnodes.insert(std::make_pair(ffpPtr, &l));
      }
      if (l.type == FILENODE)
      {
          return;
      }
      for (auto& childPair : l.children)
      {
          collectAllLocalNodes(fingerprints, localnodes, *childPair.second, fsidnodes);
      }
  }
  
  // Collects all `File`s by storing them in `files`, keyed by FileFingerprint.
  // Stores all fingerprints in `fingerprints` for later reference.
  void collectAllFiles(bool& success, FingerprintCache& fingerprints, FingerprintFileMap& files,
                       Sync& sync, MegaApp& app, FileSystemAccess& fsaccess, LocalPath& localpath,
                       LocalPath& localdebris)
  {
      auto insertFingerprint = [&files, &fingerprints](FileSystemAccess& fsaccess, FileAccess& fa,
                                                       LocalPath& path, const set<LocalPath>& paths)
      {
          LightFileFingerprint ffp;
          if (computeFingerprint(ffp, fsaccess, fa, path, paths))
          {
              const auto ffpPtr = fingerprints.add(std::move(ffp));
              files.insert(std::make_pair(ffpPtr, FsFile{fa.fsid, path}));
          }
      };
  
      auto fa = fsaccess.newfileaccess(false);
      if (!fa->fopen(localpath, true, false))
      {
          LOG_err << "Unable to open path: " << localpath.toPath(fsaccess);
          success = false;
          return;
      }
      if (fa->mIsSymLink)
      {
          LOG_debug << "Ignoring symlink: " << localpath.toPath(fsaccess);
          return;
      }
      if (!fa->fsidvalid)
      {
          LOG_err << "Invalid fs id for: " << localpath.toPath(fsaccess);
          success = false;
          return;
      }
  
      if (fa->type == FILENODE)
      {
          insertFingerprint(fsaccess, *fa, localpath, {});
      }
      else if (fa->type == FOLDERNODE)
      {
          const auto paths = collectAllPathsInFolder(sync, app, fsaccess, localpath, localdebris);
          insertFingerprint(fsaccess, *fa, localpath, paths);
          fa.reset();
          for (const auto& path : paths)
          {
              LocalPath tmpPath = path;
              collectAllFiles(success, fingerprints, files, sync, app, fsaccess, tmpPath, localdebris);
          }
      }
      else
      {
          assert(false && "Invalid file type");
          success = false;
          return;
      }
  }
  
  // Assigns fs IDs from `files` to those `localnodes` that match the fingerprints found in `files`.
  // If there are multiple matches we apply a best-path heuristic.
  size_t assignFilesystemIdsImpl(const FingerprintCache& fingerprints, FingerprintLocalNodeMap& localnodes,
                                 FingerprintFileMap& files, handlelocalnode_map& fsidnodes, FileSystemAccess& fsaccess)
  {
      LocalPath nodePath;
      size_t assignmentCount = 0;
      for (const auto& fp : fingerprints.all())
      {
          const auto nodeRange = localnodes.equal_range(&fp);
          const auto nodeCount = std::distance(nodeRange.first, nodeRange.second);
          if (nodeCount <= 0)
          {
              continue;
          }
  
          const auto fileRange = files.equal_range(&fp);
          const auto fileCount = std::distance(fileRange.first, fileRange.second);
          if (fileCount <= 0)
          {
              // without files we cannot assign fs IDs to these localnodes, so no need to keep them
              localnodes.erase(nodeRange.first, nodeRange.second);
              continue;
          }
  
          struct Element
          {
              int score;
              handle fsid;
              LocalNode* l;
          };
          std::vector<Element> elements;
          elements.reserve(nodeCount * fileCount);
  
          for (auto nodeIt = nodeRange.first; nodeIt != nodeRange.second; ++nodeIt)
          {
              auto l = nodeIt->second;
              if (l != l->sync->localroot.get()) // never assign fs ID to the root localnode
              {
                  nodePath = l->getLocalPath();
                  for (auto fileIt = fileRange.first; fileIt != fileRange.second; ++fileIt)
                  {
                      auto& filePath = fileIt->second.path;
                      const auto score = computeReversePathMatchScore(nodePath, filePath, fsaccess);
                      if (score > 0) // leaf name must match
                      {
                          elements.push_back({score, fileIt->second.fsid, l});
                      }
                  }
              }
          }
  
          // Sort in descending order by score. Elements with highest score come first
          std::sort(elements.begin(), elements.end(), [](const Element& e1, const Element& e2)
                                                      {
                                                          return e1.score > e2.score;
                                                      });
  
          std::unordered_set<handle> usedFsIds;
          for (const auto& e : elements)
          {
              if (e.l->fsid == mega::UNDEF // node not assigned
                  && usedFsIds.find(e.fsid) == usedFsIds.end()) // fsid not used
              {
                  e.l->setfsid(e.fsid, fsidnodes);
                  usedFsIds.insert(e.fsid);
                  ++assignmentCount;
              }
          }
  
          // the fingerprint that these files and localnodes correspond to has now finished processing
          files.erase(fileRange.first, fileRange.second);
          localnodes.erase(nodeRange.first, nodeRange.second);
      }
      return assignmentCount;
  }
  
  } // anonymous
  
  int computeReversePathMatchScore(const LocalPath& path1, const LocalPath& path2, const FileSystemAccess& fsaccess)
  {
      if (path1.empty() || path2.empty())
      {
          return 0;
      }
  
      const auto path1End = path1.localpath.size() - 1;
      const auto path2End = path2.localpath.size() - 1;
  
      size_t index = 0;
      size_t separatorBias = 0;
      LocalPath accumulated;
      while (index <= path1End && index <= path2End)
      {
          const auto value1 = path1.localpath[path1End - index];
          const auto value2 = path2.localpath[path2End - index];
          if (value1 != value2)
          {
              break;
          }
          accumulated.localpath.push_back(value1);
  
          ++index;
  
          if (!accumulated.localpath.empty())
          {
              if (accumulated.localpath.back() == LocalPath::localPathSeparator)
              {
                  ++separatorBias;
                  accumulated.clear();
              }
          }
      }
  
      if (index > path1End && index > path2End) // we got to the beginning of both paths (full score)
      {
          return static_cast<int>(index - separatorBias);
      }
      else // the paths only partly match
      {
          return static_cast<int>(index - separatorBias - accumulated.localpath.size());
      }
  }
  
  bool assignFilesystemIds(Sync& sync, MegaApp& app, FileSystemAccess& fsaccess, handlelocalnode_map& fsidnodes,
                           LocalPath& localdebris)
  {
      auto& rootpath = sync.localroot->localname;
      LOG_info << "Assigning fs IDs at rootpath: " << rootpath.toPath(fsaccess);
  
      auto fa = fsaccess.newfileaccess(false);
      if (!fa->fopen(rootpath, true, false))
      {
          LOG_err << "Unable to open rootpath";
          return false;
      }
      if (fa->type != FOLDERNODE)
      {
          LOG_err << "rootpath not a folder";
          assert(false);
          return false;
      }
      if (fa->mIsSymLink)
      {
          LOG_err << "rootpath is a symlink";
          assert(false);
          return false;
      }
      fa.reset();
  
      bool success = true;
  
      FingerprintCache fingerprints;
  
      FingerprintLocalNodeMap localnodes;
      collectAllLocalNodes(fingerprints, localnodes, *sync.localroot, fsidnodes);
      LOG_info << "Number of localnodes: " << localnodes.size();
  
      if (localnodes.empty())
      {
          return success;
      }
  
      FingerprintFileMap files;
      collectAllFiles(success, fingerprints, files, sync, app, fsaccess, rootpath, localdebris);
      LOG_info << "Number of files: " << files.size();
  
      LOG_info << "Number of fingerprints: " << fingerprints.all().size();
      const auto assignmentCount = assignFilesystemIdsImpl(fingerprints, localnodes, files, fsidnodes, fsaccess);
      LOG_info << "Number of fsid assignments: " << assignmentCount;
  
      return success;
  }
  
  
  
  SyncConfig::SyncConfig(LocalPath localPath,
                         std::string name,
                         const handle remoteNode,
                         const std::string &remotePath,
                         const fsfp_t localFingerprint,
                         std::vector<std::string> regExps,
                         const bool enabled,
                         const SyncConfig::Type syncType,
                         const SyncError error,
                         const SyncWarning warning,
                         mega::handle hearBeatID)
      : mEnabled{enabled}
      , mLocalPath{std::move(localPath)}
      , mName{std::move(name)}
      , mRemoteNode{remoteNode}
      , mOrigninalPathOfRemoteRootNode{remotePath}
      , mLocalFingerprint{localFingerprint}
      , mRegExps{std::move(regExps)}
      , mSyncType{syncType}
      , mError{error}
      , mBackupId(hearBeatID)
      , mExternalDrivePath()
      , mWarning{warning}
  {}
  
  bool SyncConfig::operator==(const SyncConfig& rhs) const
  {
      return mEnabled == rhs.mEnabled
             && mExternalDrivePath == rhs.mExternalDrivePath
             && mLocalPath == rhs.mLocalPath
             && mName == rhs.mName
             && mRemoteNode == rhs.mRemoteNode
             && mOrigninalPathOfRemoteRootNode == rhs.mOrigninalPathOfRemoteRootNode
             && mLocalFingerprint == rhs.mLocalFingerprint
             && mRegExps == rhs.mRegExps
             && mSyncType == rhs.mSyncType
             && mError == rhs.mError
             && mBackupId == rhs.mBackupId
             && mWarning == rhs.mWarning;
  }
  
  bool SyncConfig::operator!=(const SyncConfig& rhs) const
  {
      return !(*this == rhs);
  }
  
  bool SyncConfig::getEnabled() const
  {
      return mEnabled;
  }
  
  void SyncConfig::setEnabled(bool enabled)
  {
      mEnabled = enabled;
  }
  
  const LocalPath& SyncConfig::getLocalPath() const
  {
      return mLocalPath;
  }
  
  handle SyncConfig::getRemoteNode() const
  {
      return mRemoteNode;
  }
  
  void SyncConfig::setRemoteNode(const handle &remoteNode)
  {
      mRemoteNode = remoteNode;
  }
  
  handle SyncConfig::getLocalFingerprint() const
  {
      return mLocalFingerprint;
  }
  
  void SyncConfig::setLocalFingerprint(fsfp_t fingerprint)
  {
      mLocalFingerprint = fingerprint;
  }
  
  const std::vector<std::string>& SyncConfig::getRegExps() const
  {
      return mRegExps;
  }
  
  void SyncConfig::setRegExps(std::vector<std::string>&& v)
  {
      mRegExps = std::move(v);
  }
  
  SyncConfig::Type SyncConfig::getType() const
  {
      return mSyncType;
  }
  
  
  SyncError SyncConfig::getError() const
  {
      return mError;
  }
  
  void SyncConfig::setError(SyncError value)
  {
      mError = value;
  }
  
  handle SyncConfig::getBackupId() const
  {
      return mBackupId;
  }
  
  void SyncConfig::setBackupId(const handle &backupId)
  {
      mBackupId = backupId;
  }
  
  bool SyncConfig::isExternal() const
  {
      return !mExternalDrivePath.empty();
  }
  
  bool SyncConfig::errorOrEnabledChanged()
  {
      bool changed = mError != mKnownError ||
                     mEnabled != mKnownEnabled;
  
      if (changed)
      {
          mKnownError = mError;
          mKnownEnabled = mEnabled;
      }
      return changed;
  }
  
  std::string SyncConfig::syncErrorToStr()
  {
      return syncErrorToStr(mError);
  }
  
  std::string SyncConfig::syncErrorToStr(SyncError errorCode)
  {
      switch(errorCode)
      {
      case NO_SYNC_ERROR:
          return "No error";
      case UNKNOWN_ERROR:
          return "Unknown error";
      case UNSUPPORTED_FILE_SYSTEM:
          return "File system not supported";
      case INVALID_REMOTE_TYPE:
          return "Remote node is not valid";
      case INVALID_LOCAL_TYPE:
          return "Local path is not valid";
      case INITIAL_SCAN_FAILED:
          return "Initial scan failed";
      case LOCAL_PATH_TEMPORARY_UNAVAILABLE:
          return "Local path temporarily unavailable";
      case LOCAL_PATH_UNAVAILABLE:
          return "Local path not available";
      case REMOTE_NODE_NOT_FOUND:
          return "Remote node not found";
      case STORAGE_OVERQUOTA:
          return "Reached storage quota limit";
      case BUSINESS_EXPIRED:
          return "Business account expired";
      case FOREIGN_TARGET_OVERSTORAGE:
          return "Foreign target storage quota reached";
      case REMOTE_PATH_HAS_CHANGED:
          return "Remote path has changed";
      case REMOTE_NODE_MOVED_TO_RUBBISH:
          return "Remote node moved to Rubbish Bin";
      case SHARE_NON_FULL_ACCESS:
          return "Share without full access";
      case LOCAL_FINGERPRINT_MISMATCH:
          return "Local fingerprint mismatch";
      case PUT_NODES_ERROR:
          return "Put nodes error";
      case ACTIVE_SYNC_BELOW_PATH:
          return "Active sync below path";
      case ACTIVE_SYNC_ABOVE_PATH:
          return "Active sync above path";
      case REMOTE_PATH_DELETED:
          return "Remote node has been deleted";
      case REMOTE_NODE_INSIDE_RUBBISH:
          return "Remote node is inside Rubbish Bin";
      case VBOXSHAREDFOLDER_UNSUPPORTED:
          return "Unsupported VBoxSharedFolderFS filesystem";
      case LOCAL_PATH_SYNC_COLLISION:
          return "Local path collides with an existing sync";
      case ACCOUNT_BLOCKED:
          return "Your account is blocked";
      case UNKNOWN_TEMPORARY_ERROR:
          return "Unknown temporary error";
      case TOO_MANY_ACTION_PACKETS:
          return "Too many changes in account, local state invalid";
      case LOGGED_OUT:
          return "Session closed";
      default:
          return "Undefined error";
      }
  }
  
  const char* SyncConfig::syncstatename(const syncstate_t state)
  {
      switch (state)
      {
      case SYNC_DISABLED:
          return "DISABLED";
      case SYNC_FAILED:
          return "FAILED";
      case SYNC_CANCELED:
          return "CANCELED";
      case SYNC_INITIALSCAN:
          return "INITIALSCAN";
      case SYNC_ACTIVE:
          return "ACTIVE";
      default:
          return "UNKNOWN";
      }
  }
  
  const char* SyncConfig::synctypename(const SyncConfig::Type type)
  {
      switch (type)
      {
      case SyncConfig::TYPE_BACKUP:
          return "BACKUP";
      case SyncConfig::TYPE_DOWN:
          return "DOWN";
      case SyncConfig::TYPE_UP:
          return "UP";
      case SyncConfig::TYPE_TWOWAY:
          return "TWOWAY";
      default:
          return "UNKNOWN";
      }
  }
  
  // new Syncs are automatically inserted into the session's syncs list
  // and a full read of the subtree is initiated
  Sync::Sync(UnifiedSync& us, const char* cdebris,
             LocalPath* clocaldebris, Node* remotenode, bool cinshare)
  : localroot(new LocalNode)
  , mUnifiedSync(us)
  {
      isnetwork = false;
      client = &mUnifiedSync.mClient;
      inshare = cinshare;
      tmpfa = NULL;
      initializing = true;
      updatedfilesize = ~0;
      updatedfilets = 0;
      updatedfileinitialts = 0;
  
      localbytes = 0;
      localnodes[FILENODE] = 0;
      localnodes[FOLDERNODE] = 0;
  
      state = SYNC_INITIALSCAN;
      statecachetable = NULL;
  
      fullscan = true;
      scanseqno = 0;
  
      mLocalPath = mUnifiedSync.mConfig.getLocalPath();
  
      if (cdebris)
      {
          debris = cdebris;
          localdebris = LocalPath::fromPath(debris, *client->fsaccess);
  
          dirnotify.reset(client->fsaccess->newdirnotify(mLocalPath, localdebris, client->waiter));
  
          localdebris.prependWithSeparator(mLocalPath);
      }
      else
      {
          localdebris = *clocaldebris;
  
          // FIXME: pass last segment of localdebris
          dirnotify.reset(client->fsaccess->newdirnotify(mLocalPath, localdebris, client->waiter));
      }
      dirnotify->sync = this;
  
      // set specified fsfp or get from fs if none
      const auto cfsfp = mUnifiedSync.mConfig.getLocalFingerprint();
      if (cfsfp)
      {
          fsfp = cfsfp;
      }
      else
      {
          fsfp = dirnotify->fsfingerprint();
      }
  
      fsstableids = dirnotify->fsstableids();
      LOG_info << "Filesystem IDs are stable: " << fsstableids;
  
      mFilesystemType = client->fsaccess->getlocalfstype(mLocalPath);
  
      localroot->init(this, FOLDERNODE, NULL, mLocalPath, nullptr);  // the root node must have the absolute path.  We don't store shortname, to avoid accidentally using relative paths.
      localroot->setnode(remotenode);
  
  #ifdef __APPLE__
      if (macOSmajorVersion() >= 19) //macOS catalina+
      {
          LOG_debug << "macOS 10.15+ filesystem detected. Checking fseventspath.";
          string supercrootpath = "/System/Volumes/Data" + mLocalPath.platformEncoded();
  
          int fd = open(supercrootpath.c_str(), O_RDONLY);
          if (fd == -1)
          {
              LOG_debug << "Unable to open path using fseventspath.";
              mFsEventsPath = mLocalPath.platformEncoded();
          }
          else
          {
              char buf[MAXPATHLEN];
              if (fcntl(fd, F_GETPATH, buf) < 0)
              {
                  LOG_debug << "Using standard paths to detect filesystem notifications.";
                  mFsEventsPath = mLocalPath.platformEncoded();
              }
              else
              {
                  LOG_debug << "Using fsevents paths to detect filesystem notifications.";
                  mFsEventsPath = supercrootpath;
              }
              close(fd);
          }
      }
  #endif
  
      if (client->dbaccess)
      {
          // open state cache table
          handle tableid[3];
          string dbname;
  
          auto fas = client->fsaccess->newfileaccess(false);
  
          if (fas->fopen(mLocalPath, true, false))
          {
              tableid[0] = fas->fsid;
              tableid[1] = remotenode->nodehandle;
              tableid[2] = client->me;
  
              dbname.resize(sizeof tableid * 4 / 3 + 3);
              dbname.resize(Base64::btoa((byte*)tableid, sizeof tableid, (char*)dbname.c_str()));
  
              statecachetable = client->dbaccess->open(client->rng, *client->fsaccess, dbname);
  
              readstatecache();
          }
      }
  }
  
  Sync::~Sync()
  {
      // must be set to prevent remote mass deletion while rootlocal destructor runs
      mDestructorRunning = true;
  
      // unlock tmp lock
      tmpfa.reset();
  
      // stop all active and pending downloads
      if (localroot->node)
      {
          TreeProcDelSyncGet tdsg;
          // Create a committer to ensure we update the transfer database in an efficient single commit,
          // if there are transactions in progress.
          DBTableTransactionCommitter committer(client->tctable);
          client->proctree(localroot->node, &tdsg);
      }
  
      // The database is closed; deleting localnodes will not remove them
      delete statecachetable;
  
      client->syncactivity = true;
  
      {
          // Create a committer and recursively delete all the associated LocalNodes, and their associated transfer and file objects.
          // If any have transactions in progress, the committer will ensure we update the transfer database in an efficient single commit.
          DBTableTransactionCommitter committer(client->tctable);
          localroot.reset();
      }
  }
  
  void Sync::addstatecachechildren(uint32_t parent_dbid, idlocalnode_map* tmap, LocalPath& localpath, LocalNode *p, int maxdepth)
  {
      auto range = tmap->equal_range(parent_dbid);
  
      for (auto it = range.first; it != range.second; it++)
      {
          ScopedLengthRestore restoreLen(localpath);
  
          localpath.appendWithSeparator(it->second->localname, true);
  
          LocalNode* l = it->second;
          Node* node = l->node.release_unchecked();
          handle fsid = l->fsid;
          m_off_t size = l->size;
  
          // clear localname to force newnode = true in setnameparent
          l->localname.clear();
  
          // if we already have the shortname from database, use that, otherwise (db is from old code) look it up
          std::unique_ptr<LocalPath> shortname;
          if (l->slocalname_in_db)
          {
              // null if there is no shortname, or the shortname matches the localname.
              shortname.reset(l->slocalname.release());
          }
          else
          {
              shortname = client->fsaccess->fsShortname(localpath);
          }
  
          l->init(this, l->type, p, localpath, std::move(shortname));
  
  #ifdef DEBUG
          auto fa = client->fsaccess->newfileaccess(false);
          if (fa->fopen(localpath))  // exists, is file
          {
              auto sn = client->fsaccess->fsShortname(localpath);
              assert(!l->localname.empty() &&
                  ((!l->slocalname && (!sn || l->localname == *sn)) ||
                  (l->slocalname && sn && !l->slocalname->empty() && *l->slocalname != l->localname && *l->slocalname == *sn)));
          }
  #endif
  
          l->parent_dbid = parent_dbid;
          l->size = size;
          l->setfsid(fsid, client->fsidnode);
          l->setnode(node);
  
          if (!l->slocalname_in_db)
          {
              statecacheadd(l);
              if (insertq.size() > 50000)
              {
                  cachenodes();  // periodically output updated nodes with shortname updates, so people who restart megasync still make progress towards a fast startup
              }
          }
  
          if (maxdepth)
          {
              addstatecachechildren(l->dbid, tmap, localpath, l, maxdepth - 1);
          }
      }
  }
  
  bool Sync::readstatecache()
  {
      if (statecachetable && state == SYNC_INITIALSCAN)
      {
          string cachedata;
          idlocalnode_map tmap;
          uint32_t cid;
          LocalNode* l;
  
          statecachetable->rewind();
  
          // bulk-load cached nodes into tmap
          while (statecachetable->next(&cid, &cachedata, &client->key))
          {
              if ((l = LocalNode::unserialize(this, &cachedata)))
              {
                  l->dbid = cid;
                  tmap.insert(pair<int32_t,LocalNode*>(l->parent_dbid,l));
              }
          }
  
          // recursively build LocalNode tree, set scanseqnos to sync's current scanseqno
          addstatecachechildren(0, &tmap, localroot->localname, localroot.get(), 100);
          cachenodes();
  
          // trigger a single-pass full scan to identify deleted nodes
          fullscan = true;
          scanseqno++;
  
          return true;
      }
  
      return false;
  }
  
  SyncConfig& Sync::getConfig()
  {
      return mUnifiedSync.mConfig;
  }
  
  // remove LocalNode from DB cache
  void Sync::statecachedel(LocalNode* l)
  {
      if (state == SYNC_CANCELED)
      {
          return;
      }
  
      insertq.erase(l);
  
      if (l->dbid)
      {
          deleteq.insert(l->dbid);
      }
  }
  
  // insert LocalNode into DB cache
  void Sync::statecacheadd(LocalNode* l)
  {
      if (state == SYNC_CANCELED)
      {
          return;
      }
  
      if (l->dbid)
      {
          deleteq.erase(l->dbid);
      }
  
      insertq.insert(l);
  }
  
  void Sync::cachenodes()
  {
      if (statecachetable && (state == SYNC_ACTIVE || (state == SYNC_INITIALSCAN && insertq.size() > 100)) && (deleteq.size() || insertq.size()))
      {
          LOG_debug << "Saving LocalNode database with " << insertq.size() << " additions and " << deleteq.size() << " deletions";
          statecachetable->begin();
  
          // deletions
          for (set<uint32_t>::iterator it = deleteq.begin(); it != deleteq.end(); it++)
          {
              statecachetable->del(*it);
          }
  
          deleteq.clear();
  
          // additions - we iterate until completion or until we get stuck
          bool added;
  
          do {
              added = false;
  
              for (set<LocalNode*>::iterator it = insertq.begin(); it != insertq.end(); )
              {
                  if ((*it)->parent->dbid || (*it)->parent == localroot.get())
                  {
                      statecachetable->put(MegaClient::CACHEDLOCALNODE, *it, &client->key);
                      insertq.erase(it++);
                      added = true;
                  }
                  else it++;
              }
          } while (added);
  
          statecachetable->commit();
  
          if (insertq.size())
          {
              LOG_err << "LocalNode caching did not complete";
          }
      }
  }
  
  void Sync::changestate(syncstate_t newstate, SyncError newSyncError, bool newEnableFlag, bool notifyApp)
  {
      getConfig().setError(newSyncError);
      getConfig().setEnabled(newEnableFlag);
  
      if (newstate != state)
      {
          auto oldstate = state;
          state = newstate;
          fullscan = false;
  
          if (notifyApp)
          {
              bool wasActive = oldstate == SYNC_ACTIVE;
              bool nowActive = newstate == SYNC_ACTIVE;
              if (wasActive != nowActive)
              {
                  mUnifiedSync.mClient.app->syncupdate_active(getConfig().getBackupId(), nowActive);
              }
          }
      }
  
      if (newstate != SYNC_CANCELED)
      {
          mUnifiedSync.changedConfigState(notifyApp);
      }
  }
  
  // walk path and return corresponding LocalNode and its parent
  // path must be relative to l or start with the root prefix if l == NULL
  // path must be a full sync path, i.e. start with localroot->localname
  // NULL: no match, optionally returns residual path
  LocalNode* Sync::localnodebypath(LocalNode* l, const LocalPath& localpath, LocalNode** parent, LocalPath* outpath)
  {
      assert(!outpath || outpath->empty());
  
      size_t subpathIndex = 0;
  
      if (!l)
      {
          // verify matching localroot prefix - this should always succeed for
          // internal use
          if (!localroot->localname.isContainingPathOf(localpath, &subpathIndex))
          {
              if (parent)
              {
                  *parent = NULL;
              }
  
              return NULL;
          }
  
          l = localroot.get();
      }
  
  
      LocalPath component;
  
      while (localpath.nextPathComponent(subpathIndex, component))
      {
          if (parent)
          {
              *parent = l;
          }
  
          localnode_map::iterator it;
          if ((it = l->children.find(&component)) == l->children.end()
              && (it = l->schildren.find(&component)) == l->schildren.end())
          {
              // no full match: store residual path, return NULL with the
              // matching component LocalNode in parent
              if (outpath)
              {
                  *outpath = std::move(component);
                  auto remainder = localpath.subpathFrom(subpathIndex);
                  if (!remainder.empty())
                  {
                      outpath->appendWithSeparator(remainder, false);
                  }
              }
  
              return NULL;
          }
  
          l = it->second;
      }
  
      // full match: no residual path, return corresponding LocalNode
      if (outpath)
      {
          outpath->clear();
      }
      return l;
  }
  
  bool Sync::assignfsids()
  {
      return assignFilesystemIds(*this, *client->app, *client->fsaccess, client->fsidnode,
                                 localdebris);
  }
  
  // scan localpath, add or update child nodes, call recursively for folder nodes
  // localpath must be prefixed with Sync
  bool Sync::scan(LocalPath* localpath, FileAccess* fa)
  {
      if (fa)
      {
          assert(fa->type == FOLDERNODE);
      }
      if (!localdebris.isContainingPathOf(*localpath))
      {
          DirAccess* da;
          LocalPath localname;
          string name;
          bool success;
  
          if (SimpleLogger::logCurrentLevel >= logDebug)
          {
              LOG_debug << "Scanning folder: " << localpath->toPath(*client->fsaccess);
          }
  
          da = client->fsaccess->newdiraccess();
  
          // scan the dir, mark all items with a unique identifier
          if ((success = da->dopen(localpath, fa, false)))
          {
              while (da->dnext(*localpath, localname, client->followsymlinks))
              {
                  name = localname.toName(*client->fsaccess, mFilesystemType);
  
                  ScopedLengthRestore restoreLen(*localpath);
                  localpath->appendWithSeparator(localname, false);
  
                  // check if this record is to be ignored
                  if (client->app->sync_syncable(this, name.c_str(), *localpath))
                  {
                      // skip the sync's debris folder
                      if (!localdebris.isContainingPathOf(*localpath))
                      {
                          LocalNode *l = NULL;
                          if (initializing)
                          {
                              // preload all cached LocalNodes
                              l = checkpath(NULL, localpath, nullptr, nullptr, false, da);
                          }
  
                          if (!l || l == (LocalNode*)~0)
                          {
                              // new record: place in notification queue
                              dirnotify->notify(DirNotify::DIREVENTS, NULL, LocalPath(*localpath));
                          }
                      }
                  }
                  else
                  {
                      LOG_debug << "Excluded: " << name;
                  }
              }
          }
  
          delete da;
  
          return success;
      }
      else return false;
  }
  
  // check local path - if !localname, localpath is relative to l, with l == NULL
  // being the root of the sync
  // if localname is set, localpath is absolute and localname its last component
  // path references a new FOLDERNODE: returns created node
  // path references a existing FILENODE: returns node
  // otherwise, returns NULL
  LocalNode* Sync::checkpath(LocalNode* l, LocalPath* input_localpath, string* const localname, dstime *backoffds, bool wejustcreatedthisfolder, DirAccess* iteratingDir)
  {
      LocalNode* ll = l;
      bool newnode = false, changed = false;
      bool isroot;
  
      LocalNode* parent;
      string path;           // UTF-8 representation of tmppath
      LocalPath tmppath;     // full path represented by l + localpath
      LocalPath newname;     // portion of tmppath not covered by the existing
                             // LocalNode structure (always the last path component
                             // that does not have a corresponding LocalNode yet)
  
      if (localname)
      {
          // shortcut case (from within syncdown())
          isroot = false;
          parent = l;
          l = NULL;
  
          path = input_localpath->toPath(*client->fsaccess);
          assert(path.size());
      }
      else
      {
          // construct full filesystem path in tmppath
          if (l)
          {
              tmppath = l->getLocalPath();
          }
  
          if (!input_localpath->empty())
          {
              tmppath.appendWithSeparator(*input_localpath, false);
          }
  
          // look up deepest existing LocalNode by path, store remainder (if any)
          // in newname
          LocalNode *tmp = localnodebypath(l, *input_localpath, &parent, &newname);
          size_t index = 0;
  
          if (newname.findNextSeparator(index))
          {
              LOG_warn << "Parent not detected yet. Unknown remainder: " << newname.toPath(*client->fsaccess);
              if (parent)
              {
                  LocalPath notifyPath = parent->getLocalPath();
                  notifyPath.appendWithSeparator(newname.subpathTo(index), true);
                  dirnotify->notify(DirNotify::DIREVENTS, l, std::move(notifyPath), true);
              }
              return NULL;
          }
  
          l = tmp;
  
          path = tmppath.toPath(*client->fsaccess);
  
          // path invalid?
          if ( ( !l && newname.empty() ) || !path.size())
          {
              LOG_warn << "Invalid path: " << path;
              return NULL;
          }
  
          string name = !newname.empty() ? newname.toName(*client->fsaccess, mFilesystemType) : l->name;
  
          if (!client->app->sync_syncable(this, name.c_str(), tmppath))
          {
              LOG_debug << "Excluded: " << path;
              return NULL;
          }
  
          isroot = l == localroot.get() && newname.empty();
      }
  
      LOG_verbose << "Scanning: " << path << " in=" << initializing << " full=" << fullscan << " l=" << l;
      LocalPath* localpathNew = localname ? input_localpath : &tmppath;
  
      if (parent)
      {
          if (state != SYNC_INITIALSCAN && !parent->node)
          {
              LOG_warn << "Parent doesn't exist yet: " << path;
              return (LocalNode*)~0;
          }
      }
  
      // attempt to open/type this file
      auto fa = client->fsaccess->newfileaccess(false);
  
      if (initializing || fullscan)
      {
          // find corresponding LocalNode by file-/foldername
          size_t lastpart = localpathNew->getLeafnameByteIndex(*client->fsaccess);
  
          LocalPath fname(localpathNew->subpathFrom(lastpart));
  
          LocalNode* cl = (parent ? parent : localroot.get())->childbyname(&fname);
          if (initializing && cl)
          {
              // the file seems to be still in the folder
              // mark as present to prevent deletions if the file is not accesible
              // in that case, the file would be checked again after the initialization
              cl->deleted = false;
              cl->setnotseen(0);
              l->scanseqno = scanseqno;
          }
  
          // match cached LocalNode state during initial/rescan to prevent costly re-fingerprinting
          // (just compare the fsids, sizes and mtimes to detect changes)
          if (fa->fopen(*localpathNew, false, false, iteratingDir))
          {
              if (cl && fa->fsidvalid && fa->fsid == cl->fsid)
              {
                  // node found and same file
                  l = cl;
                  l->deleted = false;
                  l->setnotseen(0);
  
                  // if it's a file, size and mtime must match to qualify
                  if (l->type != FILENODE || (l->size == fa->size && l->mtime == fa->mtime))
                  {
                      LOG_verbose << "Cached localnode is still valid. Type: " << l->type << "  Size: " << l->size << "  Mtime: " << l->mtime << " fsid " << (fa->fsidvalid ? toHandle(fa->fsid) : "NO");
                      l->scanseqno = scanseqno;
  
                      if (l->type == FOLDERNODE)
                      {
                          scan(localpathNew, fa.get());
                      }
                      else
                      {
                          localbytes += l->size;
                      }
  
                      return l;
                  }
              }
          }
          else
          {
              LOG_warn << "Error opening file during the initialization: " << path;
          }
  
          if (initializing)
          {
              if (cl)
              {
                  LOG_verbose << "Outdated localnode. Type: " << cl->type << "  Size: " << cl->size << "  Mtime: " << cl->mtime
                              << "    FaType: " << fa->type << "  FaSize: " << fa->size << "  FaMtime: " << fa->mtime;
              }
              else
              {
                  LOG_verbose << "New file. FaType: " << fa->type << "  FaSize: " << fa->size << "  FaMtime: " << fa->mtime;
              }
              return NULL;
          }
  
          fa = client->fsaccess->newfileaccess(false);
      }
  
      if (fa->fopen(*localpathNew, true, false))
      {
          if (!isroot)
          {
              if (l)
              {
                  if (l->type == fa->type)
                  {
                      // mark as present
                      l->setnotseen(0);
  
                      if (fa->type == FILENODE)
                      {
                          // has the file been overwritten or changed since the last scan?
                          // or did the size or mtime change?
                          if (fa->fsidvalid)
                          {
                              // if fsid has changed, the file was overwritten
                              // (FIXME: handle type changes)
                              if (l->fsid != fa->fsid)
                              {
                                  handlelocalnode_map::iterator it;
  #ifdef _WIN32
                                  const char *colon;
  #endif
                                  fsfp_t fp1, fp2;
  
                                  // was the file overwritten by moving an existing file over it?
                                  if ((it = client->fsidnode.find(fa->fsid)) != client->fsidnode.end()
                                          && (l->sync == it->second->sync
                                              || ((fp1 = l->sync->dirnotify->fsfingerprint())
                                                  && (fp2 = it->second->sync->dirnotify->fsfingerprint())
                                                  && (fp1 == fp2)
                                              #ifdef _WIN32
                                                  // only consider fsid matches between different syncs for local drives with the
                                                  // same drive letter, to prevent problems with cloned Volume IDs
                                                  && (colon = strstr(parent->sync->localroot->name.c_str(), ":"))
                                                  && !memcmp(parent->sync->localroot->name.c_str(),
                                                         it->second->sync->localroot->name.c_str(),
                                                         colon - parent->sync->localroot->name.c_str())
                                              #endif
                                                  )
                                              )
                                      )
                                  {
                                      // catch the not so unlikely case of a false fsid match due to
                                      // e.g. a file deletion/creation cycle that reuses the same inode
                                      if (it->second->mtime != fa->mtime || it->second->size != fa->size)
                                      {
                                          l->mtime = -1;  // trigger change detection
                                          delete it->second;   // delete old LocalNode
                                      }
                                      else
                                      {
                                          LOG_debug << "File move/overwrite detected";
  
                                          // delete existing LocalNode...
                                          delete l;
  
                                          // ...move remote node out of the way...
                                          client->execsyncdeletions();
  
                                          // ...and atomically replace with moved one
                                          client->app->syncupdate_local_move(this, it->second, path.c_str());
  
                                          // (in case of a move, this synchronously updates l->parent and l->node->parent)
                                          it->second->setnameparent(parent, localpathNew, client->fsaccess->fsShortname(*localpathNew));
  
                                          // mark as seen / undo possible deletion
                                          it->second->setnotseen(0);
  
                                          statecacheadd(it->second);
  
                                          return it->second;
                                      }
                                  }
                                  else
                                  {
                                      l->mtime = -1;  // trigger change detection
                                  }
                              }
                          }
  
                          // no fsid change detected or overwrite with unknown file:
                          if (fa->mtime != l->mtime || fa->size != l->size)
                          {
                              if (fa->fsidvalid && l->fsid != fa->fsid)
                              {
                                  l->setfsid(fa->fsid, client->fsidnode);
                              }
  
                              m_off_t dsize = l->size > 0 ? l->size : 0;
  
                              if (l->genfingerprint(fa.get()) && l->size >= 0)
                              {
                                  localbytes -= dsize - l->size;
                              }
  
                              client->app->syncupdate_local_file_change(this, l, path.c_str());
  
                              DBTableTransactionCommitter committer(client->tctable);
                              client->stopxfer(l, &committer); // TODO:  can we use one committer for all the files in the folder?  Or for the whole recursion?
                              l->bumpnagleds();
                              l->deleted = false;
  
                              client->syncactivity = true;
  
                              statecacheadd(l);
  
                              fa.reset();
  
                              if (isnetwork && l->type == FILENODE)
                              {
                                  LOG_debug << "Queueing extra fs notification for modified file";
                                  dirnotify->notify(DirNotify::EXTRA, NULL, LocalPath(*localpathNew));
                              }
                              return l;
                          }
                      }
                      else
                      {
                          // (we tolerate overwritten folders, because we do a
                          // content scan anyway)
                          if (fa->fsidvalid && fa->fsid != l->fsid)
                          {
                              l->setfsid(fa->fsid, client->fsidnode);
                              newnode = true;
                          }
                      }
                  }
                  else
                  {
                      LOG_debug << "node type changed: recreate";
                      delete l;
                      l = NULL;
                  }
              }
  
              // new node
              if (!l)
              {
                  // rename or move of existing node?
                  handlelocalnode_map::iterator it;
  #ifdef _WIN32
                  const char *colon;
  #endif
                  fsfp_t fp1, fp2;
                  if (fa->fsidvalid && (it = client->fsidnode.find(fa->fsid)) != client->fsidnode.end()
                      // additional checks to prevent wrong fsid matches
                      && it->second->type == fa->type
                      && (!parent
                          || (it->second->sync == parent->sync)
                          || ((fp1 = it->second->sync->dirnotify->fsfingerprint())
                              && (fp2 = parent->sync->dirnotify->fsfingerprint())
                              && (fp1 == fp2)
                          #ifdef _WIN32
                              // allow moves between different syncs only for local drives with the
                              // same drive letter, to prevent problems with cloned Volume IDs
                              && (colon = strstr(parent->sync->localroot->name.c_str(), ":"))
                              && !memcmp(parent->sync->localroot->name.c_str(),
                                     it->second->sync->localroot->name.c_str(),
                                     colon - parent->sync->localroot->name.c_str())
                          #endif
                              )
                         )
                      && ((it->second->type != FILENODE && !wejustcreatedthisfolder)
                          || (it->second->mtime == fa->mtime && it->second->size == fa->size)))
                  {
                      LOG_debug << client->clientname << "Move detected by fsid " << toHandle(fa->fsid) << " in checkpath. Type: " << it->second->type << " new path: " << path << " old localnode: " << it->second->localnodedisplaypath(*client->fsaccess);
  
                      if (fa->type == FILENODE && backoffds)
                      {
                          // logic to detect files being updated in the local computer moving the original file
                          // to another location as a temporary backup
  
                          m_time_t currentsecs = m_time();
                          if (!updatedfileinitialts)
                          {
                              updatedfileinitialts = currentsecs;
                          }
  
                          if (currentsecs >= updatedfileinitialts)
                          {
                              if (currentsecs - updatedfileinitialts <= FILE_UPDATE_MAX_DELAY_SECS)
                              {
                                  bool waitforupdate = false;
                                  auto local = it->second->getLocalPath();
                                  auto prevfa = client->fsaccess->newfileaccess(false);
  
                                  bool exists = prevfa->fopen(local);
                                  if (exists)
                                  {
                                      LOG_debug << "File detected in the origin of a move";
  
                                      if (currentsecs >= updatedfilets)
                                      {
                                          if ((currentsecs - updatedfilets) < (FILE_UPDATE_DELAY_DS / 10))
                                          {
                                              LOG_verbose << "currentsecs = " << currentsecs << "  lastcheck = " << updatedfilets
                                                        << "  currentsize = " << prevfa->size << "  lastsize = " << updatedfilesize;
                                              LOG_debug << "The file was checked too recently. Waiting...";
                                              waitforupdate = true;
                                          }
                                          else if (updatedfilesize != prevfa->size)
                                          {
                                              LOG_verbose << "currentsecs = " << currentsecs << "  lastcheck = " << updatedfilets
                                                        << "  currentsize = " << prevfa->size << "  lastsize = " << updatedfilesize;
                                              LOG_debug << "The file size has changed since the last check. Waiting...";
                                              updatedfilesize = prevfa->size;
                                              updatedfilets = currentsecs;
                                              waitforupdate = true;
                                          }
                                          else
                                          {
                                              LOG_debug << "The file size seems stable";
                                          }
                                      }
                                      else
                                      {
                                          LOG_warn << "File checked in the future";
                                      }
  
                                      if (!waitforupdate)
                                      {
                                          if (currentsecs >= prevfa->mtime)
                                          {
                                              if (currentsecs - prevfa->mtime < (FILE_UPDATE_DELAY_DS / 10))
                                              {
                                                  LOG_verbose << "currentsecs = " << currentsecs << "  mtime = " << prevfa->mtime;
                                                  LOG_debug << "File modified too recently. Waiting...";
                                                  waitforupdate = true;
                                              }
                                              else
                                              {
                                                  LOG_debug << "The modification time seems stable.";
                                              }
                                          }
                                          else
                                          {
                                              LOG_warn << "File modified in the future";
                                          }
                                      }
                                  }
                                  else
                                  {
                                      if (prevfa->retry)
                                      {
                                          LOG_debug << "The file in the origin is temporarily blocked. Waiting...";
                                          waitforupdate = true;
                                      }
                                      else
                                      {
                                          LOG_debug << "There isn't anything in the origin path";
                                      }
                                  }
  
                                  if (waitforupdate)
                                  {
                                      LOG_debug << "Possible file update detected.";
                                      *backoffds = FILE_UPDATE_DELAY_DS;
                                      return NULL;
                                  }
                              }
                              else
                              {
                                  int creqtag = client->reqtag;
                                  client->reqtag = 0;
                                  client->sendevent(99438, "Timeout waiting for file update");
                                  client->reqtag = creqtag;
                              }
                          }
                          else
                          {
                              LOG_warn << "File check started in the future";
                          }
                      }
  
                      client->app->syncupdate_local_move(this, it->second, path.c_str());
  
                      // (in case of a move, this synchronously updates l->parent
                      // and l->node->parent)
                      it->second->setnameparent(parent, localpathNew, client->fsaccess->fsShortname(*localpathNew));
  
                      // make sure that active PUTs receive their updated filenames
                      client->updateputs();
  
                      statecacheadd(it->second);
  
                      // unmark possible deletion
                      it->second->setnotseen(0);
  
                      // immediately scan folder to detect deviations from cached state
                      if (fullscan && fa->type == FOLDERNODE)
                      {
                          scan(localpathNew, fa.get());
                      }
                  }
                  else if (fa->mIsSymLink)
                  {
                      LOG_debug << "checked path is a symlink.  Parent: " << (parent ? parent->name : "NO");
                      //doing nothing for the moment
                  }
                  else
                  {
                      // this is a new node: add
                      LOG_debug << "New localnode.  Parent: " << (parent ? parent->name : "NO") << " fsid " << (fa->fsidvalid ? toHandle(fa->fsid) : "NO");
                      l = new LocalNode;
                      l->init(this, fa->type, parent, *localpathNew, client->fsaccess->fsShortname(*localpathNew));
  
                      if (fa->fsidvalid)
                      {
                          l->setfsid(fa->fsid, client->fsidnode);
                      }
  
                      newnode = true;
                  }
              }
          }
  
          if (l)
          {
              // detect file changes or recurse into new subfolders
              if (l->type == FOLDERNODE)
              {
                  if (newnode)
                  {
                      scan(localpathNew, fa.get());
                      client->app->syncupdate_local_folder_addition(this, l, path.c_str());
  
                      if (!isroot)
                      {
                          statecacheadd(l);
                      }
                  }
                  else
                  {
                      l = NULL;
                  }
              }
              else
              {
                  if (isroot)
                  {
                      // root node cannot be a file
                      LOG_err << "The local root node is a file";
                      changestate(SYNC_FAILED, INVALID_LOCAL_TYPE, false, true);
                  }
                  else
                  {
                      if (fa->fsidvalid && l->fsid != fa->fsid)
                      {
                          l->setfsid(fa->fsid, client->fsidnode);
                      }
  
                      if (l->size > 0)
                      {
                          localbytes -= l->size;
                      }
  
                      if (l->genfingerprint(fa.get()))
                      {
                          changed = true;
                          l->bumpnagleds();
                          l->deleted = false;
                      }
  
                      if (l->size > 0)
                      {
                          localbytes += l->size;
                      }
  
                      if (newnode)
                      {
                          client->app->syncupdate_local_file_addition(this, l, path.c_str());
                      }
                      else if (changed)
                      {
                          client->app->syncupdate_local_file_change(this, l, path.c_str());
                          DBTableTransactionCommitter committer(client->tctable); // TODO:  can we use one committer for all the files in the folder?  Or for the whole recursion?
                          client->stopxfer(l, &committer);
                      }
  
                      if (newnode || changed)
                      {
                          statecacheadd(l);
                      }
                  }
              }
          }
  
          if (changed || newnode)
          {
              if (isnetwork && l->type == FILENODE)
              {
                  LOG_debug << "Queueing extra fs notification for new file";
                  dirnotify->notify(DirNotify::EXTRA, NULL, LocalPath(*localpathNew));
              }
  
              client->syncactivity = true;
          }
      }
      else
      {
          LOG_warn << "Error opening file";
          if (fa->retry)
          {
              // fopen() signals that the failure is potentially transient - do
              // nothing and request a recheck
              LOG_warn << "File blocked. Adding notification to the retry queue: " << path;
              dirnotify->notify(DirNotify::RETRY, ll, LocalPath(*localpathNew));
              client->syncfslockretry = true;
              client->syncfslockretrybt.backoff(SCANNING_DELAY_DS);
              client->blockedfile = *localpathNew;
          }
          else if (l)
          {
              // immediately stop outgoing transfer, if any
              if (l->transfer)
              {
                  DBTableTransactionCommitter committer(client->tctable); // TODO:  can we use one committer for all the files in the folder?  Or for the whole recursion?
                  client->stopxfer(l, &committer);
              }
  
              client->syncactivity = true;
  
              // in fullscan mode, missing files are handled in bulk in deletemissing()
              // rather than through setnotseen()
              if (!fullscan)
              {
                  l->setnotseen(1);
              }
          }
  
          l = NULL;
      }
  
      return l;
  }
  
  bool Sync::checkValidNotification(int q, Notification& notification)
  {
      // This code moved from filtering before going on notifyq, to filtering after when it's thread-safe to do so
  
      if (q == DirNotify::DIREVENTS || q == DirNotify::EXTRA)
      {
          Notification next;
          while (dirnotify->notifyq[q].peekFront(next)
              && next.localnode == notification.localnode && next.path == notification.path)
          {
              dirnotify->notifyq[q].popFront(next);  // this is the only thread removing from the queue so it will be the same item
              if (!notification.timestamp || !next.timestamp)
              {
                  notification.timestamp = 0;  // immediate
              }
              else
              {
                  notification.timestamp = std::max(notification.timestamp, next.timestamp);
              }
              LOG_debug << "Next notification repeats, skipping duplicate";
          }
      }
  
      if (notification.timestamp && !initializing && q == DirNotify::DIREVENTS)
      {
          LocalPath tmppath;
          if (notification.localnode)
          {
              tmppath = notification.localnode->getLocalPath();
          }
  
          if (!notification.path.empty())
          {
              tmppath.appendWithSeparator(notification.path, false);
          }
  
          attr_map::iterator ait;
          auto fa = client->fsaccess->newfileaccess(false);
          bool success = fa->fopen(tmppath, false, false);
          LocalNode *ll = localnodebypath(notification.localnode, notification.path);
          if ((!ll && !success && !fa->retry) // deleted file
              || (ll && success && ll->node && ll->node->localnode == ll
                  && (ll->type != FILENODE || (*(FileFingerprint *)ll) == (*(FileFingerprint *)ll->node))
                  && (ait = ll->node->attrs.map.find('n')) != ll->node->attrs.map.end()
                  && ait->second == ll->name
                  && fa->fsidvalid && fa->fsid == ll->fsid && fa->type == ll->type
                  && (ll->type != FILENODE || (ll->mtime == fa->mtime && ll->size == fa->size))))
          {
              LOG_debug << "Self filesystem notification skipped";
              return false;
          }
      }
      return true;
  }
  
  // add or refresh local filesystem item from scan stack, add items to scan stack
  // returns 0 if a parent node is missing, ~0 if control should be yielded, or the time
  // until a retry should be made (500 ms minimum latency).
  dstime Sync::procscanq(int q)
  {
      dstime dsmin = Waiter::ds - SCANNING_DELAY_DS;
      LocalNode* l;
  
      Notification notification;
      while (dirnotify->notifyq[q].popFront(notification))
      {
          if (!checkValidNotification(q, notification))
          {
              continue;
          }
  
          LOG_verbose << "Scanning... Remaining files: " << dirnotify->notifyq[q].size();
  
          if (notification.timestamp > dsmin)
          {
              LOG_verbose << "Scanning postponed. Modification too recent";
              dirnotify->notifyq[q].unpopFront(notification);
              return notification.timestamp - dsmin;
          }
  
          if ((l = notification.localnode) != (LocalNode*)~0)
          {
              dstime backoffds = 0;
              LOG_verbose << "Checkpath: " << notification.path.toPath(*client->fsaccess);
  
              l = checkpath(l, &notification.path, NULL, &backoffds, false, nullptr);
              if (backoffds)
              {
                  LOG_verbose << "Scanning deferred during " << backoffds << " ds";
                  notification.timestamp = Waiter::ds + backoffds - SCANNING_DELAY_DS;
                  dirnotify->notifyq[q].unpopFront(notification);
                  return backoffds;
              }
              updatedfilesize = ~0;
              updatedfilets = 0;
              updatedfileinitialts = 0;
  
              // defer processing because of a missing parent node?
              if (l == (LocalNode*)~0)
              {
                  LOG_verbose << "Scanning deferred";
                  dirnotify->notifyq[q].unpopFront(notification);
                  return 0;
              }
          }
          else
          {
              string utf8path = notification.path.toPath(*client->fsaccess);
              LOG_debug << "Notification skipped: " << utf8path;
          }
  
          // we return control to the application in case a filenode was added
          // (in order to avoid lengthy blocking episodes due to multiple
          // consecutive fingerprint calculations)
          // or if new nodes are being added due to a copy/delete operation
          if ((l && l != (LocalNode*)~0 && l->type == FILENODE) || client->syncadding)
          {
              break;
          }
      }
  
      if (dirnotify->notifyq[q].empty())
      {
          if (q == DirNotify::DIREVENTS)
          {
              client->syncactivity = true;
          }
      }
      else if (dirnotify->notifyq[!q].empty())
      {
          cachenodes();
      }
  
      return dstime(~0);
  }
  
  // delete all child LocalNodes that have been missing for two consecutive scans (*l must still exist)
  void Sync::deletemissing(LocalNode* l)
  {
      LocalPath path;
      std::unique_ptr<FileAccess> fa;
      for (localnode_map::iterator it = l->children.begin(); it != l->children.end(); )
      {
          if (scanseqno-it->second->scanseqno > 1)
          {
              if (!fa)
              {
                  fa = client->fsaccess->newfileaccess();
              }
              client->unlinkifexists(it->second, fa.get(), path);
              delete it++->second;
          }
          else
          {
              deletemissing(it->second);
              it++;
          }
      }
  }
  
  bool Sync::updateSyncRemoteLocation(Node* n, bool forceCallback)
  {
      return mUnifiedSync.updateSyncRemoteLocation(n, forceCallback);
  }
  
  bool Sync::movetolocaldebris(LocalPath& localpath)
  {
      char buf[32];
      struct tm tms;
      string day, localday;
      bool havedir = false;
      struct tm* ptm = m_localtime(m_time(), &tms);
  
      for (int i = -3; i < 100; i++)
      {
          ScopedLengthRestore restoreLen(localdebris);
  
          if (i == -2 || i > 95)
          {
              LOG_verbose << "Creating local debris folder";
              client->fsaccess->mkdirlocal(localdebris, true);
          }
  
          sprintf(buf, "%04d-%02d-%02d", ptm->tm_year + 1900, ptm->tm_mon + 1, ptm->tm_mday);
  
          if (i >= 0)
          {
              sprintf(strchr(buf, 0), " %02d.%02d.%02d.%02d", ptm->tm_hour,  ptm->tm_min, ptm->tm_sec, i);
          }
  
          day = buf;
          localdebris.appendWithSeparator(LocalPath::fromPath(day, *client->fsaccess), true);
  
          if (i > -3)
          {
              LOG_verbose << "Creating daily local debris folder";
              havedir = client->fsaccess->mkdirlocal(localdebris, false) || client->fsaccess->target_exists;
          }
  
          localdebris.appendWithSeparator(localpath.subpathFrom(localpath.getLeafnameByteIndex(*client->fsaccess)), true);
  
          client->fsaccess->skip_errorreport = i == -3;  // we expect a problem on the first one when the debris folders or debris day folders don't exist yet
          if (client->fsaccess->renamelocal(localpath, localdebris, false))
          {
              client->fsaccess->skip_errorreport = false;
              return true;
          }
          client->fsaccess->skip_errorreport = false;
  
          if (client->fsaccess->transient_error)
          {
              return false;
          }
  
          if (havedir && !client->fsaccess->target_exists)
          {
              return false;
          }
      }
  
      return false;
  }
  
  m_off_t Sync::getInflightProgress()
  {
      m_off_t progressSum = 0;
  
      for (auto tslot : client->tslots)
      {
          for (auto file : tslot->transfer->files)
          {
              if (auto ln = dynamic_cast<LocalNode*>(file))
              {
                  if (ln->sync == this)
                  {
                      progressSum += tslot->progressreported;
                  }
              }
              else if (auto sfg = dynamic_cast<SyncFileGet*>(file))
              {
                  if (sfg->sync == this)
                  {
                      progressSum += tslot->progressreported;
                  }
              }
          }
      }
  
      return progressSum;
  }
  
  
  UnifiedSync::UnifiedSync(MegaClient& mc, const SyncConfig& c)
      : mClient(mc), mConfig(c)
  {
      mNextHeartbeat.reset(new HeartBeatSyncInfo());
  }
  
  
  error UnifiedSync::enableSync(bool resetFingerprint, bool notifyApp)
  {
      assert(!mSync);
      mConfig.mError = NO_SYNC_ERROR;
  
      if (resetFingerprint)
      {
          mConfig.setLocalFingerprint(0); //This will cause the local filesystem fingerprint to be recalculated
      }
  
      LocalPath rootpath;
      std::unique_ptr<FileAccess> openedLocalFolder;
      Node* remotenode;
      bool inshare, isnetwork;
      error e = mClient.checkSyncConfig(mConfig, rootpath, openedLocalFolder, remotenode, inshare, isnetwork);
  
      if (e)
      {
          // error and enable flag were already changed
          changedConfigState(notifyApp);
          return e;
      }
  
      e = startSync(&mClient, DEBRISFOLDER, nullptr, remotenode, inshare, isnetwork, false, rootpath, openedLocalFolder);
      mClient.syncactivity = true;
      changedConfigState(notifyApp);
  
      mClient.syncs.mHeartBeatMonitor->updateOrRegisterSync(*this);
  
      return e;
  }
  
  bool UnifiedSync::updateSyncRemoteLocation(Node* n, bool forceCallback)
  {
      bool changed = false;
      if (n)
      {
          auto newpath = n->displaypath();
          if (newpath != mConfig.mOrigninalPathOfRemoteRootNode)
          {
              mConfig.mOrigninalPathOfRemoteRootNode = newpath;
              changed = true;
          }
  
          if (mConfig.getRemoteNode() != n->nodehandle)
          {
              mConfig.setRemoteNode(n->nodehandle);
              changed = true;
          }
      }
      else //unset remote node: failed!
      {
          if (mConfig.getRemoteNode() != UNDEF)
          {
              mConfig.setRemoteNode(UNDEF);
              changed = true;
          }
      }
  
      if (changed || forceCallback)
      {
          mClient.app->syncupdate_remote_root_changed(mConfig);
      }
  
      //persist
      if (auto syncdb = mClient.syncs.syncConfigDB())
      {
          syncdb->add(mConfig);
      }
  
      return changed;
  }
  
  
  
  error UnifiedSync::startSync(MegaClient* client, const char* debris, LocalPath* localdebris, Node* remotenode, bool inshare,
                               bool isNetwork, bool delayInitialScan, LocalPath& rootpath, std::unique_ptr<FileAccess>& openedLocalFolder)
  {
      //check we are not in any blocking situation
      using CType = CacheableStatus::Type;
      bool overStorage = client->mCachedStatus.lookup(CType::STATUS_STORAGE, STORAGE_UNKNOWN) >= STORAGE_RED;
      bool businessExpired = client->mCachedStatus.lookup(CType::STATUS_BUSINESS, BIZ_STATUS_UNKNOWN) == BIZ_STATUS_EXPIRED;
      bool blocked = client->mCachedStatus.lookup(CType::STATUS_BLOCKED, 0) == 1;
  
      mConfig.mError = NO_SYNC_ERROR;
      mConfig.mEnabled = true;
  
      // the order is important here: a user needs to resolve blocked in order to resolve storage
      if (overStorage)
      {
          mConfig.mError = STORAGE_OVERQUOTA;
      }
      else if (businessExpired)
      {
          mConfig.mError = BUSINESS_EXPIRED;
      }
      else if (blocked)
      {
          mConfig.mError = ACCOUNT_BLOCKED;
      }
  
      if (mConfig.mError)
      {
          // save configuration but avoid creating active sync, and set as temporary disabled:
          mClient.syncs.saveSyncConfig(mConfig);
          return API_EFAILED;
      }
  
      auto prevFingerprint = mConfig.getLocalFingerprint();
  
      assert(!mSync);
      mSync.reset(new Sync(*this, debris, localdebris, remotenode, inshare));
      mConfig.setLocalFingerprint(mSync->fsfp);
  
      if (prevFingerprint && prevFingerprint != mConfig.getLocalFingerprint())
      {
          LOG_err << "New sync local fingerprint mismatch. Previous: " << prevFingerprint
              << "  Current: " << mConfig.getLocalFingerprint();
          mSync->changestate(SYNC_FAILED, LOCAL_FINGERPRINT_MISMATCH, false, true);
          mConfig.mError = LOCAL_FINGERPRINT_MISMATCH;
          mConfig.mEnabled = false;
          mSync.reset();
          return API_EFAILED;
      }
  
      mSync->isnetwork = isNetwork;
  
      if (!mSync->fsstableids)
      {
          if (mSync->assignfsids())
          {
              LOG_info << "Successfully assigned fs IDs for filesystem with unstable IDs";
          }
          else
          {
              LOG_warn << "Failed to assign some fs IDs for filesystem with unstable IDs";
          }
      }
  
      if (delayInitialScan)
      {
          client->syncs.saveSyncConfig(mConfig);
      }
      else
      {
          LOG_debug << "Initial scan sync: " << mConfig.getLocalPath().toPath(*client->fsaccess);
  
          if (mSync->scan(&rootpath, openedLocalFolder.get()))
          {
              client->syncsup = false;
              mSync->initializing = false;
              LOG_debug << "Initial scan finished. New / modified files: " << mSync->dirnotify->notifyq[DirNotify::DIREVENTS].size();
  
              // Sync constructor now receives the syncConfig as reference, to be able to write -at least- fingerprints for new syncs
              client->syncs.saveSyncConfig(mConfig);
          }
          else
          {
              LOG_err << "Initial scan failed";
              mSync->changestate(SYNC_FAILED, INITIAL_SCAN_FAILED, mConfig.getEnabled(), true);
  
              mSync.reset();
              return API_EFAILED;
          }
      }
      return API_OK;
  }
  
  void UnifiedSync::changedConfigState(bool notifyApp)
  {
      if (mConfig.errorOrEnabledChanged())
      {
          LOG_debug << "Sync " << toHandle(mConfig.mBackupId) << " enabled/error changed to " << mConfig.mEnabled << "/" << mConfig.mError;
  
          mClient.syncs.saveSyncConfig(mConfig);
          if (notifyApp)
          {
              mClient.app->syncupdate_stateconfig(mConfig.getBackupId());
          }
          mClient.abortbackoff(false);
      }
  }
  
  Syncs::Syncs(MegaClient& mc)
    : mClient(mc)
  {
      mHeartBeatMonitor.reset(new BackupMonitor(&mClient));
  }
  
  JSONSyncConfigDB* Syncs::syncConfigDB()
  {
      // Have we already created the database?
      if (mSyncConfigDB)
      {
          // Yep, return a reference to the caller.
          return mSyncConfigDB.get();
      }
  
      // Is the client using a database?
      if (!mClient.dbaccess)
      {
          // Nope and we need it for the configuration path.
          return nullptr;
      }
  
      // Can we get our hands on an IO context?
      if (!syncConfigIOContext())
      {
          // We need it if we want to write the DB to disk.
          return nullptr;
      }
  
      // Where the database will be stored.
      auto dbPath = mClient.dbaccess->rootPath();
  
      // Create the database.
      mSyncConfigDB.reset(new JSONSyncConfigDB(dbPath));
  
      return mSyncConfigDB.get();
  }
  
  bool Syncs::syncConfigDBDirty()
  {
      return mSyncConfigDB && mSyncConfigDB->dirty();
  }
  
  error Syncs::syncConfigDBFlush()
  {
      if (!syncConfigDBDirty())
      {
          return API_OK;
      }
  
      LOG_debug << "Attempting to flush internal config database.";
  
      auto result = API_EAGAIN;
  
      if (auto* ioContext = syncConfigIOContext())
      {
          result = mSyncConfigDB->write(*ioContext);
      }
  
      if (result != API_OK)
      {
          LOG_err << "Couldn't flush internal config database to disk: "
                  << result;
      }
      else
      {
          LOG_debug << "Internal config database flushed to disk.";
      }
  
      return result;
  }
  
  error Syncs::syncConfigDBLoad()
  {
      LOG_debug << "Attempting to load internal sync configs from disk.";
  
      auto result = API_EAGAIN;
  
      // Can we get our hands on the internal sync config database?
      if (auto* db = syncConfigDB())
      {
          // Make sure we have a suitable IO context.
          if (auto* ioContext = syncConfigIOContext())
          {
              // Try and read the database from disk.
              result = db->read(*ioContext);
          }
  
          if (result == API_OK || result == API_ENOENT)
          {
              LOG_debug << "Loaded "
                        << db->configs().size()
                        << " internal sync config(s) from disk.";
  
              return API_OK;
          }
      }
  
      LOG_err << "Couldn't load internal sync configs from disk: "
              << result;
  
      return result;
  }
  
  JSONSyncConfigIOContext* Syncs::syncConfigIOContext()
  {
      // Has a suitable IO context already been created?
      if (mSyncConfigIOContext)
      {
          // Yep, return a reference to it.
          return mSyncConfigIOContext.get();
      }
  
      // Which user are we?
      User* self = mClient.ownuser();
      if (!self)
      {
          LOG_warn << "syncConfigIOContext: own user not available";
          return nullptr;
      }
  
      // Try and retrieve this user's config data attribute.
      auto* payload = self->getattr(ATTR_JSON_SYNC_CONFIG_DATA);
      if (!payload)
      {
          // Attribute hasn't been created yet.
          LOG_warn << "syncConfigIOContext: JSON config data is not available";
          return nullptr;
      }
  
      // Try and decrypt the payload.
      unique_ptr<TLVstore> store(
        TLVstore::containerToTLVrecords(payload, &mClient.key));
  
      if (!store)
      {
          // Attribute is malformed.
          LOG_err << "syncConfigIOContext: JSON config data is malformed";
          return nullptr;
      }
  
      // Convenience.
      constexpr size_t KEYLENGTH = SymmCipher::KEYLENGTH;
  
      // Verify payload contents.
      auto authKey = store->get("ak");
      auto cipherKey = store->get("ck");
      auto name = store->get("fn");
  
      if (authKey.size() != KEYLENGTH
          || cipherKey.size() != KEYLENGTH
          || name.size() != KEYLENGTH)
      {
          // Payload is malformed.
          LOG_err << "syncConfigIOContext: JSON config data is incomplete";
          return nullptr;
      }
  
      // Create the IO context.
      mSyncConfigIOContext.reset(
        new JSONSyncConfigIOContext(*mClient.fsaccess,
                                    std::move(authKey),
                                    std::move(cipherKey),
                                    Base64::btoa(name),
                                    mClient.rng));
  
      // Return a reference to the new IO context.
      return mSyncConfigIOContext.get();
  }
  
  void Syncs::clear()
  {
      syncConfigDBFlush();
  
      mSyncConfigDB.reset();
      mSyncConfigIOContext.reset();
      mSyncVec.clear();
      isEmpty = true;
  }
  
  error Syncs::truncate()
  {
      if (!mSyncConfigDB)
      {
          return API_OK;
      }
  
      auto* ioContext = syncConfigIOContext();
  
      if (!ioContext)
      {
          return API_EAGAIN;
      }
  
      auto result = mSyncConfigDB->truncate(*ioContext);
  
      if (result != API_OK)
      {
          auto& fsAccess = *mClient.fsaccess;
  
          LOG_warn << "Unable to truncate config DB: "
                   << mSyncConfigDB->dbPath().toPath(fsAccess);
      }
  
      return result;
  }
  
  void Syncs::resetSyncConfigDb()
  {
      mSyncConfigDB.reset();
      static_cast<void>(syncConfigDB());
  }
  
  auto Syncs::appendNewSync(const SyncConfig& c, MegaClient& mc) -> UnifiedSync*
  {
      isEmpty = false;
      mSyncVec.push_back(unique_ptr<UnifiedSync>(new UnifiedSync(mc, c)));
  
      saveSyncConfig(c);
  
      return mSyncVec.back().get();
  }
  
  Sync* Syncs::runningSyncByBackupId(handle backupId) const
  {
      for (auto& s : mSyncVec)
      {
          if (s->mSync && s->mConfig.getBackupId() == backupId)
          {
              return s->mSync.get();
          }
      }
      return nullptr;
  }
  
  SyncConfig* Syncs::syncConfigByBackupId(handle backupId) const
  {
      for (auto& s : mSyncVec)
      {
          if (s->mConfig.getBackupId() == backupId)
          {
              return &s->mConfig;
          }
      }
  
      return nullptr;
  }
  
  void Syncs::forEachUnifiedSync(std::function<void(UnifiedSync&)> f)
  {
      for (auto& s : mSyncVec)
      {
          f(*s);
      }
  }
  
  void Syncs::forEachRunningSync(std::function<void(Sync* s)> f)
  {
      for (auto& s : mSyncVec)
      {
          if (s->mSync)
          {
              f(s->mSync.get());
          }
      }
  }
  
  void Syncs::forEachRunningSyncContainingNode(Node* node, std::function<void(Sync* s)> f)
  {
      for (auto& s : mSyncVec)
      {
          if (s->mSync)
          {
              if (s->mSync->localroot->node &&
                  node->isbelow(s->mSync->localroot->node))
              {
                  f(s->mSync.get());
              }
          }
      }
  }
  
  bool Syncs::forEachRunningSync_shortcircuit(std::function<bool(Sync* s)> f)
  {
      for (auto& s : mSyncVec)
      {
          if (s->mSync)
          {
              if (!f(s->mSync.get()))
              {
                  return false;
              }
          }
      }
      return true;
  }
  
  void Syncs::forEachSyncConfig(std::function<void(const SyncConfig&)> f)
  {
      for (auto& s : mSyncVec)
      {
          f(s->mConfig);
      }
  }
  
  bool Syncs::hasRunningSyncs()
  {
      for (auto& s : mSyncVec)
      {
          if (s->mSync) return true;
      }
      return false;
  }
  
  unsigned Syncs::numRunningSyncs()
  {
      unsigned n = 0;
      for (auto& s : mSyncVec)
      {
          if (s->mSync) ++n;
      }
      return n;
  }
  
  Sync* Syncs::firstRunningSync()
  {
      for (auto& s : mSyncVec)
      {
          if (s->mSync) return s->mSync.get();
      }
      return nullptr;
  }
  
  void Syncs::stopCancelledFailedDisabled()
  {
      for (auto& unifiedSync : mSyncVec)
      {
          if (unifiedSync->mSync && (
              unifiedSync->mSync->state == SYNC_CANCELED ||
              unifiedSync->mSync->state == SYNC_FAILED ||
              unifiedSync->mSync->state == SYNC_DISABLED))
          {
              unifiedSync->mSync.reset();
          }
      }
  }
  
  void Syncs::purgeRunningSyncs()
  {
      // Called from locallogout (which always happens on ~MegaClient as well as on request)
      // Any syncs that are running should be resumed on next start.
      // We stop the syncs here, but don't call the client to say they are stopped.
      // And localnode databases are preserved.
      for (auto& s : mSyncVec)
      {
          if (s->mSync)
          {
              // Deleting the sync will close/save the sync's localnode database file in its current state.
              // And then delete objects in RAM.
              s->mSync.reset();
          }
      }
  }
  
  void Syncs::disableSyncs(SyncError syncError, bool newEnabledFlag)
  {
      bool anySyncDisabled = false;
      disableSelectedSyncs([&](SyncConfig&, Sync* s){
  
          if (s)
          {
              anySyncDisabled = true;
              return true;
          }
          return false;
      }, syncError, newEnabledFlag);
  
      if (anySyncDisabled)
      {
          LOG_info << "Disabled syncs. error = " << syncError;
          mClient.app->syncs_disabled(syncError);
      }
  }
  
  void Syncs::disableSelectedSyncs(std::function<bool(SyncConfig&, Sync*)> selector, SyncError syncError, bool newEnabledFlag)
  {
      for (auto i = mSyncVec.size(); i--; )
      {
          if (selector(mSyncVec[i]->mConfig, mSyncVec[i]->mSync.get()))
          {
              if (auto sync = mSyncVec[i]->mSync.get())
              {
                  sync->changestate(SYNC_DISABLED, syncError, newEnabledFlag, true); //This will cause the later deletion of Sync (not MegaSyncPrivate) object
                  mClient.syncactivity = true;
              }
              else
              {
                  mSyncVec[i]->mConfig.setError(syncError);
                  mSyncVec[i]->mConfig.setEnabled(newEnabledFlag);
                  mSyncVec[i]->changedConfigState(true);
              }
          }
      }
  }
  
  void Syncs::removeSelectedSyncs(std::function<bool(SyncConfig&, Sync*)> selector)
  {
      for (auto i = mSyncVec.size(); i--; )
      {
          if (selector(mSyncVec[i]->mConfig, mSyncVec[i]->mSync.get()))
          {
              removeSyncByIndex(i);
          }
      }
  }
  
  void Syncs::removeSyncByIndex(size_t index)
  {
      if (index < mSyncVec.size())
      {
          if (auto& syncPtr = mSyncVec[index]->mSync)
          {
              syncPtr->changestate(SYNC_CANCELED, UNKNOWN_ERROR, false, false);
  
              if (syncPtr->statecachetable)
              {
                  syncPtr->statecachetable->remove();
                  delete syncPtr->statecachetable;
                  syncPtr->statecachetable = NULL;
              }
              syncPtr.reset(); // deletes sync
          }
  
          // call back before actual removal (intermediate layer may need to make a temp copy to call client app)
          auto backupId = mSyncVec[index]->mConfig.getBackupId();
          mClient.app->sync_removed(backupId);
  
          removeSyncConfigByBackupId(backupId);
          mClient.syncactivity = true;
          mSyncVec.erase(mSyncVec.begin() + index);
  
          isEmpty = mSyncVec.empty();
      }
  }
  
  error Syncs::removeSyncConfigByBackupId(handle backupId)
  {
      error result = API_OK;
  
      if (auto* db = syncConfigDB())
      {
          result = db->removeByBackupId(backupId);
      }
  
      if (result == API_ENOENT)
      {
          LOG_err << "Found no config for backupId: "
                  << toHandle(backupId)
                  << " upon sync removal.";
      }
  
      return result;
  }
  
  error Syncs::enableSyncByBackupId(handle backupId, bool resetFingerprint, UnifiedSync*& syncPtrRef)
  {
      for (auto& s : mSyncVec)
      {
          if (s->mConfig.getBackupId() == backupId)
          {
              syncPtrRef = s.get();
              if (!s->mSync)
              {
                  return s->enableSync(resetFingerprint, true);
              }
              return API_EEXIST;
          }
      }
      return API_ENOENT;
  }
  
  error Syncs::saveSyncConfig(const SyncConfig& config)
  {
      assert(!config.isExternal());
  
      if (auto* db = syncConfigDB())
      {
          return db->add(config), API_OK;
      }
  
      return API_ENOENT;
  }
  
  // restore all configured syncs that were in a temporary error state (not manually disabled)
  void Syncs::enableResumeableSyncs()
  {
      bool anySyncRestored = false;
  
      for (auto& unifiedSync : mSyncVec)
      {
          if (!unifiedSync->mSync)
          {
              if (unifiedSync->mConfig.getEnabled())
              {
                  SyncError syncError = unifiedSync->mConfig.getError();
                  LOG_debug << "Restoring sync: " << toHandle(unifiedSync->mConfig.getBackupId()) << " " << unifiedSync->mConfig.getLocalPath().toPath(*mClient.fsaccess) << " fsfp= " << unifiedSync->mConfig.getLocalFingerprint() << " old error = " << syncError;
  
                  error e = unifiedSync->enableSync(false, true);
                  if (!e)
                  {
                      anySyncRestored = true;
                  }
              }
              else
              {
                  LOG_verbose << "Skipping restoring sync: " << unifiedSync->mConfig.getLocalPath().toPath(*mClient.fsaccess)
                      << " enabled=" << unifiedSync->mConfig.getEnabled() << " error=" << unifiedSync->mConfig.getError();
              }
          }
      }
  
      if (anySyncRestored)
      {
          mClient.app->syncs_restored();
      }
  }
  
  void Syncs::resumeResumableSyncsOnStartup()
  {
      if (mClient.loggedin() != FULLACCOUNT) return;
  
      bool firstSyncResumed = false;
  
      if (syncConfigDBLoad() != API_OK)
      {
          return;
      }
  
      for (auto& pair : syncConfigDB()->configs())
      {
          mSyncVec.push_back(unique_ptr<UnifiedSync>(new UnifiedSync(mClient, pair.second)));
          isEmpty = false;
      }
  
      for (auto& unifiedSync : mSyncVec)
      {
          if (!unifiedSync->mSync)
          {
              if (unifiedSync->mConfig.mOrigninalPathOfRemoteRootNode.empty()) //should only happen if coming from old cache
              {
                  auto node = mClient.nodebyhandle(unifiedSync->mConfig.getRemoteNode());
                  unifiedSync->updateSyncRemoteLocation(node, false); //updates cache & notice app of this change
                  if (node)
                  {
                      auto newpath = node->displaypath();
                      unifiedSync->mConfig.mOrigninalPathOfRemoteRootNode = newpath;//update loaded config
                  }
              }
  
              if (unifiedSync->mConfig.getEnabled())
              {
                  if (!firstSyncResumed)
                  {
                      mClient.app->syncs_about_to_be_resumed();
                      firstSyncResumed = true;
                  }
  
  #ifdef __APPLE__
                  unifiedSync->mConfig.setLocalFingerprint(0); //for certain MacOS, fsfp seems to vary when restarting. we set it to 0, so that it gets recalculated
  #endif
                  LOG_debug << "Resuming cached sync: " << toHandle(unifiedSync->mConfig.getBackupId()) << " " << unifiedSync->mConfig.getLocalPath().toPath(*mClient.fsaccess) << " fsfp= " << unifiedSync->mConfig.getLocalFingerprint() << " error = " << unifiedSync->mConfig.getError();
  
                  unifiedSync->enableSync(false, false);
                  LOG_debug << "Sync autoresumed: " << toHandle(unifiedSync->mConfig.getBackupId()) << " " << unifiedSync->mConfig.getLocalPath().toPath(*mClient.fsaccess) << " fsfp= " << unifiedSync->mConfig.getLocalFingerprint() << " error = " << unifiedSync->mConfig.getError();
  
                  mClient.app->sync_auto_resume_result(*unifiedSync, true);
              }
              else
              {
                  LOG_debug << "Sync loaded (but not resumed): " << toHandle(unifiedSync->mConfig.getBackupId()) << " " << unifiedSync->mConfig.getLocalPath().toPath(*mClient.fsaccess) << " fsfp= " << unifiedSync->mConfig.getLocalFingerprint() << " error = " << unifiedSync->mConfig.getError();
                  mClient.app->sync_auto_resume_result(*unifiedSync, false);
              }
          }
      }
  }
  
  const unsigned int JSONSyncConfigDB::NUM_SLOTS = 2; //number of configuration versions to save. Do not set to > 10.
  
  JSONSyncConfigDB::JSONSyncConfigDB(const LocalPath& dbPath,
                                     const LocalPath& drivePath)
    : mDBPath(dbPath)
    , mDrivePath(drivePath)
    , mBackupIdToConfig()
    , mSlot(0)
    , mDirty(false)
  {
  }
  
  JSONSyncConfigDB::JSONSyncConfigDB(const LocalPath& dbPath)
    : mDBPath(dbPath)
    , mDrivePath()
    , mBackupIdToConfig()
    , mSlot(0)
    , mDirty(false)
  {
  }
  
  JSONSyncConfigDB::~JSONSyncConfigDB()
  {
      // Drop the configs.
      clear(false);
  }
  
  const SyncConfig* JSONSyncConfigDB::add(const SyncConfig& config,
                                          const bool flush)
  {
      auto it = mBackupIdToConfig.find(config.getBackupId());
  
      // Do we already have a config with this tag?
      if (it != mBackupIdToConfig.end())
      {
          // Mark the database as being dirty.
          mDirty |= flush;
  
          // Update the config.
          it->second = config;
  
          // We're done.
          return &it->second;
      }
  
      // Add the config to the database.
      auto result = mBackupIdToConfig.emplace(config.getBackupId(), config);
  
      // Mark the database as being dirty.
      mDirty |= flush;
  
      // We're done.
      return &result.first->second;
  }
  
  void JSONSyncConfigDB::clear(const bool flush)
  {
      // Are there any configs to remove?
      if (mBackupIdToConfig.empty())
      {
          // Nope.
          return;
      }
  
      // Mark the database as being dirty.
      mDirty |= flush;
  
      // Clear the config database.
      mBackupIdToConfig.clear();
  }
  
  const JSONSyncConfigMap& JSONSyncConfigDB::configs() const
  {
      return mBackupIdToConfig;
  }
  
  bool JSONSyncConfigDB::dirty() const
  {
      return mDirty;
  }
  
  const LocalPath& JSONSyncConfigDB::dbPath() const
  {
      return mDBPath;
  }
  
  const LocalPath& JSONSyncConfigDB::drivePath() const
  {
      return mDrivePath;
  }
  
  const SyncConfig* JSONSyncConfigDB::getByBackupId(handle backupId) const
  {
      auto it = mBackupIdToConfig.find(backupId);
  
      if (it != mBackupIdToConfig.end())
      {
          return &it->second;
      }
  
      return nullptr;
  }
  
  const SyncConfig* JSONSyncConfigDB::getByRootHandle(handle targetHandle) const
  {
      if (targetHandle == UNDEF)
      {
          return nullptr;
      }
  
      for (auto& it : mBackupIdToConfig)
      {
          if (it.second.getRemoteNode() == targetHandle)
          {
              return &it.second;
          }
      }
  
      return nullptr;
  }
  
  error JSONSyncConfigDB::read(JSONSyncConfigIOContext& ioContext)
  {
      vector<unsigned int> confSlots;
  
      // Determine which slots we should load first, if any.
      if (ioContext.getSlotsInOrder(mDBPath, confSlots) != API_OK)
      {
          // Couldn't get a list of slots.
          return API_ENOENT;
      }
  
      // Try and load the database from one of the slots.
      for (const auto& slot : confSlots)
      {
          // Can we read the database from this slot?
          if (read(ioContext, slot) == API_OK)
          {
              // Update the slot number.
              mSlot = (slot + 1) % NUM_SLOTS;
  
              // Yep, loaded.
              return API_OK;
          }
      }
  
      // Up to date with disk.
      mDirty = false;
  
      // Couldn't load the database.
      return API_EREAD;
  }
  
  error JSONSyncConfigDB::removeByBackupId(handle backupId)
  {
      // Remove the config, if present and flush.
      return removeByBackupId(backupId, true);
  }
  
  error JSONSyncConfigDB::removeByRootHandle(const handle targetHandle)
  {
      // Any config present with the given target handle?
      if (const auto* config = getByRootHandle(targetHandle))
      {
          // Yep, remove it.
          return removeByBackupId(config->getBackupId(), true);
      }
  
      // Nope.
      return API_ENOENT;
  }
  
  error JSONSyncConfigDB::truncate(JSONSyncConfigIOContext& ioContext)
  {
      // Purge configs from memory.
      mBackupIdToConfig.clear();
  
      // Database no longer has any changes.
      mDirty = false;
  
      // Reset slot counter to initial value.
      mSlot = 0;
  
      // Purge (existing) slots from disk.
      return ioContext.remove(mDBPath);
  }
  
  error JSONSyncConfigDB::write(JSONSyncConfigIOContext& ioContext)
  {
      JSONWriter writer;
  
      // Serialize the database.
      ioContext.serialize(mBackupIdToConfig, writer);
  
      // Try and write the database out to disk.
      if (ioContext.write(mDBPath, writer.getstring(), mSlot) != API_OK)
      {
          // Couldn't write the database out to disk.
          return API_EWRITE;
      }
  
      // Rotate the slot.
      mSlot = (mSlot + 1) % NUM_SLOTS;
  
      // Remove the old file so there's no race with fuzzy file timestamps in fast-running tests
      ioContext.remove(mDBPath, mSlot);
  
      // Up to date with disk.
      mDirty = false;
  
      return API_OK;
  }
  
  error JSONSyncConfigDB::read(JSONSyncConfigIOContext& ioContext,
                               const unsigned int slot)
  {
      // Try and read the database from the specified slot.
      string data;
  
      if (ioContext.read(mDBPath, data, slot) != API_OK)
      {
          // Couldn't read the database.
          return API_EREAD;
      }
  
      // Try and deserialize the configs contained in the database.
      JSONSyncConfigMap configs;
      JSON reader(data);
  
      if (!ioContext.deserialize(mDBPath, configs, reader, slot))
      {
          // Couldn't deserialize the configs.
          return API_EREAD;
      }
  
      // Remove configs that aren't present on disk.
      auto i = mBackupIdToConfig.begin();
      auto j = mBackupIdToConfig.end();
  
      while (i != j)
      {
          const auto backupId = i++->first;
  
          if (!configs.count(backupId))
          {
              removeByBackupId(backupId, false);
          }
      }
  
      // Add (or update) configs.
      for (auto& it : configs)
      {
          // Correct config's drive path.
          it.second.mExternalDrivePath = mDrivePath;
  
          // Add / update the config.
          add(it.second, false);
      }
  
      return API_OK;
  }
  
  error JSONSyncConfigDB::removeByBackupId(handle backupId, const bool flush)
  {
      auto it = mBackupIdToConfig.find(backupId);
  
      // Any config present with the given tag?
      if (it == mBackupIdToConfig.end())
      {
          // Nope.
          return API_ENOENT;
      }
  
      // Mark the database as being dirty.
      mDirty |= flush;
  
      // Remove the config from the database.
      mBackupIdToConfig.erase(it);
  
      // We're done.
      return API_OK;
  }
  
  const string JSONSyncConfigIOContext::NAME_PREFIX = "megaclient_syncconfig_";
  
  JSONSyncConfigIOContext::JSONSyncConfigIOContext(FileSystemAccess& fsAccess,
                                                   const string& authKey,
                                                   const string& cipherKey,
                                                   const string& name,
                                                   PrnGen& rng)
    : mCipher()
    , mFsAccess(fsAccess)
    , mName(LocalPath::fromPath(NAME_PREFIX + name, mFsAccess))
    , mRNG(rng)
    , mSigner()
  {
      // Convenience.
      constexpr size_t KEYLENGTH = SymmCipher::KEYLENGTH;
  
      // These attributes *must* be sane.
      assert(authKey.size() == KEYLENGTH);
      assert(cipherKey.size() == KEYLENGTH);
      assert(name.size() == Base64Str<KEYLENGTH>::STRLEN);
  
      // Load the authentication key into our internal signer.
      mSigner.setkey(reinterpret_cast<const byte*>(authKey.data()), KEYLENGTH);
  
      // Load the encryption key into our internal cipher.
      mCipher.setkey(reinterpret_cast<const byte*>(cipherKey.data()));
  }
  
  JSONSyncConfigIOContext::~JSONSyncConfigIOContext()
  {
  }
  
  bool JSONSyncConfigIOContext::deserialize(const LocalPath& dbPath,
                                            JSONSyncConfigMap& configs,
                                            JSON& reader,
                                            const unsigned int slot) const
  {
      auto path = dbFilePath(dbPath, slot).toPath(mFsAccess);
  
      LOG_debug << "Attempting to deserialize config DB: "
                << path;
  
      if (deserialize(configs, reader))
      {
          LOG_debug << "Successfully deserialized config DB: "
                    << path;
  
          return true;
      }
  
      LOG_debug << "Unable to deserialize config DB: "
                << path;
  
      return false;
  }
  
  bool JSONSyncConfigIOContext::deserialize(JSONSyncConfigMap& configs,
                                            JSON& reader) const
  {
      const auto TYPE_SYNCS = MAKENAMEID2('s', 'y');
  
      if (!reader.enterobject())
      {
          return false;
      }
  
      for ( ; ; )
      {
          switch (reader.getnameid())
          {
          case EOO:
              return reader.leaveobject();
  
          case TYPE_SYNCS:
          {
              if (!reader.enterarray())
              {
                  return false;
              }
  
              while (reader.enterobject())
              {
                  SyncConfig config;
  
                  if (deserialize(config, reader))
                  {
                      // So move is well-defined.
                      const auto backupId = config.mBackupId;
  
                      configs.emplace(backupId, std::move(config));
                  }
                  else
                  {
                      LOG_err << "Failed to deserialize a sync config";
                      assert(false);
                  }
  
                  reader.leaveobject();
              }
  
              if (!reader.leavearray())
              {
                  return false;
              }
  
              break;
          }
  
          default:
              if (!reader.storeobject())
              {
                  return false;
              }
              break;
          }
      }
  }
  
  error JSONSyncConfigIOContext::getSlotsInOrder(const LocalPath& dbPath,
                                                 vector<unsigned int>& confSlots)
  {
      using std::isdigit;
      using std::sort;
  
      using SlotTimePair = pair<unsigned int, m_time_t>;
  
      // Glob for configuration directory.
      LocalPath globPath = dbPath;
  
      globPath.appendWithSeparator(mName, false);
      globPath.append(LocalPath::fromPath(".?", mFsAccess));
  
      // Open directory for iteration.
      unique_ptr<DirAccess> dirAccess(mFsAccess.newdiraccess());
  
      if (!dirAccess->dopen(&globPath, nullptr, true))
      {
          // Couldn't open directory for iteration.
          return API_ENOENT;
      }
  
      auto fileAccess = mFsAccess.newfileaccess(false);
      LocalPath filePath;
      vector<SlotTimePair> slotTimes;
      nodetype_t type;
  
      // Iterate directory.
      while (dirAccess->dnext(globPath, filePath, false, &type))
      {
          // Skip directories.
          if (type != FILENODE)
          {
              continue;
          }
  
          // Determine slot suffix.
          const char suffix = filePath.toPath(mFsAccess).back();
  
          // Skip invalid suffixes.
          if (!isdigit(suffix))
          {
              continue;
          }
  
          // Determine file's modification time.
          if (!fileAccess->fopen(filePath))
          {
              // Couldn't stat file.
              continue;
          }
  
          // Record this slot-time pair.
          unsigned int slot = suffix - 0x30; // convert char to int
          slotTimes.emplace_back(slot, fileAccess->mtime);
      }
  
      // Sort the list of slot-time pairs.
      sort(slotTimes.begin(),
           slotTimes.end(),
           [](const SlotTimePair& lhs, const SlotTimePair& rhs)
           {
               // Order by descending modification time.
               if (lhs.second != rhs.second)
               {
                   return lhs.second > rhs.second;
               }
  
               // Otherwise by descending slot.
               return lhs.first > rhs.first;
           });
  
      // Transmit sorted list of slots to the caller.
      for (const auto& slotTime : slotTimes)
      {
          confSlots.emplace_back(slotTime.first);
      }
  
      return API_OK;
  }
  
  error JSONSyncConfigIOContext::read(const LocalPath& dbPath,
                                      string& data,
                                      const unsigned int slot)
  {
      // Generate path to the configuration file.
      LocalPath path = dbFilePath(dbPath, slot);
  
      LOG_debug << "Attempting to read config DB: "
                << path.toPath(mFsAccess);
  
      // Try and open the file for reading.
      auto fileAccess = mFsAccess.newfileaccess(false);
  
      if (!fileAccess->fopen(path, true, false))
      {
          // Couldn't open the file for reading.
          LOG_debug << "Unable to open config DB for reading: "
                    << path.toPath(mFsAccess);
  
          return API_EREAD;
      }
  
      // Try and read the data from the file.
      string d;
  
      if (!fileAccess->fread(&d, static_cast<unsigned>(fileAccess->size), 0, 0x0))
      {
          // Couldn't read the file.
          LOG_debug << "Unable to read config DB: "
                    << path.toPath(mFsAccess);
  
          return API_EREAD;
      }
  
      // Try and decrypt the data.
      if (!decrypt(d, data))
      {
          // Couldn't decrypt the data.
          LOG_debug << "Unable to decrypt config DB: "
                    << path.toPath(mFsAccess);
  
          return API_EREAD;
      }
  
      LOG_debug << "Config DB successfully read from disk: "
                << path.toPath(mFsAccess)
                << ": "
                << data;
  
      return API_OK;
  }
  
  error JSONSyncConfigIOContext::remove(const LocalPath& dbPath,
                                        const unsigned int slot)
  {
      LocalPath path = dbFilePath(dbPath, slot);
  
      if (!mFsAccess.unlinklocal(path))
      {
          LOG_warn << "Unable to remove config DB: "
                   << path.toPath(mFsAccess);
  
          return API_EWRITE;
      }
  
      return API_OK;
  }
  
  error JSONSyncConfigIOContext::remove(const LocalPath& dbPath)
  {
      vector<unsigned int> confSlots;
  
      // What slots are present on disk?
      if (getSlotsInOrder(dbPath, confSlots) == API_ENOENT)
      {
          // None so nothing to do.
          return API_ENOENT;
      }
  
      bool result = true;
  
      // Remove the slots from disk.
      for (auto confSlot : confSlots)
      {
          result &= remove(dbPath, confSlot) == API_OK;
      }
  
      // Signal success only if all slots could be removed.
      return result ? API_OK : API_EWRITE;
  }
  
  void JSONSyncConfigIOContext::serialize(const JSONSyncConfigMap& configs,
                                          JSONWriter& writer) const
  {
      writer.beginobject();
      writer.beginarray("sy");
  
      for (const auto& it : configs)
      {
          serialize(it.second, writer);
      }
  
      writer.endarray();
      writer.endobject();
  }
  
  error JSONSyncConfigIOContext::write(const LocalPath& dbPath,
                                       const string& data,
                                       const unsigned int slot)
  {
      LocalPath path = dbPath;
  
      LOG_debug << "Attempting to write config DB: "
                << dbPath.toPath(mFsAccess)
                << " / "
                << slot;
  
      // Try and create the backup configuration directory.
      if (!(mFsAccess.mkdirlocal(path) || mFsAccess.target_exists))
      {
          LOG_debug << "Unable to create config DB directory: "
                    << dbPath.toPath(mFsAccess);
  
          // Couldn't create the directory and it doesn't exist.
          return API_EWRITE;
      }
  
      // Generate the rest of the path.
      path = dbFilePath(dbPath, slot);
  
      // Open the file for writing.
      auto fileAccess = mFsAccess.newfileaccess(false);
  
      if (!fileAccess->fopen(path, false, true))
      {
          // Couldn't open the file for writing.
          LOG_debug << "Unable to open config DB for writing: "
                    << path.toPath(mFsAccess);
  
          return API_EWRITE;
      }
  
      // Ensure the file is empty.
      if (!fileAccess->ftruncate())
      {
          // Couldn't truncate the file.
          LOG_debug << "Unable to truncate config DB: "
                    << path.toPath(mFsAccess);
  
          return API_EWRITE;
      }
  
      // Encrypt the configuration data.
      const string d = encrypt(data);
  
      // Write the encrypted configuration data.
      auto* bytes = reinterpret_cast<const byte*>(&d[0]);
  
      if (!fileAccess->fwrite(bytes, static_cast<unsigned>(d.size()), 0x0))
      {
          // Couldn't write out the data.
          LOG_debug << "Unable to write config DB: "
                    << path.toPath(mFsAccess);
  
          return API_EWRITE;
      }
  
      LOG_debug << "Config DB successfully written to disk: "
                << path.toPath(mFsAccess)
                << ": "
                << data;
  
      return API_OK;
  }
  
  LocalPath JSONSyncConfigIOContext::dbFilePath(const LocalPath& dbPath,
                                                const unsigned int slot) const
  {
      using std::to_string;
  
      LocalPath path = dbPath;
  
      path.appendWithSeparator(mName, false);
      path.append(LocalPath::fromPath("." + to_string(slot), mFsAccess));
  
      return path;
  }
  
  bool JSONSyncConfigIOContext::decrypt(const string& in, string& out)
  {
      // Handy constants.
      const size_t IV_LENGTH       = SymmCipher::KEYLENGTH;
      const size_t MAC_LENGTH      = 32;
      const size_t METADATA_LENGTH = IV_LENGTH + MAC_LENGTH;
  
      // Is the file too short to be valid?
      if (in.size() <= METADATA_LENGTH)
      {
          return false;
      }
  
      // For convenience (format: <data><iv><hmac>)
      const byte* data = reinterpret_cast<const byte*>(&in[0]);
      const byte* iv   = &data[in.size() - METADATA_LENGTH];
      const byte* mac  = &data[in.size() - MAC_LENGTH];
  
      byte cmac[MAC_LENGTH];
  
      // Compute HMAC on file.
      mSigner.add(data, in.size() - MAC_LENGTH);
      mSigner.get(cmac);
  
      // Is the file corrupt?
      if (memcmp(cmac, mac, MAC_LENGTH))
      {
          return false;
      }
  
      // Try and decrypt the file.
      return mCipher.cbc_decrypt_pkcs_padding(data,
                                              in.size() - METADATA_LENGTH,
                                              iv,
                                              &out);
  }
  
  bool JSONSyncConfigIOContext::deserialize(SyncConfig& config, JSON& reader) const
  {
      const auto TYPE_BACKUP_ID       = MAKENAMEID2('i', 'd');
      const auto TYPE_ENABLED         = MAKENAMEID2('e', 'n');
      const auto TYPE_FINGERPRINT     = MAKENAMEID2('f', 'p');
      const auto TYPE_LAST_ERROR      = MAKENAMEID2('l', 'e');
      const auto TYPE_LAST_WARNING    = MAKENAMEID2('l', 'w');
      const auto TYPE_NAME            = MAKENAMEID1('n');
      const auto TYPE_SOURCE_PATH     = MAKENAMEID2('s', 'p');
      const auto TYPE_SYNC_TYPE       = MAKENAMEID2('s', 't');
      const auto TYPE_TARGET_HANDLE   = MAKENAMEID2('t', 'h');
      const auto TYPE_TARGET_PATH     = MAKENAMEID2('t', 'p');
      const auto TYPE_EXCLUSION_RULES = MAKENAMEID2('e', 'r');
  
      for ( ; ; )
      {
          switch (reader.getnameid())
          {
          case EOO:
              // success if we reached the end of the object
              return *reader.pos == '}';
  
          case TYPE_ENABLED:
              config.mEnabled = reader.getbool();
              break;
  
          case TYPE_FINGERPRINT:
              config.mLocalFingerprint = reader.getfp();
              break;
  
          case TYPE_LAST_ERROR:
              config.mError =
                static_cast<SyncError>(reader.getint32());
              break;
  
          case TYPE_LAST_WARNING:
              config.mWarning =
                static_cast<SyncWarning>(reader.getint32());
              break;
  
          case TYPE_NAME:
              reader.storebinary(&config.mName);
              break;
  
          case TYPE_SOURCE_PATH:
          {
              string sourcePath;
  
              reader.storebinary(&sourcePath);
  
              config.mLocalPath =
                LocalPath::fromPath(sourcePath, mFsAccess);
  
              break;
          }
  
          case TYPE_SYNC_TYPE:
              config.mSyncType =
                static_cast<SyncConfig::Type>(reader.getint32());
              break;
  
          case TYPE_BACKUP_ID:
              config.mBackupId = reader.gethandle(sizeof(handle));
              break;
  
          case TYPE_TARGET_HANDLE:
              config.mRemoteNode = reader.gethandle(MegaClient::NODEHANDLE);
              if ((config.mRemoteNode & 0xFFFFFFFFFFFF) == (UNDEF & 0xFFFFFFFFFFFF))
              {
                  // we can have a much nicer solution when NodeHandle is merged from the sync rework branch
                  config.mRemoteNode = UNDEF;
              }
              break;
  
          case TYPE_TARGET_PATH:
              reader.storebinary(&config.mOrigninalPathOfRemoteRootNode);
              break;
  
          case TYPE_EXCLUSION_RULES:
          {
              if (!reader.enterarray()) return false;
              string s;
              while (reader.storeobject(&s))
              {
                  config.mRegExps.push_back(Base64::atob(s));
              }
              if (!reader.leavearray()) return false;
              break;
          }
  
          default:
              if (!reader.storeobject())
              {
                  return false;
              }
              break;
          }
      }
  }
  
  string JSONSyncConfigIOContext::encrypt(const string& data)
  {
      byte iv[SymmCipher::KEYLENGTH];
  
      // Generate initialization vector.
      mRNG.genblock(iv, sizeof(iv));
  
      string d;
  
      // Encrypt file using IV.
      mCipher.cbc_encrypt_pkcs_padding(&data, iv, &d);
  
      // Add IV to file.
      d.append(std::begin(iv), std::end(iv));
  
      byte mac[32];
  
      // Compute HMAC on file (including IV).
      mSigner.add(reinterpret_cast<const byte*>(&d[0]), d.size());
      mSigner.get(mac);
  
      // Add HMAC to file.
      d.append(std::begin(mac), std::end(mac));
  
      // We're done.
      return d;
  }
  
  void JSONSyncConfigIOContext::serialize(const SyncConfig& config,
                                          JSONWriter& writer) const
  {
      writer.beginobject();
      writer.arg("id", config.getBackupId(), sizeof(handle));
      writer.arg_B64("sp", config.mLocalPath.toPath(mFsAccess));
      writer.arg_B64("n", config.mName);
      writer.arg_B64("tp", config.mOrigninalPathOfRemoteRootNode);
      writer.arg_fsfp("fp", config.mLocalFingerprint);
      writer.arg("th", config.mRemoteNode, MegaClient::NODEHANDLE);
      writer.arg("le", config.mError);
      writer.arg("lw", config.mWarning);
      writer.arg("st", config.mSyncType);
      writer.arg("en", config.mEnabled);
  
      writer.beginarray("er");
      for (auto& s : config.mRegExps)
      {
          // store as binary so the strings get btoa'd so no JSON injections
          writer.element_B64(s);
      }
      writer.endarray();
      writer.endobject();
  }
  
  } // namespace
  
  #endif
  