text: /**
   * Copyright 2016-2018 F5 Networks, Inc.
   *
   * Licensed under the Apache License, Version 2.0 (the "License");
   * you may not use this file except in compliance with the License.
   * You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  
  'use strict';
  
  const fs = require('fs');
  const q = require('q');
  const CloudProvider = require('../lib/cloudProvider');
  const util = require('../lib/util');
  const cryptoUtil = require('../lib/cryptoUtil');
  const childProcess = require('child_process');
  
  const BigIp = require('../lib/bigIp');
  const AutoscaleInstance = require('../lib/autoscaleInstance');
  const Logger = require('../lib/logger');
  const cloudProviderFactory = require('../lib/cloudProviderFactory');
  const dnsProviderFactory = require('../lib/dnsProviderFactory');
  const ipc = require('../lib/ipc');
  const commonOptions = require('./commonOptions');
  const BACKUP = require('../lib/sharedConstants').BACKUP;
  
  (function run() {
      const DEFAULT_MAX_DISCONNECTED_MS = 3 * 60000; // 3 minute
      const MIN_MS_BETWEEN_JOIN_REQUESTS = 5 * 60000; // 5 minutes
      const PRIMARY_FILE_PATH = '/config/cloud/master';
  
      const PASSPHRASE_LENGTH = 18;
  
      const AUTOSCALE_PRIVATE_KEY = 'cloudLibsAutoscalePrivate.key';
      const AUTOSCALE_PRIVATE_KEY_FOLDER = 'CloudLibsAutoscale';
  
      const UCS_BACKUP_PREFIX = 'ucsAutosave_';
      const UCS_BACKUP_DEFAULT_MAX_FILES = 7;
      const UCS_BACKUP_DIRECTORY = '/var/local/ucs';
      const DEFAULT_AUTOSCALE_TIMEOUT_IN_MINUTES = 10;
  
      let logger;
  
      const runner = {
  
          /**
           * Runs the autoscale script
           *
           * Provider is passed in only for testing. In production, provider will be instantiated
           * based on the --cloud option
           *
           * @param {String[]} argv                        - The process arguments.
           * @param {Ojbect}   [testOpts]                  - Options used during testing
           * @param {Object}   [testOpts.autoscleProvider] - Mock for provider.
           * @param {Object}   [testOpts.bigIp]            - Mock for BigIp.
           * @param {Function} [cb]                        - Optional cb to call when done
           */
          run(argv, testOpts, cb) {
              const DEFAULT_LOG_FILE = '/tmp/autoscale.log';
              const ARGS_FILE_ID = `autoscale_${Date.now()}`;
              const KEYS_TO_MASK = ['-p', '--password', '--big-iq-password'];
  
              const OPTIONS_TO_UNDEFINE = [
                  'bigIqPasswordUri',
                  'bigIqPassword',
                  'password',
                  'passwordUrl'
              ];
  
              const loggerOptions = {};
              const providerOptions = {};
              const dnsProviderOptions = {};
              const optionsForTest = {};
  
              let externalTag = {};
              let bigIp;
              let loggableArgs;
              let logFileName;
              let primaryInstance;
              let primaryIid;
              let primaryBad;
              let primaryBadReason;
              let newPrimary;
              let cloudProvider;
              let dnsProvider;
  
              Object.assign(optionsForTest, testOpts);
  
              try {
                  /* eslint-disable max-len */
                  const options = commonOptions.getCommonOptions(DEFAULT_LOG_FILE)
                      .option(
                          '--cloud <cloud_provider>',
                          'Cloud provider (aws | azure | etc.)'
                      )
                      .option(
                          '--provider-options <cloud_options>',
                          'Options specific to cloud_provider. Ex: param1:value1,param2:value2',
                          util.map,
                          providerOptions
                      )
                      .option(
                          '-c, --cluster-action <type>',
                          'join (join a cluster) | update (update cluster to match existing instances | unblock-sync (allow other devices to sync to us) | backup-ucs (save a ucs to cloud storage)'
                      )
                      .option(
                          '--device-group <device_group>',
                          'Device group name.'
                      )
                      .option(
                          '    --full-load-on-sync',
                          '    Enable full load on sync. Default false.'
                      )
                      .option(
                          '    --asm-sync',
                          '    Enable ASM sync. Default sets ASM sync if ASM is provisioned.'
                      )
                      .option(
                          '    --network-failover',
                          '    Enable network failover. Default false.'
                      )
                      .option(
                          '    --no-auto-sync',
                          '    Enable auto sync. Default false (auto sync).'
                      )
                      .option(
                          '    --no-save-on-auto-sync',
                          '    Enable save on sync if auto sync is enabled. Default false (save on auto sync).'
                      )
                      .option(
                          '--block-sync',
                          'If this device is primary, do not allow other devices to sync to us. This prevents other devices from syncing to it until we are called again with --cluster-action unblock-sync.'
                      )
                      .option(
                          '--static',
                          'Indicates that this instance is not autoscaled. Default false (instance is autoscaled)'
                      )
                      .option(
                          '--external-tag <tag>',
                          'If there are instances in the autoscale cluster that are not autoscaled, the cloud tag applied to those instances. Format \'key:<tag_key>,value:<tag_value>\'', util.map, externalTag
                      )
                      .option(
                          '--license-pool',
                          'BIG-IP was licensed from a BIG-IQ license pool. This is so licenses can be revoked when BIG-IPs are scaled in. Supply the following:'
                      )
                      .option(
                          '    --big-iq-host <ip_address or FQDN>',
                          '    IP address or FQDN of BIG-IQ'
                      )
                      .option(
                          '    --big-iq-user <user>',
                          '    BIG-IQ admin user name'
                      )
                      .option(
                          '    --big-iq-password [password]',
                          '    BIG-IQ admin user password.'
                      )
                      .option(
                          '    --big-iq-password-uri [password_uri]',
                          '    URI (file, http(s), arn) to location that contains BIG-IQ admin user password. Use this or --big-iq-password.'
                      )
                      .option(
                          '    --big-iq-password-encrypted',
                          '    Indicates that the BIG-IQ password is encrypted.'
                      )
                      .option(
                          '    --license-pool-name <pool_name>',
                          '    Name of BIG-IQ license pool.'
                      )
                      .option(
                          '    --big-ip-mgmt-address <big_ip_address>',
                          '    IP address or FQDN of BIG-IP management port. Use this if BIG-IP reports an address not reachable from BIG-IQ.'
                      )
                      .option(
                          '    --big-ip-mgmt-port <big_ip_port>',
                          '    Port for the management address. Use this if the BIG-IP is not reachable from BIG-IQ via the port used in --port'
                      )
                      .option(
                          '    --no-unreachable',
                          '    Do not use the unreachable API even if it is supported by BIG-IQ.'
                      )
                      .option(
                          '--dns <dns_provider>',
                          '    Update the specified DNS provider when autoscaling occurs (gtm is the only current provider)'
                      )
                      .option(
                          '    --dns-ip-type <address_type>',
                          '    Type of ip address to use (public | private).'
                      )
                      .option(
                          '    --dns-app-port <port>',
                          '    Port on which application is listening on for health check'
                      )
                      .option(
                          '    --dns-provider-options <dns_provider_options>',
                          '    Options specific to dns_provider. Ex: param1:value1,param2:value2',
                          util.map,
                          dnsProviderOptions
                      )
                      .option(
                          '--max-ucs-files <max_ucs_files_to_save>',
                          'When running cluster action backup-ucs, maximum number of backup files to keep.',
                          UCS_BACKUP_DEFAULT_MAX_FILES
                      )
                      .option(
                          '--autoscale-timeout <autoscale_timeout>',
                          'Number of minutes after which autoscale process should be killed',
                          DEFAULT_AUTOSCALE_TIMEOUT_IN_MINUTES
                      )
                      .option(
                          '--primary-disconnected-time <primary_disconnected_time>',
                          'Time (in milliseconds) after which primary host is considered to be expired',
                          DEFAULT_MAX_DISCONNECTED_MS
                      )
                      .parse(argv);
                  /* eslint-enable max-len */
  
                  loggerOptions.console = options.console;
                  loggerOptions.logLevel = options.logLevel;
                  loggerOptions.module = module;
  
                  if (options.output) {
                      loggerOptions.fileName = options.output;
                  }
  
                  logger = Logger.getLogger(loggerOptions);
                  util.setLoggerOptions(loggerOptions);
                  cryptoUtil.setLoggerOptions(loggerOptions);
  
                  // Remove specific options with no provided value
                  OPTIONS_TO_UNDEFINE.forEach((opt) => {
                      if (typeof options[opt] === 'boolean') {
                          logger.debug(`No value set for option ${opt}. Removing option.`);
                          options[opt] = undefined;
                      }
                  });
  
                  // Expose options for test code
                  this.options = options;
  
                  if (!options.password && !options.passwordUrl) {
                      util.logAndExit('One of --password or --password-url is required.', 'error', 1);
                  }
  
                  // When running in cloud init, we need to exit so that cloud init can complete and
                  // allow the BIG-IP services to start
                  if (options.background) {
                      logFileName = options.output || DEFAULT_LOG_FILE;
                      logger.info('Spawning child process to do the work. Output will be in', logFileName);
                      util.runInBackgroundAndExit(process, logFileName);
                  }
  
                  // Log the input, but don't log passwords
                  loggableArgs = argv.slice();
                  for (let i = 0; i < loggableArgs.length; i++) {
                      if (KEYS_TO_MASK.indexOf(loggableArgs[i]) !== -1) {
                          loggableArgs[i + 1] = '*******';
                      }
                  }
                  logger.info(`${loggableArgs[1]} called with`, loggableArgs.join(' '));
  
                  // Get the concrete autoscale provider instance
                  cloudProvider = optionsForTest.cloudProvider;
                  if (!cloudProvider) {
                      cloudProvider = cloudProviderFactory.getCloudProvider(
                          options.cloud,
                          {
                              loggerOptions,
                              clOptions: options
                          }
                      );
                  }
  
                  // If updating DNS, get the concrete DNS provider instance
                  if (options.dns) {
                      dnsProvider = dnsProviderFactory.getDnsProvider(
                          options.dns,
                          {
                              loggerOptions,
                              clOptions: options
                          }
                      );
                  }
  
                  // Save args in restart script in case we need to reboot to recover from an error
                  util.saveArgs(argv, ARGS_FILE_ID)
                      .then(() => {
                          if (options.waitFor) {
                              logger.info('Waiting for', options.waitFor);
                              return ipc.once(options.waitFor);
                          }
                          return q();
                      })
                      .then(() => {
                          // Whatever we're waiting for is done, so don't wait for
                          // that again in case of a reboot
                          return util.saveArgs(argv, ARGS_FILE_ID, ['--wait-for']);
                      })
                      .then(() => {
                          if (options.clusterAction === 'join' || options.clusterAction === 'update') {
                              return getAutoscaleProcessInfo();
                          }
                          return q();
                      })
                      .then((results) => {
                          // Stop processing if there is an other running Autoscale process
                          // with cluster action of join or update
                          if (results && results.processCount && results.processCount > 1) {
                              logger.silly(`Running process count: ${results.processCount}`);
                              logger.silly(`Execution time in mins: ${parseInt(results.executionTime, 10)}`);
                              if (results.executionTime
                                  && parseInt(results.executionTime, 10) < options.autoscaleTimeout) {
                                  util.logAndExit('Another autoscale process already running. ' +
                                      'Exiting.', 'warn', 1);
                                  return q.reject('Another autoscale process already running.');
                              }
                              logger.info('Terminating the autoscale script execution.');
                              util.terminateProcessById(results.pid);
                              util.logAndExit('Long running autoscale processed terminated. Exiting...');
                              return q.reject('Long running autoscale processed terminated. Exiting...');
                          }
                          return q();
                      })
                      .then(() => {
                          logger.info('Initializing autoscale provider');
                          return cloudProvider.init(providerOptions, { autoscale: true });
                      })
                      .then(() => {
                          if (options.dns) {
                              logger.info('Initializing DNS provider');
                              return dnsProvider.init(dnsProviderOptions);
                          }
                          return q();
                      })
                      .then(() => {
                          logger.info('Getting this instance ID.');
                          return cloudProvider.getInstanceId();
                      })
                      .then((response) => {
                          logger.debug('This instance ID:', response);
                          this.instanceId = response;
  
                          logger.info('Getting info on all instances.');
                          if (Object.keys(externalTag).length === 0) {
                              externalTag = undefined;
                          }
                          return cloudProvider.getInstances({
                              externalTag,
                              instanceId: response
                          });
                      })
                      .then((response) => {
                          this.instances = response || {};
                          logger.debug('instances:', this.instances);
  
                          if (Object.keys(this.instances).length === 0) {
                              util.logAndExit('Instance list is empty. Exiting.', 'error', 1);
                              return q.reject('Instance list is empty. Exiting.');
                          }
  
                          this.instance = this.instances[this.instanceId];
                          if (!this.instance) {
                              util.logAndExit('Our instance ID is not in instance list. Exiting', 'error', 1);
                              return q.reject('Our instance ID is not in instance list. Exiting');
                          }
  
                          this.instance.status = this.instance.status || AutoscaleInstance.INSTANCE_STATUS_OK;
                          logger.silly('Instance status:', this.instance.status);
  
                          if (this.instance.status === AutoscaleInstance.INSTANCE_STATUS_BECOMING_PRIMARY
                              && !isPrimaryExpired(this.instance, options)) {
                              util.logAndExit('Currently becoming primary. Exiting.', 'info');
                              return q.reject('Currently becoming primary. Exiting.');
                          }
  
                          if (optionsForTest.bigIp) {
                              bigIp = optionsForTest.bigIp;
                              return q();
                          }
                          bigIp = new BigIp({ loggerOptions });
  
                          logger.info('Initializing BIG-IP.');
                          return bigIp.init(
                              options.host,
                              options.user,
                              options.password || options.passwordUrl,
                              {
                                  port: options.port,
                                  passwordIsUrl: typeof options.passwordUrl !== 'undefined',
                                  passwordEncrypted: options.passwordEncrypted
                              }
                          );
                      })
                      .then(() => {
                          return bigIp.list('/tm/sys/global-settings');
                      })
                      .then((globalSettings) => {
                          this.instance.hostname = globalSettings.hostname;
                          return cloudProvider.putInstance(this.instanceId, this.instance);
                      })
                      .then(() => {
                          return cloudProvider.bigIpReady();
                      })
                      .then(() => {
                          return bigIp.deviceInfo();
                      })
                      .then((response) => {
                          this.instance.machineId = response.machineId; // we need this for revoke on BIG-IQ 5.3
                          this.instance.macAddress = response.hostMac; // we need this for revoke on BIG-IQ 5.4
                          this.instance.version = response.version;
                          markVersions(this.instances);
                          return cloudProvider.putInstance(this.instanceId, this.instance);
                      })
                      .then(() => {
                          let status = CloudProvider.STATUS_UNKNOWN;
  
                          logger.info('Determining primary instance id.');
                          primaryInstance = getPrimaryInstance(this.instances);
  
                          if (primaryInstance) {
                              if (!primaryInstance.instance.versionOk) {
                                  primaryBadReason = 'version not most recent in group';
                                  logger.silly(primaryBadReason);
                                  status = CloudProvider.STATUS_VERSION_NOT_UP_TO_DATE;
                                  primaryBad = true;
                              } else if (!isPrimaryExternalValueOk(primaryInstance.id, this.instances)) {
                                  // if there are external instances in the mix, make sure the primary
                                  // is one of them
                                  primaryBadReason = 'primary is not external, ' +
                                                     'but there are external instances';
                                  logger.silly(primaryBadReason);
                                  status = CloudProvider.STATUS_NOT_EXTERNAL;
                                  primaryBad = true;
                              } else if (!primaryInstance.instance.providerVisible) {
                                  // The cloud provider does not currently see this instance
                                  status = CloudProvider.STATUS_NOT_IN_CLOUD_LIST;
                              } else {
                                  primaryIid = primaryInstance.id;
  
                                  if (this.instanceId === primaryIid) {
                                      this.instance.isPrimary = true;
                                  }
  
                                  status = CloudProvider.STATUS_OK;
                              }
                          }
  
                          return updatePrimaryStatus.call(this, cloudProvider, status);
                      })
                      .then(() => {
                          // If the primary is not visible, check to see if it's been gone
                          // for a while or if this is a random error
                          if (primaryInstance && !primaryBad && isPrimaryExpired(this.instance, options)) {
                              primaryBad = true;
                              primaryBadReason = 'primary is expired';
                          }
  
                          if (primaryIid) {
                              logger.info('Possible primary ID:', primaryIid);
                              return cloudProvider.isValidPrimary(primaryIid, this.instances);
                          } else if (primaryBad) {
                              logger.info('Old primary no longer valid:', primaryBadReason);
                              return q();
                          }
  
                          logger.info('No primary ID found.');
                          return q();
                      })
                      .then((validPrimary) => {
                          logger.silly(
                              'validPrimary:',
                              validPrimary,
                              ', primaryInstance: ',
                              primaryInstance,
                              ', primaryBad:',
                              primaryBad
                          );
  
                          if (validPrimary) {
                              // true validPrimary means we have a valid primaryIid, just pass it on
                              logger.info('Valid primary ID:', primaryIid);
                              return primaryIid;
                          }
  
                          // false or undefined validPrimary means no primaryIid or invalid primaryIid
                          if (validPrimary === false) {
                              logger.info('Invalid primary ID:', primaryIid);
                              cloudProvider.primaryInvalidated(primaryIid);
                          }
  
                          // if no primary, primary is visible or expired, elect, otherwise, wait
                          if (!primaryInstance ||
                              primaryInstance.instance.providerVisible ||
                              primaryBad) {
                              logger.info('Electing primary.');
                              return cloudProvider.electPrimary(this.instances);
                          }
                          return q();
                      })
                      .then((response) => {
                          const now = new Date();
  
                          if (response) {
                              // we just elected a primary
                              primaryIid = response;
                              this.instance.isPrimary = (this.instanceId === primaryIid);
                              logger.info('Using primary ID:', primaryIid);
                              logger.info(
                                  'This instance',
                                  (this.instance.isPrimary ? 'is' : 'is not'),
                                  'primary'
                              );
  
                              if (this.instance.primaryStatus.instanceId !== primaryIid) {
                                  logger.info('New primary elected');
                                  newPrimary = true;
  
                                  this.instance.primaryStatus = {
                                      instanceId: primaryIid,
                                      status: CloudProvider.STATUS_OK,
                                      lastUpdate: now,
                                      lastStatusChange: now
                                  };
  
                                  return cloudProvider.putInstance(this.instanceId, this.instance);
                              }
                          }
                          return q();
                      })
                      .then(() => {
                          if (this.instance.isPrimary && newPrimary) {
                              this.instance.status = AutoscaleInstance.INSTANCE_STATUS_BECOMING_PRIMARY;
                              return cloudProvider.putInstance(this.instanceId, this.instance);
                          }
                          return q();
                      })
                      .then(() => {
                          if (this.instance.isPrimary && newPrimary) {
                              return becomePrimary.call(this, cloudProvider, bigIp, options);
                          }
                          return q();
                      })
                      .then((response) => {
                          if (
                              this.instance.status === AutoscaleInstance.INSTANCE_STATUS_BECOMING_PRIMARY &&
                              response === true
                          ) {
                              this.instance.status = AutoscaleInstance.INSTANCE_STATUS_OK;
                              logger.silly('Became primary');
                              return cloudProvider.putInstance(this.instanceId, this.instance);
                          } else if (response === false) {
                              logger.warn('Error writing primary file');
                          }
                          return q();
                      })
                      .then(() => {
                          if (primaryIid && this.instance.status === AutoscaleInstance.INSTANCE_STATUS_OK) {
                              return cloudProvider.primaryElected(primaryIid);
                          }
                          return q();
                      })
                      .then(() => {
                          if (primaryIid && this.instance.status === AutoscaleInstance.INSTANCE_STATUS_OK) {
                              return cloudProvider.tagPrimaryInstance(primaryIid, this.instances);
                          }
                          return q();
                      })
                      .then(() => {
                          let message;
                          if (this.instance.status === AutoscaleInstance.INSTANCE_STATUS_OK) {
                              switch (options.clusterAction) {
                              case 'join':
                                  logger.info('Cluster action join');
                                  return handleJoin.call(
                                      this,
                                      cloudProvider,
                                      bigIp,
                                      primaryIid,
                                      options
                                  );
                              case 'update':
                                  logger.info('Cluster action update');
                                  return bigIp.deviceState(this.instance.hostname)
                                      .then((response) => {
                                          if (response && response.configsyncIp !== this.instance.privateIp) {
                                              return bigIp.cluster.configSyncIp(this.instance.privateIp);
                                          }
                                          return q();
                                      })
                                      .then(() => {
                                          return handleUpdate.call(
                                              this,
                                              cloudProvider,
                                              bigIp,
                                              primaryIid,
                                              primaryBad || newPrimary,
                                              options
                                          );
                                      });
                              case 'unblock-sync':
                                  logger.info('Cluster action unblock-sync');
                                  return bigIp.cluster.configSyncIp(this.instance.privateIp);
                              case 'backup-ucs': {
                                  logger.info('Cluster action backup-ucs');
                                  const clusterMetadata = {
                                      instanceId: this.instanceId,
                                      instance: this.instance,
                                      instances: this.instances
                                  };
                                  return handleBackupUcs.call(
                                      this,
                                      cloudProvider,
                                      clusterMetadata,
                                      bigIp,
                                      options
                                  );
                              }
                              default:
                                  message = `Unknown cluster action ${options.clusterAction}`;
                                  logger.warn(message);
                                  return q.reject(message);
                              }
                          } else {
                              logger.debug('Instance status not OK. Waiting.', this.instance.status);
                              return q();
                          }
                      })
                      .then(() => {
                          if (this.instance.status === AutoscaleInstance.INSTANCE_STATUS_OK) {
                              if (cloudProvider.hasFeature(CloudProvider.FEATURE_MESSAGING)
                                  && (options.clusterAction === 'join' || options.clusterAction === 'update')) {
                                  logger.info('Checking for messages');
                                  return handleMessages.call(this, cloudProvider, bigIp, options);
                              }
                          } else {
                              logger.debug('Instance status not OK. Waiting.', this.instance.status);
                          }
                          return q();
                      })
                      .then(() => {
                          if (options.dns
                              && (options.clusterAction === 'join' || options.clusterAction === 'update')) {
                              logger.info('Updating DNS');
  
                              const instancesForDns = [];
  
                              Object.keys(this.instances).forEach((instanceId) => {
                                  const instance = this.instances[instanceId];
                                  const ip =
                                      (options.dnsIpType === 'public' ? instance.publicIp : instance.privateIp);
  
                                  if (instance.hostname) {
                                      instancesForDns.push(
                                          {
                                              ip,
                                              name: instance.hostname,
                                              port: options.dnsAppPort
                                          }
                                      );
                                  }
                              });
                              return dnsProvider.update(instancesForDns);
                          }
                          return q();
                      })
                      .catch((err) => {
                          if (err && err.code && err.message) {
                              logger.error('autoscaling error code:', err.code, 'message:', err.message);
                          } else {
                              logger.error('autoscaling error:', err && err.message ? err.message : err);
                          }
                          return err;
                      })
                      .done((err) => {
                          util.deleteArgs(ARGS_FILE_ID);
  
                          if (cb) {
                              cb(err);
                          }
  
                          // Exit so that any listeners don't keep us alive
                          util.logAndExit('Autoscale finished.');
                      });
  
                  // If we reboot, exit - otherwise cloud providers won't know we're done
                  ipc.once('REBOOT')
                      .then(() => {
                          util.logAndExit('REBOOT signaled. Exiting.');
                      });
              } catch (err) {
                  if (logger) {
                      logger.error('autoscale error:', err);
                  }
  
                  if (cb) {
                      cb();
                  }
              }
          }
      };
  
      /**
       * Handles --cluster-action join
       *
       * Called with this bound to the caller
       */
      function handleJoin(provider, bigIp, primaryIid, options) {
          const deferred = q.defer();
  
          logger.info('Cluster action JOIN');
  
          logger.info('Initializing encryption');
          initEncryption.call(this, provider, bigIp)
              .then(() => {
                  let promise;
  
                  // If we are primary and are replacing an expired primary, other instances
                  // will join to us. Just set our config sync ip.
                  if (this.instance.isPrimary) {
                      if (!provider.hasFeature(CloudProvider.FEATURE_MESSAGING)) {
                          logger.info('Storing primary credentials.');
                          promise = provider.putPrimaryCredentials();
                      } else {
                          promise = q();
                      }
  
                      promise
                          .then((response) => {
                              logger.debug(response);
  
                              // Configure cm configsync-ip on this BIG-IP node
                              if (!options.blockSync) {
                                  logger.info('Setting config sync IP.');
                                  return bigIp.cluster.configSyncIp(this.instance.privateIp);
                              }
                              logger.info('Not seting config sync IP because block-sync is specified.');
                              return q();
                          })
                          .then(() => {
                              deferred.resolve();
                          })
                          .catch((err) => {
                              // rethrow here, otherwise error is hidden
                              throw err;
                          });
                  } else {
                      // We're not the primary
  
                      // Make sure the primary file is not on our disk
                      if (fs.existsSync(PRIMARY_FILE_PATH)) {
                          fs.unlinkSync(PRIMARY_FILE_PATH);
                      }
  
                      // Configure cm configsync-ip on this BIG-IP node and join the cluster
                      logger.info('Setting config sync IP.');
                      bigIp.cluster.configSyncIp(this.instance.privateIp)
                          .then(() => {
                              // If there is a primary, join it. Otherwise wait for an update event
                              // when we have a primary.
                              if (primaryIid) {
                                  return joinCluster.call(this, provider, bigIp, primaryIid, options);
                              }
                              return q();
                          })
                          .then(() => {
                              deferred.resolve();
                          })
                          .catch((err) => {
                              // rethrow here, otherwise error is hidden
                              throw err;
                          });
                  }
              });
  
          return deferred.promise;
      }
  
      /**
       * Handles --cluster-action update
       *
       * Called with this bound to the caller
       */
      function handleUpdate(provider, bigIp, primaryIid, primaryBadOrNew, options) {
          logger.info('Cluster action UPDATE');
  
          if (this.instance.isPrimary && !primaryBadOrNew) {
              return checkClusteredDevices.call(this, provider, bigIp);
          } else if (!this.instance.isPrimary) {
              // We're not the primary, make sure the primary file is not on our disk
              if (fs.existsSync(PRIMARY_FILE_PATH)) {
                  fs.unlinkSync(PRIMARY_FILE_PATH);
              }
  
              // If there is a new primary, join the cluster
              if (primaryBadOrNew && primaryIid) {
                  return joinCluster.call(this, provider, bigIp, primaryIid, options);
              } else if (primaryIid) {
                  // Double check that we are clustered
                  return bigIp.list('/tm/cm/trust-domain/Root')
                      .then((response) => {
                          if (!response || response.status === 'standalone') {
                              logger.info('This instance is not in cluster. Requesting join.');
                              return joinCluster.call(this, provider, bigIp, primaryIid, options);
                          }
  
                          return q();
                      })
                      .catch((err) => {
                          throw err;
                      });
              }
          }
          return q();
      }
  
      /**
       * Called with this bound to the caller
       */
      function handleMessages(provider, bigIp, options) {
          const deferred = q.defer();
          const instanceIdsBeingAdded = [];
          const actions = [];
          const actionPromises = [];
  
          let messageMetadata = [];
  
          if (this.instance.isPrimary && !options.blockSync) {
              actions.push(CloudProvider.MESSAGE_ADD_TO_CLUSTER);
          }
  
          if (!this.instance.isPrimary) {
              actions.push(CloudProvider.MESSAGE_SYNC_COMPLETE);
          }
  
          provider.getMessages(actions, { toInstanceId: this.instanceId })
              .then((messages) => {
                  const readPromises = [];
                  const messagesArray = messages ? messages.slice() : [];
  
                  logger.debug('Handling', messages.length, 'message(s)');
  
                  messagesArray.forEach((message) => {
                      messageMetadata.push(
                          {
                              action: message.action,
                              toInstanceId: message.toInstanceId,
                              fromInstanceId: message.fromInstanceId
                          }
                      );
                      readPromises.push(readMessageData.call(this, provider, bigIp, message.data));
                  });
  
                  logger.silly('number of messages to read:', readPromises.length);
  
                  return q.all(readPromises);
              })
              .then((readMessages) => {
                  let metadata;
                  let messageData;
  
                  const alreadyAdding = function (instanceId) {
                      return instanceIdsBeingAdded.find((element) => {
                          return instanceId === element.toInstanceId;
                      });
                  };
  
                  const readMessagesArray = readMessages ? readMessages.slice() : [];
  
                  logger.silly('number of read messages:', readMessagesArray.length);
  
                  for (let i = 0; i < readMessagesArray.length; ++i) {
                      metadata = messageMetadata[i];
                      logger.silly('metadata:', metadata);
  
                      try {
                          messageData = JSON.parse(readMessagesArray[i]);
                      } catch (err) {
                          logger.warn('JSON.parse error:', err);
                          messageData = undefined;
                          deferred.reject(new Error('Unable to JSON parse message'));
                      }
  
                      if (messageData) {
                          let discard = false;
                          switch (metadata.action) {
                          // Add an instance to our cluster
                          case CloudProvider.MESSAGE_ADD_TO_CLUSTER:
                              logger.silly('message MESSAGE_ADD_TO_CLUSTER');
  
                              if (alreadyAdding(metadata.fromInstanceId)) {
                                  logger.debug('Already adding', metadata.fromInstanceId, ', discarding');
                                  discard = true;
                              }
  
                              if (!discard) {
                                  instanceIdsBeingAdded.push({
                                      toInstanceId: metadata.fromInstanceId,
                                      fromUser: bigIp.user,
                                      fromPassword: bigIp.password
                                  });
  
                                  actionPromises.push(
                                      bigIp.cluster.joinCluster(
                                          messageData.deviceGroup,
                                          messageData.host,
                                          messageData.username,
                                          messageData.password,
                                          true,
                                          {
                                              remotePort: messageData.port,
                                              remoteHostname: messageData.hostname,
                                              passwordEncrypted: false
                                          }
                                      )
                                  );
                              }
  
                              break;
  
                          // sync is complete
                          case CloudProvider.MESSAGE_SYNC_COMPLETE:
                              logger.silly('message MESSAGE_SYNC_COMPLETE');
                              actionPromises.push(
                                  provider.syncComplete(messageData.fromUser, messageData.fromPassword)
                              );
  
                              break;
                          default:
                              logger.warn('Unknown message action', metadata.action);
                          }
                      }
                  }
  
                  return q.all(actionPromises);
              })
              .then((responses) => {
                  const messagePromises = [];
                  let messageData;
  
                  const responsesArray = responses ? responses.slice() : [];
  
                  if (instanceIdsBeingAdded.length > 0) {
                      messageMetadata = [];
  
                      logger.silly('responses from join cluster', responsesArray);
                      for (let i = 0; i < responsesArray.length; i++) {
                          // responsesArray[i] === true iff that instance was successfully synced
                          if (responsesArray[i] === true) {
                              logger.silly(
                                  'sync is complete for instance',
                                  instanceIdsBeingAdded[i].toInstanceId
                              );
  
                              messageMetadata.push(
                                  {
                                      action: CloudProvider.MESSAGE_SYNC_COMPLETE,
                                      toInstanceId: instanceIdsBeingAdded[i].toInstanceId,
                                      fromInstanceId: this.instanceId
                                  }
                              );
                              messageData = {
                                  fromUser: instanceIdsBeingAdded[i].fromUser,
                                  fromPassword: instanceIdsBeingAdded[i].fromPassword
                              };
  
                              messagePromises.push(
                                  prepareMessageData.call(
                                      this,
                                      provider,
                                      instanceIdsBeingAdded[i].toInstanceId,
                                      JSON.stringify(messageData)
                                  )
                              );
                          }
                      }
                  }
  
                  return q.all(messagePromises);
              })
              .then((preppedMessageData) => {
                  const syncCompletePromises = [];
                  let metadata;
                  let messageData;
  
                  for (let i = 0; i < preppedMessageData.length; i++) {
                      metadata = messageMetadata[i];
                      messageData = preppedMessageData[i];
  
                      syncCompletePromises.push(
                          provider.sendMessage(
                              CloudProvider.MESSAGE_SYNC_COMPLETE,
                              {
                                  toInstanceId: metadata.toInstanceId,
                                  fromInstanceId: metadata.fromInstanceId,
                                  data: messageData
                              }
                          )
                      );
                  }
  
                  return q.all(syncCompletePromises);
              })
              .then(() => {
                  deferred.resolve();
              })
              .catch((err) => {
                  logger.warn('Error handling messages', err);
                  deferred.reject(err);
              });
  
          return deferred.promise;
      }
  
      function validateUploadedUcs(provider, ucsFileName) {
          const ucsFilePath = `${BACKUP.UCS_LOCAL_TMP_DIRECTORY}/${ucsFileName}`;
          return provider.getStoredUcs()
              .then((ucsData) => {
                  logger.silly(`ucsFilePath: ${ucsFilePath}`);
                  return util.writeUcsFile(ucsFilePath, ucsData);
              })
              .then(() => {
                  return util.runShellCommand(`gzip -t -v ${ucsFilePath}`);
              })
              .then((response) => {
                  if (response.indexOf('NOT OK') !== -1) {
                      return q.resolve({
                          status: 'CORRUPTED',
                          filePath: ucsFilePath
                      });
                  }
                  logger.silly('Validated integrity of recenetly generated UCS file.');
                  return q.resolve({
                      status: 'OK',
                      filePath: ucsFilePath
                  });
              })
              .catch((err) => {
                  logger.warn('Error while validating ucs', err);
                  return q.reject(err);
              });
      }
  
      function handleBackupUcs(provider, clusterMetadata, bigIp, options) {
          if (!this.instance.isPrimary
              || this.instance.status !== AutoscaleInstance.INSTANCE_STATUS_OK) {
              logger.debug('not primary or not ready, skipping ucs backup');
              return q();
          }
  
          const now = new Date().getTime();
          const ucsName = `${UCS_BACKUP_PREFIX}${now}`;
  
          logger.info('Backing up UCS');
          // ajv, which is installed as a dependency of the Azure node SDK has a couple files that start
          // with '$'. prior to 13.1, Meanwhile, mcpd has a bug which fails to save a ucs if it runs
          // into a file that starts with a '$'. So, let's just move the file. It's a bit ugly, but
          // there's not really a better place to do this since it's a one-off bug. All of
          // f5-cloud-libs is removed before ucs is loaded, so we don't need to do anything on that end.
          this.instance.lastBackup = now;
          let isUcsFileValid = false;
          return cleanupAjv(bigIp)
              .then(() => {
                  return bigIp.saveUcs(ucsName);
              })
              .then(() => {
                  return provider.storeUcs(
                      `${UCS_BACKUP_DIRECTORY}/${ucsName}.ucs`,
                      options.maxUcsFiles,
                      UCS_BACKUP_PREFIX
                  );
              })
              .then(() => {
                  logger.silly(`lastest ucs file: ${ucsName}.ucs`);
                  return removeOldUcsFiles(`${ucsName}.ucs`);
              })
              .then(() => {
                  return validateUploadedUcs(provider, `${ucsName}.ucs`);
              })
              .then((results) => {
                  fs.unlinkSync(results.filePath);
                  logger.silly('Removed local UCS file used in validation.');
                  if (results.status !== 'OK') {
                      provider.deleteStoredUcs(`${ucsName}.ucs`);
                      return q.resolve();
                  }
                  isUcsFileValid = true;
                  return q.resolve();
              })
              .then(() => {
                  if (!isUcsFileValid) {
                      return q.reject(new Error('Validation of ' +
                          'generated UCS file failed; ' +
                          'recently generated UCS file appears to be corrupted.'));
                  }
                  logger.debug(`Update instance metadata; lastBackUp to ${this.instance.lastBackup}`);
                  return provider.putInstance(clusterMetadata.instanceId, this.instance);
              })
              .then(() => {
                  return q.resolve();
              })
              .catch((err) => {
                  logger.info('Error backing up ucs', err);
                  if (fs.existsSync(`${BACKUP.UCS_LOCAL_TMP_DIRECTORY}/${ucsName}.ucs`)) {
                      fs.unlinkSync(`${BACKUP.UCS_LOCAL_TMP_DIRECTORY}/${ucsName}.ucs`);
                  }
                  provider.deleteStoredUcs(`${ucsName}.ucs`);
                  return q.reject(err);
              });
      }
  
      /**
       * Handles becoming primary.
       *
       * @returns {Promise} promise which is resolved tieh true if successful
       */
      function becomePrimary(provider, bigIp, options) {
          let hasUcs = false;
          const promises = [];
          logger.info('Becoming primary.');
          logger.info('Checking if need to restore UCS.');
          /*
               - By default, lastBackup time is set to the begining of epoch (i.e. 2678400000)
               - When backup created, lastBackup value will be set to time of backup
               - lastBackup will be updated on each in-sync host to match primary lastBackup time
               - lastBackup will be used to confirm if a host was in-sync with previous primary to
                 check if ucs restore is needed to copy over custom configs
           */
          if (this.instance.lastBackup === new Date(1970, 1, 1).getTime()) {
              logger.silly('will attempt to restore ucs; ' +
                  'this instance never was in synced with previous primary');
              promises.push(provider.getStoredUcs());
          } else {
              logger.silly('no need to restore ucs; this instance was in sync with previous primary');
          }
          return Promise.all(promises)
              .then((response) => {
                  if (response && response.length === 1 && response[0]) {
                      hasUcs = true;
                      return loadUcs(provider, bigIp, response[0], options.cloud);
                  }
                  return q();
              })
              .then(() => {
                  logger.silly('setting lastBackup to current time since this instnace is primary now.');
                  // this is done to prefer running config
                  this.instance.lastBackup = new Date().getTime();
                  // If we loaded UCS, re-initialize encryption so our keys
                  // match each other and update lastBackup
                  if (hasUcs) {
                      return initEncryption.call(this, provider, bigIp);
                  }
                  return q();
              })
              .then(() => {
                  return bigIp.list('/tm/sys/global-settings');
              })
              .then((globalSettings) => {
                  const hostname = globalSettings ? globalSettings.hostname : undefined;
  
                  if (hostname) {
                      this.instance.hostname = hostname;
                  } else {
                      logger.debug('hostname not found in this.instance or globalSettings');
                  }
  
                  return bigIp.list('/tm/sys/provision');
              })
              .then((response) => {
                  const modulesProvisioned = {};
  
                  if (response && response.length > 0) {
                      response.forEach((module) => {
                          modulesProvisioned[module.name] = !(module.level === 'none');
                      });
                  }
  
                  // Make sure device group exists
                  logger.info('Creating device group.');
  
                  const deviceGroupOptions = {
                      autoSync: options.autoSync,
                      saveOnAutoSync: options.saveOnAutoSync,
                      fullLoadOnSync: options.fullLoadOnSync,
                      asmSync: modulesProvisioned.asm || options.asmSync,
                      networkFailover: options.networkFailover
                  };
  
                  return bigIp.cluster.createDeviceGroup(
                      options.deviceGroup,
                      'sync-failover',
                      [this.instance.hostname],
                      deviceGroupOptions
                  );
              })
              .then(() => {
                  logger.info('Writing primary file.');
                  return writePrimaryFile(hasUcs);
              });
      }
  
      /**
       * Called with this bound to the caller
       */
      function joinCluster(provider, bigIp, primaryIid, options) {
          const TEMP_USER_NAME_LENGHTH = 10; // these are hex bytes - user name will be 20 chars
          const TEMP_USER_PASSWORD_LENGTH = 24; // use a multiple of 6 to prevent '=' at the end
  
          const now = new Date();
  
          let managementIp;
          let tempPassword;
          let tempUser;
  
          if (!primaryIid) {
              return q.reject(new Error('Must have a primary ID to join'));
          }
  
          // don't send request too often - primary might be in process of syncing
          this.instance.lastJoinRequest = this.instance.lastJoinRequest || new Date(1970, 0);
          const elapsedMsFromLastJoin = now - new Date(this.instance.lastJoinRequest);
          if (elapsedMsFromLastJoin < MIN_MS_BETWEEN_JOIN_REQUESTS) {
              logger.silly('Join request is too soon after last join request.', elapsedMsFromLastJoin, 'ms');
              return q();
          }
  
          logger.info('Joining cluster.');
  
          if (provider.hasFeature(CloudProvider.FEATURE_MESSAGING)) {
              this.instance.lastJoinRequest = now;
              return provider.putInstance(this.instanceId, this.instance)
                  .then((response) => {
                      logger.debug(response);
                      logger.debug('Resetting current device trust');
                      return bigIp.cluster.resetTrust();
                  })
                  .then((response) => {
                      logger.debug(response);
  
                      // Make sure we don't have a current copy of the device group
                      return bigIp.cluster.deleteDeviceGroup(options.deviceGroup);
                  })
                  .then((response) => {
                      logger.debug(response);
                      return bigIp.deviceInfo();
                  })
                  .then((response) => {
                      managementIp = response.managementAddress;
  
                      // Get a random user name to use
                      return cryptoUtil.generateRandomBytes(TEMP_USER_NAME_LENGHTH, 'hex');
                  })
                  .then((respomse) => {
                      tempUser = respomse;
  
                      // Get a random password for the user
                      return cryptoUtil.generateRandomBytes(TEMP_USER_PASSWORD_LENGTH, 'base64');
                  })
                  .then((response) => {
                      tempPassword = response;
  
                      // Create the temp user account
                      return bigIp.onboard.updateUser(tempUser, tempPassword, 'admin');
                  })
                  .then(() => {
                      logger.debug('Sending message to join cluster.');
  
                      const messageData = {
                          host: managementIp,
                          port: bigIp.port,
                          username: tempUser,
                          password: tempPassword,
                          hostname: this.instance.hostname,
                          deviceGroup: options.deviceGroup
                      };
  
                      return prepareMessageData.call(this, provider, primaryIid, JSON.stringify(messageData));
                  })
                  .then((preppedData) => {
                      if (preppedData) {
                          return provider.sendMessage(
                              CloudProvider.MESSAGE_ADD_TO_CLUSTER,
                              {
                                  toInstanceId: primaryIid,
                                  fromInstanceId: this.instanceId,
                                  data: preppedData
                              }
                          );
                      }
                      logger.debug('No encrypted data received');
                      return q();
                  })
                  .catch((err) => {
                      // need to bubble up nested errors
                      return q.reject(err);
                  });
          }
  
          // not using messaging, just send the request via iControl REST
          const primaryInstance = this.instances[primaryIid];
  
          this.instance.lastJoinRequest = now;
          return provider.putInstance(this.instanceId, this.instance)
              .then((response) => {
                  logger.debug(response);
                  logger.debug('Resetting current device trust');
                  return bigIp.cluster.resetTrust();
              })
              .then((response) => {
                  logger.debug(response);
  
                  // Make sure we don't have a current copy of the device group
                  return bigIp.cluster.deleteDeviceGroup(options.deviceGroup);
              })
              .then((response) => {
                  logger.debug(response);
                  return provider.getPrimaryCredentials(primaryInstance.mgmtIp, options.port);
              })
              .then((credentials) => {
                  logger.debug('Sending request to join cluster.');
                  return bigIp.cluster.joinCluster(
                      options.deviceGroup,
                      primaryInstance.mgmtIp,
                      credentials.username,
                      credentials.password,
                      false,
                      {
                          remotePort: options.port
                      }
                  );
              });
      }
  
      /**
       * Get the count of running Autoscale process,
       * its pid and current execution time with actions of join or update.
       */
      function getAutoscaleProcessInfo() {
          const actions = 'cluster-action update|' +
              '-c update|cluster-action join|' +
              '-c join|cluster-action backup-ucs';
          const grepCommand = `grep autoscale.js | grep -E '${actions}' | grep -v 'grep autoscale.js'`;
          const results = {};
  
  
          return util.getProcessCount(grepCommand)
              .then((response) => {
                  if (response) {
                      results.processCount = response;
                  }
                  return util.getProcessExecutionTimeWithPid(grepCommand);
              })
              .then((response) => {
                  if (response) {
                      results.pid = response.split('-')[0];
                      results.executionTime = response.split('-')[1].split(':')[0];
                      logger.silly(`Longer running autoscale process id: ${results.pid}`);
                  }
                  return q(results);
              })
              .catch((err) => {
                  logger.error('Could not determine if another autoscale script is running');
                  return q.reject(err);
              });
      }
  
      function checkClusteredDevices(provider, bigIp) {
          return bigIp.cluster.getCmSyncStatus()
              .then((response) => {
                  // response is an object of two lists (connected/disconnected) from getCmSyncStatus()
                  logger.silly('cmSyncStatus:', response);
                  const promises = [];
                  const disconnected = response ? response.disconnected : [];
                  const connected = response ? response.connected : [];
                  const hostnames = [];
                  const hostnamesToRemove = [];
  
                  if (disconnected.length > 0) {
                      logger.info('Possibly disconnected devices:', disconnected);
  
                      // get a list of hostnames still in the instances list
                      Object.keys(this.instances).forEach((instanceId) => {
                          if (this.instances[instanceId].hostname) {
                              hostnames.push(this.instances[instanceId].hostname);
                          }
                      });
  
                      // make sure this is not still in the instances list
                      disconnected.forEach((hostname) => {
                          if (hostnames.indexOf(hostname) === -1) {
                              logger.info('Disconnected device:', hostname);
                              hostnamesToRemove.push(hostname);
                          }
                      });
  
                      if (hostnamesToRemove.length > 0) {
                          logger.info('Removing devices from cluster:', hostnamesToRemove);
                          promises.push(bigIp.cluster.removeFromCluster(hostnamesToRemove));
                      }
                  }
  
                  if (connected.length > 0) {
                      connected.forEach((hostaname) => {
                          Object.keys(this.instances).forEach((instanceId) => {
                              if (this.instances[instanceId].hostname === hostaname &&
                                  this.instances[instanceId].lastBackup < this.instance.lastBackup) {
                                  this.instances[instanceId].lastBackup = this.instance.lastBackup;
                                  logger.silly(`Update lastBackUp on connected instance: ${instanceId}`);
                                  promises.push(provider.putInstance(
                                      instanceId,
                                      this.instances[instanceId]
                                  ));
                              }
                          });
                      });
                  }
                  return Promise.all(promises);
              })
              .catch((err) => {
                  logger.warn('Could not get sync status');
                  return q.reject(err);
              });
      }
  
      /**
       * If the provider supports encryption, initializes and stores keys.
       *
       * Called with this bound to the caller.
       */
      function initEncryption(provider, bigIp) {
          const PRIVATE_KEY_OUT_FILE = '/tmp/tempPrivateKey.pem';
  
          let passphrase;
  
          if (provider.hasFeature(CloudProvider.FEATURE_ENCRYPTION)) {
              logger.debug('Generating public/private keys for autoscaling.');
              return cryptoUtil.generateRandomBytes(PASSPHRASE_LENGTH, 'base64')
                  .then((response) => {
                      passphrase = response;
                      return cryptoUtil.generateKeyPair(
                          PRIVATE_KEY_OUT_FILE,
                          { passphrase, keyLength: '3072' }
                      );
                  })
                  .then((publicKey) => {
                      return provider.putPublicKey(this.instanceId, publicKey);
                  })
                  .then(() => {
                      return bigIp.installPrivateKey(
                          PRIVATE_KEY_OUT_FILE,
                          AUTOSCALE_PRIVATE_KEY_FOLDER,
                          AUTOSCALE_PRIVATE_KEY,
                          { passphrase }
                      );
                  })
                  .then(() => {
                      return bigIp.save();
                  })
                  .catch((err) => {
                      logger.info('initEncryption error', err && err.message ? err.message : err);
                      return q.reject(err);
                  });
          }
          return q();
      }
  
      /**
       * Gets the instance marked as primary
       *
       * @param {Object} instances - Instances map
       *
       * @returns {Object} primary instance if one is found
       *
       *                   {
       *                       id: instance_id,
       *                       instance: instance_data
       *                   }
       */
      function getPrimaryInstance(instances) {
          let instanceId;
  
          const instanceIds = Object.keys(instances);
          for (let i = 0; i < instanceIds.length; i++) {
              instanceId = instanceIds[i];
              if (instances[instanceId].isPrimary) {
                  return {
                      id: instanceId,
                      instance: instances[instanceId]
                  };
              }
          }
          return null;
      }
  
      /**
       * Checks that primary instance has the most recent
       * version of all the BIG-IP instances
       *
       * @param {Object} instances - Instances map
       */
      function markVersions(instances) {
          let highestVersion = '0.0.0';
          let instance;
  
          Object.keys(instances).forEach((instanceId) => {
              instance = instances[instanceId];
              if (instance.version && util.versionCompare(instance.version, highestVersion) > 0) {
                  highestVersion = instance.version;
              }
          });
  
          Object.keys(instances).forEach((instanceId) => {
              instance = instances[instanceId];
              if (!instance.version || util.versionCompare(instance.version, highestVersion) === 0) {
                  instance.versionOk = true;
              } else {
                  instance.versionOk = false;
              }
          });
      }
  
      /**
       * Checks that if there are external instances, the primary is
       * one of them
       *
       * @param {String} primaryId  - Instance ID of primary
       * @param {Object} instances - Instances map
       *
       * @returns {Boolean} True if there are no external instances or
       *                    if there are external instances and the primary is
       *                    one of them
       */
      function isPrimaryExternalValueOk(primaryId, instances) {
          const instanceIds = Object.keys(instances);
          let instance;
          let hasExternal;
  
          for (let i = 0; i < instanceIds.length; i++) {
              instance = instances[instanceIds[i]];
              if (instance.external) {
                  hasExternal = true;
                  break;
              }
          }
  
          if (hasExternal) {
              return !!instances[primaryId].external;
          }
  
          return true;
      }
  
      /*
       * Determines if the primary status has been bad for more than a certain
       * amount of time.
       *
       * @param {Object} instance - instance as returned by getInstances
       * @param {Object} options
       *
       * @returns {Boolean} Whether or not the primary status has been bad for too long
       */
      function isPrimaryExpired(instance, options) {
          const primaryStatus = instance.primaryStatus || {};
          let isExpired = false;
          let disconnectedMs;
  
          if (primaryStatus.status !== CloudProvider.STATUS_OK) {
              disconnectedMs = new Date() - new Date(primaryStatus.lastStatusChange);
              logger.silly('primary has been disconnected for', disconnectedMs.toString(), 'ms');
              if (disconnectedMs > options.primaryDisconnectedTime) {
                  logger.info('primary has been disconnected for too long (',
                      disconnectedMs.toString(), 'ms )');
                  isExpired = true;
              }
          }
          return isExpired;
      }
  
      function updatePrimaryStatus(provider, status) {
          const now = new Date();
          this.instance.primaryStatus = this.instance.primaryStatus || {};
          this.instance.primaryStatus.lastUpdate = now;
          if (this.instance.primaryStatus.status !== status) {
              this.instance.primaryStatus.status = status;
              this.instance.primaryStatus.lastStatusChange = now;
          }
          return provider.putInstance(this.instanceId, this.instance);
      }
  
      /**
       * Loads UCS
       *
       * @param {Object}        bigIp - bigIp instances
       * @param {Buffer|Stream} ucsData - Either a Buffer or a ReadableStream containing UCS data
       * @param {String}        cloudProvider - Cloud provider (aws, azure, etc)
       *
       * @returns {Promise} Promise that will be resolved when the UCS is loaded or rejected
       *                    if an error occurs.
       */
      function loadUcs(provider, bigIp, ucsData, cloudProvider) {
          const timeStamp = Date.now();
          const originalPath = `${BACKUP.UCS_LOCAL_TMP_DIRECTORY}/ucsOriginal_${timeStamp}.ucs`;
          const updatedPath = `${BACKUP.UCS_LOCAL_TMP_DIRECTORY}/ucsUpdated_${timeStamp}.ucs`;
          const updateScript = `${__dirname}/update_autoscale_ucs.py`;
  
          const deferred = q.defer();
  
          const preLoad = function () {
              // eslint-disable-next-line max-len
              const sedCommand = "sed -i '/sys dynad key {/ { N ; /\\n[[:space:]]\\+key[[:space:]]*\\$M\\$[^\\n]*/ { N;   /\\n[[:space:]]*}/ { d } } }' /config/bigip_base.conf";
              const loadSysConfigCommand = 'load /sys config';
  
              logger.silly('removing dynad key from base config');
              return util.runShellCommand(sedCommand)
                  .then(() => {
                      logger.silly('loading sys config');
                      return util.runTmshCommand(loadSysConfigCommand);
                  })
                  .then(() => {
                      logger.silly('waiting for BIG-IP to be ready');
                      return bigIp.ready();
                  })
                  .catch((err) => {
                      logger.warn('preload of ucs failed:', err);
                      throw err;
                  });
          };
  
          const doLoad = function () {
              const args = [
                  '--original-ucs',
                  originalPath,
                  '--updated-ucs',
                  updatedPath,
                  '--cloud-provider',
                  cloudProvider,
                  '--extract-directory',
                  `${BACKUP.UCS_LOCAL_TMP_DIRECTORY}/ucsRestore`
              ];
              const loadUcsOptions = {
                  initLocalKeys: true
              };
  
              preLoad()
                  .then(() => {
                      childProcess.execFile(updateScript, args, (childProcessErr) => {
                          if (childProcessErr) {
                              const message = `${updateScript} failed: ${childProcessErr}`;
                              logger.warn(message);
                              deferred.reject(new Error(message));
                              return;
                          }
  
                          if (!fs.existsSync(updatedPath)) {
                              logger.warn(`${updatedPath} does not exist after running ${updateScript}`);
                              deferred.reject(new Error('updated ucs not found'));
                              return;
                          }
  
                          // If we're not sharing the password, put our current user back after
                          // load
                          if (!provider.hasFeature(CloudProvider.FEATURE_SHARED_PASSWORD)) {
                              loadUcsOptions.restoreUser = true;
                          }
  
                          bigIp.loadUcs(
                              updatedPath,
                              { 'no-license': true, 'reset-trust': true, 'no-platform-check': true },
                              loadUcsOptions
                          )
                              .then(() => {
                                  // reset-trust on load does not always seem to work
                                  // use a belt-and-suspenders approach and reset now as well
                                  return bigIp.cluster.resetTrust();
                              })
                              .then(() => {
                                  // Attempt to delete the file, but ignore errors
                                  try {
                                      logger.info(`Ignoring errors: deleting originalPath: ${originalPath}
                                      deleting updatePath: ${updatedPath}`);
                                      fs.unlinkSync(originalPath);
                                      fs.unlinkSync(updatedPath);
                                  } finally {
                                      deferred.resolve();
                                  }
                              })
                              .catch((err) => {
                                  logger.info('error loading ucs', err);
                                  deferred.reject(err);
                              });
                      });
                  })
                  .catch((err) => {
                      throw err;
                  });
          };
  
          util.writeUcsFile(originalPath, ucsData)
              .then(() => {
                  doLoad();
              })
              .catch((err) => {
                  logger.warn('Error reading ucs data', err);
                  deferred.reject(err);
              });
          return deferred.promise;
      }
  
      function writePrimaryFile(ucsLoaded) {
          const deferred = q.defer();
          const primaryInfo = { ucsLoaded };
  
          // Mark ourself as primary on disk so other scripts have access to this info
          fs.writeFile(PRIMARY_FILE_PATH, JSON.stringify(primaryInfo), (err) => {
              if (err) {
                  logger.warn('Error saving primary file', err);
                  deferred.reject(err);
                  return;
              }
  
              logger.silly('Wrote primary file', PRIMARY_FILE_PATH, primaryInfo);
              deferred.resolve(true);
          });
  
          return deferred.promise;
      }
  
      function prepareMessageData(provider, instanceId, messageData) {
          if (!provider.hasFeature(CloudProvider.FEATURE_ENCRYPTION)) {
              return q(messageData);
          }
          return util.tryUntil(provider, util.MEDIUM_RETRY, provider.getPublicKey, [instanceId])
              .then((publicKey) => {
                  return cryptoUtil.encrypt(publicKey, messageData);
              });
      }
  
      function readMessageData(provider, bigIp, messageData) {
          let filePromise;
  
          if (!provider.hasFeature(CloudProvider.FEATURE_ENCRYPTION)) {
              return q(messageData);
          }
  
          if (!this.cloudPrivateKeyPath) {
              logger.silly('getting private key path');
              filePromise = bigIp.getPrivateKeyFilePath(AUTOSCALE_PRIVATE_KEY_FOLDER, AUTOSCALE_PRIVATE_KEY);
          } else {
              logger.silly('using cached key');
              filePromise = q(this.cloudPrivateKeyPath);
          }
  
          return filePromise
              .then((cloudPrivateKeyPath) => {
                  this.cloudPrivateKeyPath = cloudPrivateKeyPath;
                  return bigIp.getPrivateKeyMetadata(AUTOSCALE_PRIVATE_KEY_FOLDER, AUTOSCALE_PRIVATE_KEY);
              })
              .then((privateKeyData) => {
                  return cryptoUtil.decrypt(
                      this.cloudPrivateKeyPath,
                      messageData,
                      {
                          passphrase: privateKeyData.passphrase,
                          passphraseEncrypted: true
                      }
                  );
              });
      }
  
      function cleanupAjv(bigIp) {
          return bigIp.deviceInfo()
              .then((deviceInfo) => {
                  if (util.versionCompare(deviceInfo.version, '13.1.0') < 0) {
                      const filesToRemove = [
                          `${__dirname}/../node_modules/ajv/lib/$data.js`,
                          `${__dirname}/../node_modules/ajv/lib/refs/$data.json`
                      ];
  
                      const deferred = q.defer();
                      let filesHandled = 0;
  
                      filesToRemove.forEach((fileToRemove) => {
                          fs.stat(fileToRemove, (statError) => {
                              if (statError) {
                                  filesHandled += 1;
                                  if (filesHandled === filesToRemove.length) {
                                      deferred.resolve();
                                  }
                              } else {
                                  fs.rename(fileToRemove, fileToRemove.replace('$', 'dollar_'), (renameErr) => {
                                      if (renameErr) {
                                          logger.info('cleanupAjv unable to remove', fileToRemove);
                                      }
  
                                      filesHandled += 1;
                                      if (filesHandled === filesToRemove.length) {
                                          deferred.resolve();
                                      }
                                  });
                              }
                          });
                      });
  
                      return deferred.promise;
                  }
                  return q();
              })
              .catch((err) => {
                  logger.info('Unable to cleanup AJV', err);
                  return q.reject(err);
              });
      }
  
      function removeOldUcsFiles(latestFile) {
          const deferred = q.defer();
  
          logger.silly('removing old ucs files');
          fs.readdir(UCS_BACKUP_DIRECTORY, (err, files) => {
              if (err) {
                  logger.info(`Error reading ${UCS_BACKUP_DIRECTORY}`, err);
                  deferred.reject(err);
              } else {
                  files.forEach((file) => {
                      if (file.startsWith(UCS_BACKUP_PREFIX) && file !== latestFile) {
                          fs.unlinkSync(`${UCS_BACKUP_DIRECTORY}/${file}`);
                      }
                  });
                  deferred.resolve();
              }
          });
  
          return deferred.promise;
      }
  
      module.exports = runner;
  
      // If we're called from the command line, run
      // This allows for test code to call us as a module
      if (!module.parent) {
          runner.run(process.argv);
      }
  }());
  