text: #!/usr/bin/python3 -u
  #
  
  # Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
  # All rights reserved.
  #
  # Redistribution and use in source and binary forms, with or without
  # modification, are permitted provided that the following conditions are
  # met:
  #
  # 1. Redistributions of source code must retain the above copyright notice,
  # this list of conditions and the following disclaimer.
  #
  # 2. Redistributions in binary form must reproduce the above copyright
  # notice, this list of conditions and the following disclaimer in the
  # documentation and/or other materials provided with the distribution.
  #
  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
  # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
  # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  
  
  """Script for doing QA on Ganeti.
  
  """
  
  # pylint: disable=C0103
  # due to invalid name
  
  import copy
  import datetime
  import optparse
  import sys
  
  from qa import colors
  from qa import qa_cluster
  from qa import qa_config
  from qa import qa_daemon
  from qa import qa_env
  from qa import qa_error
  from qa import qa_filters
  from qa import qa_group
  from qa import qa_instance
  from qa import qa_iptables
  from qa import qa_monitoring
  from qa import qa_network
  from qa import qa_node
  from qa import qa_os
  from qa import qa_performance
  from qa import qa_job
  from qa import qa_rapi
  from qa import qa_tags
  from qa import qa_utils
  
  from ganeti import utils
  from ganeti import rapi # pylint: disable=W0611
  from ganeti import constants
  from ganeti import netutils
  from ganeti import pathutils
  
  import ganeti.rapi.client # pylint: disable=W0611
  from ganeti.rapi.client import UsesRapiClient
  
  _QA_PROFILE = pathutils.GetLogFilename("qa-profile")
  _PROFILE_LOG_INDENT = ""
  
  def _FormatHeader(line, end=72, mark="-", color=None):
    """Fill a line up to the end column.
  
    """
    line = (mark * 4) + " " + line + " "
    line += "-" * (end - len(line))
    line = line.rstrip()
    line = colors.colorize(line, color=color)
    return line
  
  
  def _DescriptionOf(fn):
    """Computes the description of an item.
  
    """
    if fn.__doc__:
      desc = fn.__doc__.splitlines()[0].strip()
      desc = desc.rstrip(".")
      if fn.__name__:
        desc = "[" + fn.__name__ + "] " + desc
    else:
      desc = "%r" % fn
  
    return desc
  
  
  def RunTest(fn, *args, **kwargs):
    """Runs a test after printing a header.
  
    """
    global _PROFILE_LOG_INDENT
    tstart = datetime.datetime.now()
  
    desc = _DescriptionOf(fn)
  
    print()
    print(_FormatHeader("%s start %s" % (tstart, desc),
                        color=colors.YELLOW, mark="<"))
    with open(_QA_PROFILE, "a") as f:
      f.write("%sExecuting Function %s\n" % (_PROFILE_LOG_INDENT,
                                             fn.__name__))
    _PROFILE_LOG_INDENT += " "
  
    try:
      retval = fn(*args, **kwargs)
      print(_FormatHeader("PASSED %s" % (desc, ), color=colors.GREEN))
      return retval
    except Exception as e:
      print(_FormatHeader("FAILED %s: %s" % (desc, e), color=colors.RED))
      raise
    finally:
      tstop = datetime.datetime.now()
      tdelta = tstop - tstart
      print(_FormatHeader("%s time=%s %s" % (tstop, tdelta, desc),
                          color=colors.MAGENTA, mark=">"))
      _PROFILE_LOG_INDENT = _PROFILE_LOG_INDENT[:-1]
      with open(_QA_PROFILE, "a") as f:
        f.write("%sFunction %s ran for %s\n" % (_PROFILE_LOG_INDENT,
                                                fn.__name__,
                                                tdelta))
  
  
  def ReportTestSkip(desc, testnames):
    """Reports that tests have been skipped.
  
    @type desc: string
    @param desc: string
    @type testnames: string or list of string
    @param testnames: either a single test name in the configuration
        file, or a list of testnames (which will be AND-ed together)
  
    """
    tstart = datetime.datetime.now()
    # TODO: Formatting test names when non-string names are involved
    print(_FormatHeader("%s skipping %s, test(s) %s disabled" %
                        (tstart, desc, testnames),
                        color=colors.BLUE, mark="*"))
  
  
  def RunTestIf(testnames, fn, *args, **kwargs):
    """Runs a test conditionally.
  
    @param testnames: either a single test name in the configuration
        file, or a list of testnames (which will be AND-ed together)
  
    """
    if qa_config.TestEnabled(testnames):
      RunTest(fn, *args, **kwargs)
    else:
      desc = _DescriptionOf(fn)
      ReportTestSkip(desc, testnames)
  
  
  def RunTestBlock(fn, *args, **kwargs):
    """Runs a block of tests after printing a header.
  
    """
    global _PROFILE_LOG_INDENT
    tstart = datetime.datetime.now()
  
    desc = _DescriptionOf(fn)
  
    print()
    print(_FormatHeader("BLOCK %s start %s" % (tstart, desc),
                        color=[colors.YELLOW, colors.BOLD], mark="v"))
    with open(_QA_PROFILE, "a") as f:
      f.write("%sExecuting Function %s\n" % (_PROFILE_LOG_INDENT,
                                             fn.__name__))
    _PROFILE_LOG_INDENT += " "
  
    try:
      return fn(*args, **kwargs)
    except Exception as e:
      print(_FormatHeader("BLOCK FAILED %s: %s" % (desc, e),
                          color=[colors.RED, colors.BOLD]))
      raise
    finally:
      tstop = datetime.datetime.now()
      tdelta = tstop - tstart
      print(_FormatHeader("BLOCK %s time=%s %s" % (tstop, tdelta, desc),
                          color=[colors.MAGENTA, colors.BOLD], mark="^"))
      _PROFILE_LOG_INDENT = _PROFILE_LOG_INDENT[:-1]
      with open(_QA_PROFILE, "a") as f:
        f.write("%sFunction %s ran for %s\n" % (_PROFILE_LOG_INDENT,
                                                fn.__name__,
                                                tdelta))
  
  
  def RunEnvTests():
    """Run several environment tests.
  
    """
    RunTestIf("env", qa_env.TestSshConnection)
    RunTestIf("env", qa_env.TestIcmpPing)
    RunTestIf("env", qa_env.TestGanetiCommands)
  
  
  def SetupCluster():
    """Initializes the cluster.
  
    """
  
    RunTestIf("create-cluster", qa_cluster.TestClusterInit)
    if not qa_config.TestEnabled("create-cluster"):
      # If the cluster is already in place, we assume that exclusive-storage is
      # already set according to the configuration
      qa_config.SetExclusiveStorage(qa_config.get("exclusive-storage", False))
  
    qa_rapi.SetupRapi()
  
    qa_group.ConfigureGroups()
  
    # Test on empty cluster
    RunTestIf("node-list", qa_node.TestNodeList)
    RunTestIf("instance-list", qa_instance.TestInstanceList)
    RunTestIf("job-list", qa_job.TestJobList)
  
    RunTestIf("create-cluster", qa_node.TestNodeAddAll)
    if not qa_config.TestEnabled("create-cluster"):
      # consider the nodes are already there
      qa_node.MarkNodeAddedAll()
  
    RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
    RunTestIf("test-jobqueue", qa_job.TestJobCancellation)
  
    # enable the watcher (unconditionally)
    RunTest(qa_daemon.TestResumeWatcher)
  
    RunTestIf("node-list", qa_node.TestNodeList)
  
    # Test listing fields
    RunTestIf("node-list", qa_node.TestNodeListFields)
    RunTestIf("instance-list", qa_instance.TestInstanceListFields)
    RunTestIf("job-list", qa_job.TestJobListFields)
    RunTestIf("instance-export", qa_instance.TestBackupListFields)
  
    RunTestIf("node-info", qa_node.TestNodeInfo)
  
  
  def RunClusterTests():
    """Runs tests related to gnt-cluster.
  
    """
    for test, fn in [
      ("create-cluster", qa_cluster.TestClusterInitDisk),
      ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto)
      ]:
      RunTestIf(test, fn)
  
    for test, fn in [
      ("cluster-verify", qa_cluster.TestClusterVerify),
      ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
      # TODO: add more cluster modify tests
      ("cluster-modify", qa_cluster.TestClusterModifyEmpty),
      ("cluster-modify", qa_cluster.TestClusterModifyIPolicy),
      ("cluster-modify", qa_cluster.TestClusterModifyISpecs),
      ("cluster-modify", qa_cluster.TestClusterModifyBe),
      ("cluster-modify", qa_cluster.TestClusterModifyDisk),
      ("cluster-modify", qa_cluster.TestClusterModifyDiskTemplates),
      ("cluster-modify", qa_cluster.TestClusterModifyFileStorageDir),
      ("cluster-modify", qa_cluster.TestClusterModifySharedFileStorageDir),
      ("cluster-modify", qa_cluster.TestClusterModifyInstallImage),
      ("cluster-modify", qa_cluster.TestClusterModifyUserShutdown),
      ("cluster-rename", qa_cluster.TestClusterRename),
      ("cluster-info", qa_cluster.TestClusterVersion),
      ("cluster-info", qa_cluster.TestClusterInfo),
      ("cluster-info", qa_cluster.TestClusterGetmaster),
      ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
      (["cluster-copyfile", qa_config.NoVirtualCluster],
       qa_cluster.TestClusterCopyfile),
      ("cluster-command", qa_cluster.TestClusterCommand),
      ("cluster-burnin", qa_cluster.TestClusterBurnin),
      ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
      ("cluster-master-failover",
       qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
      (["cluster-oob", qa_config.NoVirtualCluster],
       qa_cluster.TestClusterOob),
      ("cluster-instance-communication", qa_cluster.TestInstanceCommunication),
      (qa_rapi.Enabled, qa_rapi.TestVersion),
      (qa_rapi.Enabled, qa_rapi.TestEmptyCluster),
      (qa_rapi.Enabled, qa_rapi.TestRapiQuery),
      ]:
      RunTestIf(test, fn)
  
  
  def RunRepairDiskSizes():
    """Run the repair disk-sizes test.
  
    """
    RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)
  
  
  def RunOsTests():
    """Runs all tests related to gnt-os.
  
    """
    os_enabled = ["os", qa_config.NoVirtualCluster]
  
    if qa_config.TestEnabled(qa_rapi.Enabled):
      rapi_getos = qa_rapi.GetOperatingSystems
    else:
      rapi_getos = None
  
    for fn in [
      qa_os.TestOsList,
      qa_os.TestOsDiagnose,
      ]:
      RunTestIf(os_enabled, fn)
  
    for fn in [
      qa_os.TestOsValid,
      qa_os.TestOsInvalid,
      qa_os.TestOsPartiallyValid,
      ]:
      RunTestIf(os_enabled, fn, rapi_getos)
  
    for fn in [
      qa_os.TestOsModifyValid,
      qa_os.TestOsModifyInvalid,
      qa_os.TestOsStatesNonExisting,
      ]:
      RunTestIf(os_enabled, fn)
  
  
  def RunCommonInstanceTests(instance, inst_nodes):
    """Runs a few tests that are common to all disk types.
  
    """
    RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
    RunTestIf(["instance-shutdown", "instance-console", qa_rapi.Enabled],
              qa_rapi.TestRapiStoppedInstanceConsole, instance)
    RunTestIf(["instance-shutdown", "instance-modify"],
              qa_instance.TestInstanceStoppedModify, instance)
    RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
  
    # Test shutdown/start via RAPI
    RunTestIf(["instance-shutdown", qa_rapi.Enabled],
              qa_rapi.TestRapiInstanceShutdown, instance)
    RunTestIf(["instance-shutdown", qa_rapi.Enabled],
              qa_rapi.TestRapiInstanceStartup, instance)
  
    RunTestIf("instance-list", qa_instance.TestInstanceList)
  
    RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
  
    RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
    RunTestIf(["instance-modify", qa_rapi.Enabled],
              qa_rapi.TestRapiInstanceModify, instance)
  
    RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
    RunTestIf(["instance-console", qa_rapi.Enabled],
              qa_rapi.TestRapiInstanceConsole, instance)
  
    RunTestIf("instance-device-names", qa_instance.TestInstanceDeviceNames,
              instance)
    DOWN_TESTS = qa_config.Either([
      "instance-reinstall",
      "instance-rename",
      "instance-grow-disk",
      ])
  
    # shutdown instance for any 'down' tests
    RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance)
  
    # now run the 'down' state tests
    RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
    RunTestIf(["instance-reinstall", qa_rapi.Enabled],
              qa_rapi.TestRapiInstanceReinstall, instance)
  
    if qa_config.TestEnabled("instance-rename"):
      tgt_instance = qa_config.AcquireInstance()
      try:
        rename_source = instance.name
        rename_target = tgt_instance.name
        # perform instance rename to the same name
        RunTest(qa_instance.TestInstanceRenameAndBack,
                rename_source, rename_source)
        RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
                  rename_source, rename_source)
        if rename_target is not None:
          # perform instance rename to a different name, if we have one configured
          RunTest(qa_instance.TestInstanceRenameAndBack,
                  rename_source, rename_target)
          RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
                    rename_source, rename_target)
      finally:
        tgt_instance.Release()
  
    RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance)
  
    # and now start the instance again
    RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance)
  
    RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
  
    RunTestIf("tags", qa_tags.TestInstanceTags, instance)
  
    if instance.disk_template == constants.DT_DRBD8:
      RunTestIf("cluster-verify",
                qa_cluster.TestClusterVerifyDisksBrokenDRBD, instance, inst_nodes)
    RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
  
    RunTestIf(qa_rapi.Enabled, qa_rapi.TestInstance, instance)
  
    # Lists instances, too
    RunTestIf("node-list", qa_node.TestNodeList)
  
    # Some jobs have been run, let's test listing them
    RunTestIf("job-list", qa_job.TestJobList)
  
  
  def RunCommonNodeTests():
    """Run a few common node tests.
  
    """
    RunTestIf("node-volumes", qa_node.TestNodeVolumes)
    RunTestIf("node-storage", qa_node.TestNodeStorage)
    RunTestIf(["node-oob", qa_config.NoVirtualCluster], qa_node.TestOutOfBand)
  
  
  def RunGroupListTests():
    """Run tests for listing node groups.
  
    """
    RunTestIf("group-list", qa_group.TestGroupList)
    RunTestIf("group-list", qa_group.TestGroupListFields)
  
  
  def RunNetworkTests():
    """Run tests for network management.
  
    """
    RunTestIf("network", qa_network.TestNetworkAddRemove)
    RunTestIf("network", qa_network.TestNetworkConnect)
    RunTestIf(["network", "tags"], qa_network.TestNetworkTags)
  
  
  def RunFilterTests():
    """Run tests for job filter management.
  
    """
    RunTestIf("filters", qa_filters.TestFilterList)
    RunTestIf("filters", qa_filters.TestFilterListFields)
    RunTestIf("filters", qa_filters.TestFilterAddRemove)
    RunTestIf("filters", qa_filters.TestFilterReject)
    RunTestIf("filters", qa_filters.TestFilterOpCode)
    RunTestIf("filters", qa_filters.TestFilterReasonChain)
    RunTestIf("filters", qa_filters.TestFilterContinue)
    RunTestIf("filters", qa_filters.TestFilterAcceptPause)
    RunTestIf("filters", qa_filters.TestFilterWatermark)
    RunTestIf("filters", qa_filters.TestFilterRateLimit)
    RunTestIf("filters", qa_filters.TestAdHocReasonRateLimit)
  
  
  def RunGroupRwTests():
    """Run tests for adding/removing/renaming groups.
  
    """
    RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
    RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
    RunTestIf("group-rwops", qa_group.TestGroupModify)
    RunTestIf(["group-rwops", qa_rapi.Enabled], qa_rapi.TestRapiNodeGroups)
    RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
              qa_group.GetDefaultGroup())
  
  
  def RunExportImportTests(instance, inodes):
    """Tries to export and import the instance.
  
    @type inodes: list of nodes
    @param inodes: current nodes of the instance
  
    """
    # FIXME: export explicitly bails out on file based storage. other non-lvm
    # based storage types are untested, though. Also note that import could still
    # work, but is deeply embedded into the "export" case.
    if qa_config.TestEnabled("instance-export"):
      RunTest(qa_instance.TestInstanceExportNoTarget, instance)
  
      pnode = inodes[0]
      expnode = qa_config.AcquireNode(exclude=pnode)
      try:
        name = RunTest(qa_instance.TestInstanceExport, instance, expnode)
  
        RunTest(qa_instance.TestBackupList, expnode)
  
        if qa_config.TestEnabled("instance-import"):
          newinst = qa_config.AcquireInstance()
          try:
            RunTest(qa_instance.TestInstanceImport, newinst, pnode,
                    expnode, name)
            # Check if starting the instance works
            RunTest(qa_instance.TestInstanceStartup, newinst)
            RunTest(qa_instance.TestInstanceRemove, newinst)
          finally:
            newinst.Release()
      finally:
        expnode.Release()
  
    # FIXME: inter-cluster-instance-move crashes on file based instances :/
    # See Issue 414.
    if (qa_config.TestEnabled([qa_rapi.Enabled, "inter-cluster-instance-move"])):
      newinst = qa_config.AcquireInstance()
      try:
        tnode = qa_config.AcquireNode(exclude=inodes)
        try:
          RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
                  inodes, tnode)
        finally:
          tnode.Release()
      finally:
        newinst.Release()
  
  
  def RunDaemonTests(instance):
    """Test the ganeti-watcher script.
  
    """
    RunTest(qa_daemon.TestPauseWatcher)
  
    RunTestIf("instance-automatic-restart",
              qa_daemon.TestInstanceAutomaticRestart, instance)
    RunTestIf("instance-consecutive-failures",
              qa_daemon.TestInstanceConsecutiveFailures, instance)
  
    RunTest(qa_daemon.TestResumeWatcher)
  
  
  def RunHardwareFailureTests(instance, inodes):
    """Test cluster internal hardware failure recovery.
  
    """
    RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
    RunTestIf(["instance-failover", qa_rapi.Enabled],
              qa_rapi.TestRapiInstanceFailover, instance)
  
    RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
    RunTestIf(["instance-migrate", qa_rapi.Enabled],
              qa_rapi.TestRapiInstanceMigrate, instance)
  
    if qa_config.TestEnabled("instance-replace-disks"):
      # We just need alternative secondary nodes, hence "- 1"
      othernodes = qa_config.AcquireManyNodes(len(inodes) - 1, exclude=inodes)
      try:
        RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceReplaceDisks, instance)
        RunTest(qa_instance.TestReplaceDisks,
                instance, inodes, othernodes)
      finally:
        qa_config.ReleaseManyNodes(othernodes)
      del othernodes
  
    if qa_config.TestEnabled("instance-recreate-disks"):
      try:
        acquirednodes = qa_config.AcquireManyNodes(len(inodes), exclude=inodes)
        othernodes = acquirednodes
      except qa_error.OutOfNodesError:
        if len(inodes) > 1:
          # If the cluster is not big enough, let's reuse some of the nodes, but
          # with different roles. In this way, we can test a DRBD instance even on
          # a 3-node cluster.
          acquirednodes = [qa_config.AcquireNode(exclude=inodes)]
          othernodes = acquirednodes + inodes[:-1]
        else:
          raise
      try:
        RunTest(qa_instance.TestRecreateDisks,
                instance, inodes, othernodes)
      finally:
        qa_config.ReleaseManyNodes(acquirednodes)
  
    if len(inodes) >= 2:
      RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, inodes[0], inodes[1])
      RunTestIf("node-failover", qa_node.TestNodeFailover, inodes[0], inodes[1])
      RunTestIf("node-migrate", qa_node.TestNodeMigrate, inodes[0], inodes[1])
  
  
  def RunExclusiveStorageTests():
    """Test exclusive storage."""
    if not qa_config.TestEnabled("cluster-exclusive-storage"):
      return
  
    node = qa_config.AcquireNode()
    try:
      old_es = qa_cluster.TestSetExclStorCluster(False)
      qa_node.TestExclStorSingleNode(node)
  
      qa_cluster.TestSetExclStorCluster(True)
      qa_cluster.TestExclStorSharedPv(node)
  
      if qa_config.TestEnabled("instance-add-plain-disk"):
        # Make sure that the cluster doesn't have any pre-existing problem
        qa_cluster.AssertClusterVerify()
  
        # Create and allocate instances
        instance1 = qa_instance.TestInstanceAddWithPlainDisk([node])
        try:
          instance2 = qa_instance.TestInstanceAddWithPlainDisk([node])
          try:
            # cluster-verify checks that disks are allocated correctly
            qa_cluster.AssertClusterVerify()
  
            # Remove instances
            qa_instance.TestInstanceRemove(instance2)
            qa_instance.TestInstanceRemove(instance1)
          finally:
            instance2.Release()
        finally:
          instance1.Release()
  
      if qa_config.TestEnabled("instance-add-drbd-disk"):
        snode = qa_config.AcquireNode()
        try:
          qa_cluster.TestSetExclStorCluster(False)
          instance = qa_instance.TestInstanceAddWithDrbdDisk([node, snode])
          try:
            qa_cluster.TestSetExclStorCluster(True)
            exp_err = [constants.CV_EINSTANCEUNSUITABLENODE]
            qa_cluster.AssertClusterVerify(fail=True, errors=exp_err)
            qa_instance.TestInstanceRemove(instance)
          finally:
            instance.Release()
        finally:
          snode.Release()
      qa_cluster.TestSetExclStorCluster(old_es)
    finally:
      node.Release()
  
  
  def RunCustomSshPortTests():
    """Test accessing nodes with custom SSH ports.
  
    This requires removing nodes, adding them to a new group, and then undoing
    the change.
    """
    if not qa_config.TestEnabled("group-custom-ssh-port"):
      return
  
    std_port = netutils.GetDaemonPort(constants.SSH)
    port = 211
    master = qa_config.GetMasterNode()
    with qa_config.AcquireManyNodesCtx(1, exclude=master) as nodes:
      # Checks if the node(s) could be contacted through IPv6.
      # If yes, better skip the whole test.
  
      for node in nodes:
        if qa_utils.UsesIPv6Connection(node.primary, std_port):
          print("Node %s is likely to be reached using IPv6,"
                "skipping the test" % (node.primary, ))
          return
  
      for node in nodes:
        qa_node.NodeRemove(node)
      with qa_iptables.RulesContext() as r:
        with qa_group.NewGroupCtx() as group:
          qa_group.ModifyGroupSshPort(r, group, nodes, port)
  
          for node in nodes:
            qa_node.NodeAdd(node, group=group)
  
          # Make sure that the cluster doesn't have any pre-existing problem
          qa_cluster.AssertClusterVerify()
  
          # Create and allocate instances
          instance1 = qa_instance.TestInstanceAddWithPlainDisk(nodes)
          try:
            instance2 = qa_instance.TestInstanceAddWithPlainDisk(nodes)
            try:
              # cluster-verify checks that disks are allocated correctly
              qa_cluster.AssertClusterVerify()
  
              # Remove instances
              qa_instance.TestInstanceRemove(instance2)
              qa_instance.TestInstanceRemove(instance1)
            finally:
              instance2.Release()
          finally:
            instance1.Release()
  
          for node in nodes:
            qa_node.NodeRemove(node)
  
      for node in nodes:
        qa_node.NodeAdd(node)
  
      qa_cluster.AssertClusterVerify()
  
  
  def _BuildSpecDict(par, mn, st, mx):
    return {
      constants.ISPECS_MINMAX: [{
        constants.ISPECS_MIN: {par: mn},
        constants.ISPECS_MAX: {par: mx},
        }],
      constants.ISPECS_STD: {par: st},
      }
  
  
  def _BuildDoubleSpecDict(index, par, mn, st, mx):
    new_spec = {
      constants.ISPECS_MINMAX: [{}, {}],
      }
    if st is not None:
      new_spec[constants.ISPECS_STD] = {par: st}
    new_spec[constants.ISPECS_MINMAX][index] = {
      constants.ISPECS_MIN: {par: mn},
      constants.ISPECS_MAX: {par: mx},
      }
    return new_spec
  
  
  def TestIPolicyPlainInstance():
    """Test instance policy interaction with instances"""
    params = ["memory-size", "cpu-count", "disk-count", "disk-size", "nic-count"]
    if not qa_config.IsTemplateSupported(constants.DT_PLAIN):
      print("Template %s not supported" % constants.DT_PLAIN)
      return
  
    # This test assumes that the group policy is empty
    (_, old_specs) = qa_cluster.TestClusterSetISpecs()
    # We also assume to have only one min/max bound
    assert len(old_specs[constants.ISPECS_MINMAX]) == 1
    node = qa_config.AcquireNode()
    try:
      # Log of policy changes, list of tuples:
      # (full_change, incremental_change, policy_violated)
      history = []
      instance = qa_instance.TestInstanceAddWithPlainDisk([node])
      try:
        policyerror = [constants.CV_EINSTANCEPOLICY]
        for par in params:
          (iminval, imaxval) = qa_instance.GetInstanceSpec(instance.name, par)
          # Some specs must be multiple of 4
          new_spec = _BuildSpecDict(par, imaxval + 4, imaxval + 4, imaxval + 4)
          history.append((None, new_spec, True))
          if iminval > 0:
            # Some specs must be multiple of 4
            if iminval >= 4:
              upper = iminval - 4
            else:
              upper = iminval - 1
            new_spec = _BuildSpecDict(par, 0, upper, upper)
            history.append((None, new_spec, True))
          history.append((old_specs, None, False))
  
        # Test with two instance specs
        double_specs = copy.deepcopy(old_specs)
        double_specs[constants.ISPECS_MINMAX] = \
            double_specs[constants.ISPECS_MINMAX] * 2
        (par1, par2) = params[0:2]
        (_, imaxval1) = qa_instance.GetInstanceSpec(instance.name, par1)
        (_, imaxval2) = qa_instance.GetInstanceSpec(instance.name, par2)
        old_minmax = old_specs[constants.ISPECS_MINMAX][0]
        history.extend([
          (double_specs, None, False),
          # The first min/max limit is being violated
          (None,
           _BuildDoubleSpecDict(0, par1, imaxval1 + 4, imaxval1 + 4,
                                imaxval1 + 4),
           False),
          # Both min/max limits are being violated
          (None,
           _BuildDoubleSpecDict(1, par2, imaxval2 + 4, None, imaxval2 + 4),
           True),
          # The second min/max limit is being violated
          (None,
           _BuildDoubleSpecDict(0, par1,
                                old_minmax[constants.ISPECS_MIN][par1],
                                old_specs[constants.ISPECS_STD][par1],
                                old_minmax[constants.ISPECS_MAX][par1]),
           False),
          (old_specs, None, False),
          ])
  
        # Apply the changes, and check policy violations after each change
        qa_cluster.AssertClusterVerify()
        for (new_specs, diff_specs, failed) in history:
          qa_cluster.TestClusterSetISpecs(new_specs=new_specs,
                                          diff_specs=diff_specs)
          if failed:
            qa_cluster.AssertClusterVerify(warnings=policyerror)
          else:
            qa_cluster.AssertClusterVerify()
  
        qa_instance.TestInstanceRemove(instance)
      finally:
        instance.Release()
  
      # Now we replay the same policy changes, and we expect that the instance
      # cannot be created for the cases where we had a policy violation above
      for (new_specs, diff_specs, failed) in history:
        qa_cluster.TestClusterSetISpecs(new_specs=new_specs,
                                        diff_specs=diff_specs)
        if failed:
          qa_instance.TestInstanceAddWithPlainDisk([node], fail=True)
        # Instance creation with no policy violation has been tested already
    finally:
      node.Release()
  
  
  def IsExclusiveStorageInstanceTestEnabled():
    test_name = "exclusive-storage-instance-tests"
    if qa_config.TestEnabled(test_name):
      vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
      vgscmd = utils.ShellQuoteArgs([
        "vgs", "--noheadings", "-o", "pv_count", vgname,
        ])
      nodes = qa_config.GetConfig()["nodes"]
      for node in nodes:
        try:
          pvnum = int(qa_utils.GetCommandOutput(node.primary, vgscmd))
        except Exception as e:
          msg = ("Cannot get the number of PVs on %s, needed by '%s': %s" %
                 (node.primary, test_name, e))
          raise qa_error.Error(msg)
        if pvnum < 2:
          raise qa_error.Error("Node %s has not enough PVs (%s) to run '%s'" %
                               (node.primary, pvnum, test_name))
      res = True
    else:
      res = False
    return res
  
  
  def RunInstanceTests():
    """Create and exercise instances."""
  
    requested_conversions = qa_config.get("convert-disk-templates", [])
    supported_conversions = \
        set(requested_conversions).difference(constants.DTS_NOT_CONVERTIBLE_TO)
    for (test_name, templ, create_fun, num_nodes) in \
        qa_instance.available_instance_tests:
      if (qa_config.TestEnabled(test_name) and
          qa_config.IsTemplateSupported(templ)):
        inodes = qa_config.AcquireManyNodes(num_nodes)
        try:
          # run instance tests with default hvparams
          print(_FormatHeader("Starting instance tests with default HVparams"))
          RunInstanceTestsFull(create_fun, inodes, supported_conversions, templ)
  
          # iterate through alternating hvparam values (if enabled)
          if qa_config.TestEnabled("instance-iterate-hvparams"):
            hvparam_iterations = qa_cluster.PrepareHvParameterSets()
            for param, test_data in hvparam_iterations.items():
              for value in test_data["values"]:
                print(_FormatHeader("Starting reduced number of instance tests "
                                    "with hypervisor parameter %s=%s" %
                                    (param, value)))
                qa_cluster.AssertClusterHvParameterModify(param, value)
                RunInstanceTestsReduced(create_fun, inodes)
  
              qa_cluster.AssertClusterHvParameterModify(
                param, test_data["reset_value"])
          else:
            test_desc = "Iterating through hypervisor parameter values"
            ReportTestSkip(test_desc, "instance-iterate-hvparams")
        finally:
          qa_config.ReleaseManyNodes(inodes)
      else:
        test_desc = "Creating instances of template %s" % templ
        if not qa_config.TestEnabled(test_name):
          ReportTestSkip(test_desc, test_name)
        else:
          ReportTestSkip(test_desc, "disk template %s" % templ)
  
  
  def RunInstanceTestsFull(create_fun, inodes, supported_conversions, templ):
    instance = RunTest(create_fun, inodes)
    try:
      RunTestIf("instance-user-down", qa_instance.TestInstanceUserDown,
                instance)
      RunTestIf("instance-communication",
                qa_instance.TestInstanceCommunication,
                instance,
                qa_config.GetMasterNode())
      RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
      RunDaemonTests(instance)
      for node in inodes:
        RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node,
                  templ == constants.DT_DRBD8)
      if len(inodes) > 1:
        RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
                  constants.INITIAL_NODE_GROUP_NAME,
                  inodes[0].primary, inodes[1].primary)
      # This test will run once but it will cover all the supported
      # user-provided disk template conversions
      if qa_config.TestEnabled("instance-convert-disk"):
        if (len(supported_conversions) > 1 and
                instance.disk_template in supported_conversions):
          RunTest(qa_instance.TestInstanceShutdown, instance)
          RunTest(qa_instance.TestInstanceConvertDiskTemplate, instance,
                  supported_conversions)
          RunTest(qa_instance.TestInstanceStartup, instance)
          # At this point we clear the set because the requested conversions
          # has been tested
          supported_conversions.clear()
        else:
          test_desc = "Converting instance of template %s" % templ
          ReportTestSkip(test_desc, "conversion feature")
      RunTestIf("instance-modify-disks",
                qa_instance.TestInstanceModifyDisks, instance)
      RunCommonInstanceTests(instance, inodes)
      if qa_config.TestEnabled("instance-modify-primary"):
        othernode = qa_config.AcquireNode()
        RunTest(qa_instance.TestInstanceModifyPrimaryAndBack,
                instance, inodes[0], othernode)
        othernode.Release()
      RunGroupListTests()
      RunExportImportTests(instance, inodes)
      RunHardwareFailureTests(instance, inodes)
      RunRepairDiskSizes()
      RunTestIf(["rapi", "instance-data-censorship"],
                qa_rapi.TestInstanceDataCensorship, instance, inodes)
      RunTest(qa_instance.TestInstanceRemove, instance)
    finally:
      instance.Release()
    del instance
  
    qa_cluster.AssertClusterVerify()
  
  
  def RunInstanceTestsReduced(create_fun, inodes):
    instance = RunTest(create_fun, inodes)
    try:
      RunCommonInstanceTests(instance, inodes)
      RunTest(qa_instance.TestInstanceRemove, instance)
    finally:
      instance.Release()
    del instance
  
    qa_cluster.AssertClusterVerify()
  
  
  def RunMonitoringTests():
    RunTestIf("mon-collector", qa_monitoring.TestInstStatusCollector)
  
  
  PARALLEL_TEST_DICT = {
    "parallel-failover": qa_performance.TestParallelInstanceFailover,
    "parallel-migration": qa_performance.TestParallelInstanceMigration,
    "parallel-replace-disks": qa_performance.TestParallelInstanceReplaceDisks,
    "parallel-reboot": qa_performance.TestParallelInstanceReboot,
    "parallel-reinstall": qa_performance.TestParallelInstanceReinstall,
    "parallel-rename": qa_performance.TestParallelInstanceRename,
    }
  
  
  def RunPerformanceTests():
    if not qa_config.TestEnabled("performance"):
      ReportTestSkip("performance related tests", "performance")
      return
  
    # For reproducable performance, run performance tests with the watcher
    # paused.
    qa_utils.AssertCommand(["gnt-cluster", "watcher", "pause", "4h"])
  
    if qa_config.TestEnabled("jobqueue-performance"):
      RunTest(qa_performance.TestParallelMaxInstanceCreationPerformance)
      RunTest(qa_performance.TestParallelNodeCountInstanceCreationPerformance)
  
      instances = qa_performance.CreateAllInstances()
  
      RunTest(qa_performance.TestParallelModify, instances)
      RunTest(qa_performance.TestParallelInstanceOSOperations, instances)
      RunTest(qa_performance.TestParallelInstanceQueries, instances)
  
      qa_performance.RemoveAllInstances(instances)
  
      RunTest(qa_performance.TestJobQueueSubmissionPerformance)
  
    if qa_config.TestEnabled("parallel-performance"):
      if qa_config.IsTemplateSupported(constants.DT_DRBD8):
        RunTest(qa_performance.TestParallelDRBDInstanceCreationPerformance)
      if qa_config.IsTemplateSupported(constants.DT_PLAIN):
        RunTest(qa_performance.TestParallelPlainInstanceCreationPerformance)
  
    # Preparations need to be made only if some of these tests are enabled
    if qa_config.IsTemplateSupported(constants.DT_DRBD8) and \
       qa_config.TestEnabled(qa_config.Either(list(PARALLEL_TEST_DICT))):
      inodes = qa_config.AcquireManyNodes(2)
      try:
        instance = qa_instance.TestInstanceAddWithDrbdDisk(inodes)
        try:
          for (test_name, test_fn) in PARALLEL_TEST_DICT.items():
            RunTestIf(test_name, test_fn, instance)
        finally:
          instance.Release()
        qa_instance.TestInstanceRemove(instance)
      finally:
        qa_config.ReleaseManyNodes(inodes)
  
    qa_utils.AssertCommand(["gnt-cluster", "watcher", "continue"])
  
  
  def RunQa():
    """Main QA body.
  
    """
    RunTestBlock(RunEnvTests)
    SetupCluster()
  
    RunTestBlock(RunClusterTests)
    RunTestBlock(RunOsTests)
  
    RunTestIf("tags", qa_tags.TestClusterTags)
  
    RunTestBlock(RunCommonNodeTests)
    RunTestBlock(RunGroupListTests)
    RunTestBlock(RunGroupRwTests)
    RunTestBlock(RunNetworkTests)
    RunTestBlock(RunFilterTests)
  
    # The master shouldn't be readded or put offline; "delay" needs a non-master
    # node to test
    pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
    try:
      RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
      RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
      RunTestIf("delay", qa_cluster.TestDelay, pnode)
    finally:
      pnode.Release()
  
    # Make sure the cluster is clean before running instance tests
    qa_cluster.AssertClusterVerify()
  
    pnode = qa_config.AcquireNode()
    try:
      RunTestIf("tags", qa_tags.TestNodeTags, pnode)
  
      if qa_rapi.Enabled():
        RunTest(qa_rapi.TestNode, pnode)
  
        if (qa_config.TestEnabled("instance-add-plain-disk")
            and qa_config.IsTemplateSupported(constants.DT_PLAIN)):
          # Normal instance allocation via RAPI
          for use_client in [True, False]:
            rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
                                    use_client)
            try:
              if qa_config.TestEnabled("instance-plain-rapi-common-tests"):
                RunCommonInstanceTests(rapi_instance, [pnode])
              RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
            finally:
              rapi_instance.Release()
            del rapi_instance
  
          # Multi-instance allocation
          rapi_instance_one, rapi_instance_two = \
            RunTest(qa_rapi.TestRapiInstanceMultiAlloc, pnode)
  
          try:
            RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance_one, True)
            RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance_two, True)
          finally:
            rapi_instance_one.Release()
            rapi_instance_two.Release()
    finally:
      pnode.Release()
  
    config_list = [
      ("default-instance-tests", lambda: None, lambda _: None),
      (IsExclusiveStorageInstanceTestEnabled,
       lambda: qa_cluster.TestSetExclStorCluster(True),
       qa_cluster.TestSetExclStorCluster),
    ]
    for (conf_name, setup_conf_f, restore_conf_f) in config_list:
      if qa_config.TestEnabled(conf_name):
        oldconf = setup_conf_f()
        RunTestBlock(RunInstanceTests)
        restore_conf_f(oldconf)
  
    pnode = qa_config.AcquireNode()
    try:
      if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
        for shutdown in [False, True]:
          instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, [pnode])
          try:
            expnode = qa_config.AcquireNode(exclude=pnode)
            try:
              if shutdown:
                # Stop instance before exporting and removing it
                RunTest(qa_instance.TestInstanceShutdown, instance)
              RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
              RunTest(qa_instance.TestBackupList, expnode)
            finally:
              expnode.Release()
          finally:
            instance.Release()
          del expnode
          del instance
        qa_cluster.AssertClusterVerify()
  
    finally:
      pnode.Release()
  
    if qa_rapi.Enabled():
      RunTestIf("filters", qa_rapi.TestFilters)
  
    RunTestIf("cluster-upgrade", qa_cluster.TestUpgrade)
  
    RunTestBlock(RunExclusiveStorageTests)
    RunTestIf(["cluster-instance-policy", "instance-add-plain-disk"],
              TestIPolicyPlainInstance)
  
    RunTestBlock(RunCustomSshPortTests)
  
    RunTestIf(
      "instance-add-restricted-by-disktemplates",
      qa_instance.TestInstanceCreationRestrictedByDiskTemplates)
  
    RunTestIf("instance-add-osparams", qa_instance.TestInstanceAddOsParams)
    RunTestIf("instance-add-osparams", qa_instance.TestSecretOsParams)
  
    # Test removing instance with offline drbd secondary
    if qa_config.TestEnabled(["instance-remove-drbd-offline",
                              "instance-add-drbd-disk"]):
      # Make sure the master is not put offline
      snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
      try:
        pnode = qa_config.AcquireNode(exclude=snode)
        try:
          instance = qa_instance.TestInstanceAddWithDrbdDisk([pnode, snode])
          set_offline = lambda node: qa_node.MakeNodeOffline(node, "yes")
          set_online = lambda node: qa_node.MakeNodeOffline(node, "no")
          RunTest(qa_instance.TestRemoveInstanceOfflineNode, instance, snode,
                  set_offline, set_online)
        finally:
          pnode.Release()
      finally:
        snode.Release()
      qa_cluster.AssertClusterVerify()
  
    RunTestBlock(RunMonitoringTests)
  
    RunPerformanceTests()
  
    RunTestIf("cluster-destroy", qa_node.TestNodeRemoveAll)
  
    RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
  
  
  @UsesRapiClient
  def main():
    """Main program.
  
    """
    colors.check_for_colors()
  
    parser = optparse.OptionParser(usage="%prog [options] <config-file>")
    parser.add_option("--yes-do-it", dest="yes_do_it",
                      action="store_true",
                      help="Really execute the tests")
    (opts, args) = parser.parse_args()
  
    if len(args) == 1:
      (config_file, ) = args
    else:
      parser.error("Wrong number of arguments.")
  
    if not opts.yes_do_it:
      print ("Executing this script irreversibly destroys any Ganeti\n"
             "configuration on all nodes involved. If you really want\n"
             "to start testing, supply the --yes-do-it option.")
      sys.exit(1)
  
    qa_config.Load(config_file)
  
    primary = qa_config.GetMasterNode().primary
    qa_utils.StartMultiplexer(primary)
    print("SSH command for primary node: %s" %
          utils.ShellQuoteArgs(qa_utils.GetSSHCommand(primary, "")))
    print("SSH command for other nodes: %s" %
          utils.ShellQuoteArgs(qa_utils.GetSSHCommand("NODE", "")))
    try:
      RunQa()
    finally:
      qa_utils.CloseMultiplexers()
  
  if __name__ == "__main__":
    main()
  