files: # See https://pre-commit.com for more information
  # See https://pre-commit.com/hooks.html for more hooks
  repos:
    - repo: https://github.com/pre-commit/mirrors-autopep8
      rev: 'v1.5.7'
      hooks:
        - id: autopep8
          args:
            - --recursive
            - --in-place
            - --aggressive
            - --aggressive
            - --hang-closing
            - --max-line-length=99
  
    - repo: https://github.com/PyCQA/isort
      rev: "5.8.0"
      hooks:
        - id: isort
  
    - repo: https://github.com/pycqa/flake8
      rev: "4.0.1"
      hooks:
        - id: flake8
          args:
            - --max-line-length=99
          files: >
            (?x)^(
              examples/.*|
              tests/.*|
              tmt/.*
            )$
  
    - repo: https://github.com/pre-commit/pre-commit-hooks
      rev: "v3.4.0"
      hooks:
        - id: end-of-file-fixer
        - id: mixed-line-ending
        - id: trailing-whitespace
  
    - repo: https://github.com/pre-commit/mirrors-mypy
      rev: "v0.961"
      hooks:
        - id: mypy
          additional_dependencies: [types-click]
          pass_filenames: false
          args:
            - --config-file
            - mypy.ini
  
    - repo: https://github.com/python-jsonschema/check-jsonschema
      rev: 0.15.0
      hooks:
        - id: check-metaschema
          name: "Check JSON schemas validity"
          files: ^tmt/schemas/.*\.yaml
  
    - repo: https://github.com/adrienverge/yamllint
      rev: v1.26.3
      hooks:
        - id: yamllint
          files: ^tmt/schemas/.*\.yaml
  
    # Yet another static analysis - these hooks use regular expressions to
    # process Python code, and offer interesting "metalinters", checks for
    # what we do to appease flake8 and mypy linters.
    - repo: https://github.com/pre-commit/pygrep-hooks
      rev: v1.9.0
      hooks:
        # Enforce `noqa` and `type: ignore` to always appear with specific
        # error code(s).
        - id: python-check-blanket-noqa
        - id: python-check-blanket-type-ignore
  
        # Other potentially useful hooks for future consideration:
        #
        # - id: python-check-mock-methods
        # - id: python-no-eval
        # - id: python-no-log-warn
        # - id: python-use-type-annotations
        # - id: text-unicode-replacement-char
  
files: .pre-commit-config.yaml
files: # See https://pre-commit.com for more information
  # See https://pre-commit.com/hooks.html for more hooks
  repos:
    - repo: https://github.com/pre-commit/mirrors-autopep8
      rev: 'v1.5.7'
      hooks:
        - id: autopep8
          args:
            - --recursive
            - --in-place
            - --aggressive
            - --aggressive
            - --hang-closing
            - --max-line-length=99
  
    - repo: https://github.com/PyCQA/isort
      rev: "5.8.0"
      hooks:
        - id: isort
  
    - repo: https://github.com/pycqa/flake8
      rev: "4.0.1"
      hooks:
        - id: flake8
          args:
            - --max-line-length=99
          files: >
            (?x)^(
              examples/.*|
              tests/.*|
              tmt/.*
            )$
  
    - repo: https://github.com/pre-commit/pre-commit-hooks
      rev: "v3.4.0"
      hooks:
        - id: end-of-file-fixer
        - id: mixed-line-ending
        - id: trailing-whitespace
  
    - repo: https://github.com/pre-commit/mirrors-mypy
      rev: "v0.961"
      hooks:
        - id: mypy
          additional_dependencies: [types-click]
          pass_filenames: false
          args:
            - --config-file
            - mypy.ini
  
    - repo: https://github.com/python-jsonschema/check-jsonschema
      rev: 0.15.0
      hooks:
        - id: check-metaschema
          name: "Check JSON schemas validity"
          files: ^tmt/schemas/.*\.yaml
  
    - repo: https://github.com/adrienverge/yamllint
      rev: v1.26.3
      hooks:
        - id: yamllint
          files: ^tmt/schemas/.*\.yaml
  
files: .pre-commit-config.yaml
files: # coding: utf-8
  
  """ Command line interface for the Test Management Tool """
  
  import collections
  import dataclasses
  import os
  import subprocess
  import sys
  from typing import TYPE_CHECKING, Any, Callable, List, Optional, Set
  
  import click
  import fmf
  from click import echo, style
  from fmf.utils import listed
  
  import tmt
  import tmt.convert
  import tmt.export
  import tmt.identifier
  import tmt.options
  import tmt.plugins
  import tmt.steps
  import tmt.templates
  import tmt.utils
  
  if TYPE_CHECKING:
      import tmt.steps.discover
      import tmt.steps.execute
  
  # Explore available plugins (need to detect all supported methods first)
  tmt.plugins.explore()
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Click Context Object Container
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  
  @dataclasses.dataclass
  class ContextObject:
      """ Click Context Object Container """
      common: tmt.utils.Common
      fmf_context: tmt.utils.FmfContextType
      steps: Set[tmt.steps.Step]
      tree: tmt.Tree
      clean: Optional[tmt.Clean] = None
      run: Optional[tmt.Run] = None
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Custom Group
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  class CustomGroup(click.Group):
      """ Custom Click Group """
  
      def list_commands(self, context: click.core.Context) -> List[str]:
          """ Prevent alphabetical sorting """
          return list(self.commands.keys())
  
      def get_command(self, context: click.core.Context, cmd_name: str) -> Optional[click.Command]:
          """ Allow command shortening """
          # Backward-compatible 'test convert' (just temporary for now FIXME)
          cmd_name = cmd_name.replace('convert', 'import')
          # Support both story & stories
          cmd_name = cmd_name.replace('story', 'stories')
          found = click.Group.get_command(self, context, cmd_name)
          if found is not None:
              return found
          matches = [command for command in self.list_commands(context)
                     if command.startswith(cmd_name)]
          if not matches:
              return None
          elif len(matches) == 1:
              return click.Group.get_command(self, context, matches[0])
          context.fail('Did you mean {}?'.format(
              listed(sorted(matches), join='or')))
          return None
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Common Options
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  def verbose_debug_quiet(function: Callable[..., Any]) -> Callable[..., Any]:
      """ Verbose, debug and quiet output """
      for option in reversed(tmt.options.verbose_debug_quiet):
          function = option(function)
      return function
  
  
  def force_dry(function: Callable[..., Any]) -> Callable[..., Any]:
      """ Force and dry actions """
      for option in reversed(tmt.options.force_dry):
          function = option(function)
      return function
  
  
  def fix(function: Callable[..., Any]) -> Callable[..., Any]:
      """ Fix action """
      function = tmt.options.fix(function)
      return function
  
  
  def workdir_root(function: Callable[..., Any]) -> Callable[..., Any]:
      """ Where tmt workdirs are to be found """
      return tmt.options.workdir_root(function)
  
  
  def name_filter_condition(function: Callable[..., Any]) -> Callable[..., Any]:
      """ Common filter options (short & long) """
      options = [
          click.argument(
              'names', nargs=-1, metavar='[REGEXP|.]'),
          click.option(
              '-f', '--filter', 'filters', metavar='FILTER', multiple=True,
              help="Apply advanced filter (see 'pydoc fmf.filter')."),
          click.option(
              '-c', '--condition', 'conditions', metavar="EXPR", multiple=True,
              help="Use arbitrary Python expression for filtering."),
          click.option(
              '--enabled', is_flag=True,
              help="Show only enabled tests, plans or stories."),
          click.option(
              '--disabled', is_flag=True,
              help="Show only disabled tests, plans or stories."),
          click.option(
              '--link', 'links', metavar="RELATION:TARGET", multiple=True,
              help="Filter by linked objects (regular expressions are "
                   "supported for both relation and target)."),
          click.option(
              '-x', '--exclude', 'exclude', metavar='[REGEXP]', multiple=True,
              help="Exclude a regular expression from search result."),
          ]
  
      for option in reversed(options):
          function = option(function)
      return function
  
  
  def source(function: Callable[..., Any]) -> Callable[..., Any]:
      """ Option to select fmf objects by their sources """
      return click.option(
          '--source', is_flag=True, help="Select by fmf source file names instead of object names."
          )(function)
  
  
  def name_filter_condition_long(function: Callable[..., Any]) -> Callable[..., Any]:
      """ Common filter options (long only) """
      options = [
          click.argument(
              'names', nargs=-1, metavar='[REGEXP|.]'),
          click.option(
              '--filter', 'filters', metavar='FILTER', multiple=True,
              help="Apply advanced filter (see 'pydoc fmf.filter')."),
          click.option(
              '--condition', 'conditions', metavar="EXPR", multiple=True,
              help="Use arbitrary Python expression for filtering."),
          click.option(
              '--enabled', is_flag=True,
              help="Show only enabled tests, plans or stories."),
          click.option(
              '--disabled', is_flag=True,
              help="Show only disabled tests, plans or stories."),
          click.option(
              '--link', 'links', metavar="RELATION:TARGET", multiple=True,
              help="Filter by linked objects (regular expressions are "
                   "supported for both relation and target)."),
          click.option(
              '--exclude', 'exclude', metavar='[REGEXP]', multiple=True,
              help="Exclude a regular expression from search result."),
          ]
  
      for option in reversed(options):
          function = option(function)
      return function
  
  
  def implemented_verified_documented(function: Callable[..., Any]) -> Callable[..., Any]:
      """ Common story options """
  
      options = [
          click.option(
              '--implemented', is_flag=True,
              help='Implemented stories only.'),
          click.option(
              '--unimplemented', is_flag=True,
              help='Unimplemented stories only.'),
          click.option(
              '--verified', is_flag=True,
              help='Stories verified by tests.'),
          click.option(
              '--unverified', is_flag=True,
              help='Stories not verified by tests.'),
          click.option(
              '--documented', is_flag=True,
              help='Documented stories only.'),
          click.option(
              '--undocumented', is_flag=True,
              help='Undocumented stories only.'),
          click.option(
              '--covered', is_flag=True,
              help='Covered stories only.'),
          click.option(
              '--uncovered', is_flag=True,
              help='Uncovered stories only.'),
          ]
  
      for option in reversed(options):
          function = option(function)
      return function
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Main
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  @click.group(invoke_without_command=True, cls=CustomGroup)
  @click.pass_context
  @click.option(
      '-r', '--root', metavar='PATH', show_default=True,
      help="Path to the tree root, '.' by default.")
  @click.option(
      '-c', '--context', metavar='DATA', multiple=True,
      help='Set the fmf context. Use KEY=VAL or KEY=VAL1,VAL2... format '
           'to define individual dimensions or the @FILE notation to load data '
           'from provided yaml file. Can be specified multiple times. ')
  @verbose_debug_quiet
  @click.option(
      '--version', is_flag=True,
      help='Show tmt version and commit hash.')
  def main(
          click_contex: click.core.Context,
          root: str,
          context: List[str],
          **kwargs: Any) -> None:
      """ Test Management Tool """
      # Show current tmt version and exit
      if kwargs.get('version'):
          print(f"tmt version: {tmt.__version__}")
          raise SystemExit(0)
  
      # Disable coloring if NO_COLOR is set
      if 'NO_COLOR' in os.environ:
          click_contex.color = False
  
      # Save click context and fmf context for future use
      tmt.utils.Common._save_context(click_contex)
  
      # Initialize metadata tree (from given path or current directory)
      tree = tmt.Tree(root or os.curdir)
  
      # TODO: context object details need checks
      click_contex.obj = ContextObject(
          common=tmt.utils.Common(),
          fmf_context=tmt.utils.context_to_dict(context),
          steps=set(),
          tree=tree
          )
  
      # Show overview of available tests, plans and stories
      if click_contex.invoked_subcommand is None:
          tmt.Test.overview(tree)
          tmt.Plan.overview(tree)
          tmt.Story.overview(tree)
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Run
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  @main.group(chain=True, invoke_without_command=True, cls=CustomGroup)
  @click.pass_context
  @click.option(
      '-i', '--id', 'id_', help='Run id (name or directory path).', metavar="ID")
  @click.option(
      '-l', '--last', help='Execute the last run once again.', is_flag=True)
  @click.option(
      '-r', '--rm', '--remove', 'remove', is_flag=True,
      help='Remove the workdir when test run is finished.')
  @click.option(
      '--scratch', is_flag=True,
      help='Remove the run workdir before executing to start from scratch.')
  @click.option(
      '--follow', is_flag=True,
      help='Output the logfile as it grows.')
  @click.option(
      '-a', '--all', help='Run all steps, customize some.', is_flag=True)
  @click.option(
      '-u', '--until', type=click.Choice(tmt.steps.STEPS), metavar='STEP',
      help='Enable given step and all preceding steps.')
  @click.option(
      '-s', '--since', type=click.Choice(tmt.steps.STEPS), metavar='STEP',
      help='Enable given step and all following steps.')
  @click.option(
      '-A', '--after', type=click.Choice(tmt.steps.STEPS), metavar='STEP',
      help='Enable all steps after the given one.')
  @click.option(
      '-B', '--before', type=click.Choice(tmt.steps.STEPS), metavar='STEP',
      help='Enable all steps before the given one.')
  @click.option(
      '-S', '--skip', type=click.Choice(tmt.steps.STEPS), metavar='STEP',
      help='Skip given step(s) during test run execution.', multiple=True)
  @click.option(
      '-e', '--environment', metavar='KEY=VALUE|@FILE', multiple=True,
      help='Set environment variable. Can be specified multiple times. The '
           '"@" prefix marks a file to load (yaml or dotenv formats supported).')
  @click.option(
      '--environment-file', metavar='FILE|URL', multiple=True,
      help='Set environment variables from file or url (yaml or dotenv formats '
           'are supported). Can be specified multiple times.')
  @verbose_debug_quiet
  @force_dry
  def run(context: click.core.Context, id_: str, **kwargs: Any) -> None:
      """ Run test steps. """
      # Initialize
      run = tmt.Run(id_, context.obj.tree, context=context)
      context.obj.run = run
  
  
  # Steps options
  run.add_command(tmt.steps.discover.DiscoverPlugin.command())
  run.add_command(tmt.steps.provision.ProvisionPlugin.command())
  run.add_command(tmt.steps.prepare.PreparePlugin.command())
  run.add_command(tmt.steps.execute.ExecutePlugin.command())
  run.add_command(tmt.steps.report.ReportPlugin.command())
  run.add_command(tmt.steps.finish.FinishPlugin.command())
  run.add_command(tmt.steps.Login.command())
  run.add_command(tmt.steps.Reboot.command())
  
  
  @run.command(name='plans')
  @click.pass_context
  @click.option(
      '-n', '--name', 'names', metavar='[REGEXP|.]', multiple=True,
      help="Regular expression to match plan name or '.' for current directory.")
  @click.option(
      '-f', '--filter', 'filters', metavar='FILTER', multiple=True,
      help="Apply advanced filter (see 'pydoc fmf.filter').")
  @click.option(
      '-c', '--condition', 'conditions', metavar="EXPR", multiple=True,
      help="Use arbitrary Python expression for filtering.")
  @click.option(
      '--link', 'links', metavar="RELATION:TARGET", multiple=True,
      help="Filter by linked objects (regular expressions are "
           "supported for both relation and target).")
  @click.option(
      '--default', is_flag=True,
      help="Use default plans even if others are available.")
  @verbose_debug_quiet
  def run_plans(context: click.core.Context, **kwargs: Any) -> None:
      """
      Select plans which should be executed.
  
      Regular expression can be used to filter plans by name.
      Use '.' to select plans under the current working directory.
      """
      tmt.base.Plan._save_context(context)
  
  
  @run.command(name='tests')
  @click.pass_context
  @click.option(
      '-n', '--name', 'names', metavar='[REGEXP|.]', multiple=True,
      help="Regular expression to match test name or '.' for current directory.")
  @click.option(
      '-f', '--filter', 'filters', metavar='FILTER', multiple=True,
      help="Apply advanced filter (see 'pydoc fmf.filter').")
  @click.option(
      '-c', '--condition', 'conditions', metavar="EXPR", multiple=True,
      help="Use arbitrary Python expression for filtering.")
  @click.option(
      '--link', 'links', metavar="RELATION:TARGET", multiple=True,
      help="Filter by linked objects (regular expressions are "
           "supported for both relation and target).")
  @verbose_debug_quiet
  def run_tests(context: click.core.Context, **kwargs: Any) -> None:
      """
      Select tests which should be executed.
  
      Regular expression can be used to filter tests by name.
      Use '.' to select tests under the current working directory.
      """
      tmt.base.Test._save_context(context)
  
  
  # FIXME: click 8.0 renamed resultcallback to result_callback. The former
  #        name will be removed in click 8.1. However, click 8.0 will not
  #        be added to F33 and F34. Get rid of this workaround once
  #        all Fedora + EPEL releases have click 8.0 or newer available.
  run_callback = run.result_callback
  if run_callback is None:
      run_callback = run.resultcallback
  
  
  # TODO: commands is unknown, needs revisit
  @run_callback()  # type: ignore[misc]
  @click.pass_context
  def finito(
          click_context: click.core.Context,
          commands: Any,
          *args: Any,
          **kwargs: Any) -> None:
      """ Run tests if run defined """
      if hasattr(click_context.obj, 'run'):
          click_context.obj.run.go()
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Test
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  @main.group(invoke_without_command=True, cls=CustomGroup)
  @click.pass_context
  @verbose_debug_quiet
  def tests(context: click.core.Context, **kwargs: Any) -> None:
      """
      Manage tests (L1 metadata).
  
      Check available tests, inspect their metadata.
      Convert old metadata into the new fmf format.
      """
  
      # Show overview of available tests
      if context.invoked_subcommand is None:
          tmt.Test.overview(context.obj.tree)
  
  
  @tests.command(name='ls')
  @click.pass_context
  @name_filter_condition
  @verbose_debug_quiet
  def tests_ls(context: click.core.Context, **kwargs: Any) -> None:
      """
      List available tests.
  
      Regular expression can be used to filter tests by name.
      Use '.' to select tests under the current working directory.
      """
      tmt.Test._save_context(context)
      for test in context.obj.tree.tests():
          test.ls()
  
  
  @tests.command(name='show')
  @click.pass_context
  @name_filter_condition
  @verbose_debug_quiet
  def tests_show(context: click.core.Context, **kwargs: Any) -> None:
      """
      Show test details.
  
      Regular expression can be used to filter tests by name.
      Use '.' to select tests under the current working directory.
      """
      tmt.Test._save_context(context)
      for test in context.obj.tree.tests():
          test.show()
          echo()
  
  
  @tests.command(name='lint')
  @click.pass_context
  @name_filter_condition
  @source
  @fix
  @verbose_debug_quiet
  def tests_lint(context: click.core.Context, **kwargs: Any) -> None:
      """
      Check tests against the L1 metadata specification.
  
      Regular expression can be used to filter tests for linting.
      Use '.' to select tests under the current working directory.
      """
      # FIXME: Workaround https://github.com/pallets/click/pull/1840 for click 7
      context.params.update(**kwargs)
      tmt.Test._save_context(context)
      exit_code = 0
      for test in context.obj.tree.tests():
          if not test.lint():
              exit_code = 1
          echo()
      raise SystemExit(exit_code)
  
  
  _test_templates = listed(tmt.templates.TEST, join='or')
  
  
  @tests.command(name='create')
  @click.pass_context
  @click.argument('name')
  @click.option(
      '-t', '--template', metavar='TEMPLATE',
      help='Test template ({}).'.format(_test_templates),
      prompt='Template ({})'.format(_test_templates))
  @verbose_debug_quiet
  @force_dry
  def tests_create(
          context: click.core.Context,
          name: str,
          template: str,
          force: bool,
          **kwargs: Any) -> None:
      """
      Create a new test based on given template.
  
      Specify directory name or use '.' to create tests under the
      current working directory.
      """
      tmt.Test._save_context(context)
      tmt.Test.create(name, template, context.obj.tree.root, force)
  
  
  @tests.command(name='import')
  @click.pass_context
  @click.argument('paths', nargs=-1, metavar='[PATH]...')
  @click.option(
      '--nitrate / --no-nitrate', default=True,
      help='Import test metadata from Nitrate')
  @click.option(
      '--purpose / --no-purpose', default=True,
      help='Migrate description from PURPOSE file')
  @click.option(
      '--makefile / --no-makefile', default=True,
      help='Convert Beaker Makefile metadata')
  @click.option(
      '--restraint / --no-restraint', default=False,
      help='Convert restraint metadata file')
  @click.option(
      '--general / --no-general', default=True,
      help='Detect components from linked nitrate general plans '
           '(overrides Makefile/restraint component).')
  @click.option(
      '--type', 'types', metavar='TYPE', default=['multihost'], multiple=True,
      show_default=True,
      help="Convert selected types from Makefile into tags. "
           "Use 'all' to convert all detected types.")
  @click.option(
      '--disabled', default=False, is_flag=True,
      help='Import disabled test cases from Nitrate as well.')
  @click.option(
      '--manual', default=False, is_flag=True,
      help='Import manual test cases from Nitrate.')
  @click.option(
      '--plan', metavar='PLAN', type=int,
      help='Identifier of test plan from which to import manual test cases.')
  @click.option(
      '--case', metavar='CASE', type=int,
      help='Identifier of manual test case to be imported.')
  @click.option(
      '--with-script', default=False, is_flag=True,
      help='Import manual cases with non-empty script field in Nitrate.')
  @verbose_debug_quiet
  @force_dry
  def tests_import(
          context: click.core.Context,
          paths: List[str],
          makefile: bool,
          restraint: bool,
          general: bool,
          types: List[str],
          nitrate: bool,
          purpose: bool,
          disabled: bool,
          manual: bool,
          plan: int,
          case: int,
          with_script: bool,
          **kwargs: Any) -> None:
      """
      Import old test metadata into the new fmf format.
  
      Accepts one or more directories where old metadata are stored.
      By default all available sources and current directory are used.
      The following test metadata are converted for each source:
  
      \b
      makefile ..... summary, component, duration, require
      restraint .... name, description, entry_point, owner,
                     max_time, repoRequires
      purpose ...... description
      nitrate ...... contact, component, tag,
                     environment, relevancy, enabled
      """
      tmt.Test._save_context(context)
  
      if manual:
          if not (case or plan):
              raise tmt.utils.GeneralError(
                  "Option --case or --plan is mandatory when using --manual.")
          else:
              tmt.convert.read_manual(plan, case, disabled, with_script)
              return
  
      if not paths:
          paths = ['.']
      for path in paths:
          # Make sure we've got a real directory
          path = os.path.realpath(path)
          if not os.path.isdir(path):
              raise tmt.utils.GeneralError(
                  "Path '{0}' is not a directory.".format(path))
          # Gather old metadata and store them as fmf
          common, individual = tmt.convert.read(
              path, makefile, restraint, nitrate, purpose, disabled, types,
              general)
          # Add path to common metadata if there are virtual test cases
          if individual:
              root = fmf.Tree(path).root
              common['path'] = os.path.join('/', os.path.relpath(path, root))
          # Store common metadata
          common_path = os.path.join(path, 'main.fmf')
          tmt.convert.write(common_path, common)
          # Store individual data (as virtual tests)
          for testcase in individual:
              testcase_path = os.path.join(
                  path, str(testcase['extra-nitrate']) + '.fmf')
              tmt.convert.write(testcase_path, testcase)
          # Adjust runtest.sh content and permission if needed
          tmt.convert.adjust_runtest(os.path.join(path, 'runtest.sh'))
  
  
  @tests.command(name='export')
  @click.pass_context
  @name_filter_condition_long
  @click.option(
      '-h', '--how', metavar='METHOD',
      help='Use specified method for export (nitrate or polarion).')
  @click.option(
      '--nitrate', is_flag=True,
      help="Export test metadata to Nitrate, deprecated by '--how nitrate'.")
  @click.option(
      '--project-id', help='Use specific Polarion project ID.')
  @click.option(
      '--link-polarion / --no-link-polarion', default=True,
      help='Add Polarion link to fmf testcase metadata')
  @click.option(
      '--bugzilla', is_flag=True,
      help="Link Nitrate case to Bugzilla specified in the 'link' attribute "
           "with the relation 'verifies'.")
  @click.option(
      '--ignore-git-validation', is_flag=True,
      help="Ignore unpublished git changes and export to Nitrate. "
      "The case might not be able to be scheduled!")
  @click.option(
      '--create', is_flag=True,
      help="Create test cases in nitrate if they don't exist.")
  @click.option(
      '--general / --no-general', default=False,
      help="Link Nitrate case to component's General plan. Disabled by default. "
           "Note that this will unlink any previously connected general plans.")
  @click.option(
      '--link-runs / --no-link-runs', default=False,
      help="Link Nitrate case to all open runs of descendant plans of "
           "General plan. Disabled by default. Implies --general option.")
  @click.option(
      '--format', 'format_', default='yaml', show_default=True, metavar='FORMAT',
      help='Output format (yaml or dict).')
  @click.option(
      '--fmf-id', is_flag=True,
      help='Show fmf identifiers instead of test metadata.')
  @click.option(
      '--duplicate / --no-duplicate', default=False, show_default=True,
      help='Allow or prevent creating duplicates in Nitrate by searching for '
           'existing test cases with the same fmf identifier.')
  @click.option(
      '-n', '--dry', is_flag=True,
      help="Run in dry mode. No changes, please.")
  @click.option(
      '-d', '--debug', is_flag=True,
      help='Provide as much debugging details as possible.')
  def tests_export(
          context: click.core.Context,
          format_: str,
          how: str,
          nitrate: bool,
          bugzilla: bool,
          **kwargs: Any) -> None:
      """
      Export test data into the desired format.
  
      Regular expression can be used to filter tests by name.
      Use '.' to select tests under the current working directory.
      """
      tmt.Test._save_context(context)
      if nitrate:
          context.obj.common.warn(
              "Option '--nitrate' is deprecated, please use '--how nitrate' instead.")
          how = 'nitrate'
      if bugzilla and not how:
          raise tmt.utils.GeneralError(
              "The --bugzilla option is supported only with --nitrate "
              "or --polarion for now.")
  
      if how == 'nitrate' or how == 'polarion':
          for test in context.obj.tree.tests():
              test.export(format_=how)
      elif format_ in ['dict', 'yaml']:
          keys = None
          if kwargs.get('fmf_id'):
              keys = 'fmf-id'
  
          tests = [test.export(format_='dict', keys=keys) for test in
                   context.obj.tree.tests()]
          if format_ == 'dict':
              echo(tests, nl=False)
          else:
              echo(tmt.utils.dict_to_yaml(tests), nl=False)
      else:
          raise tmt.utils.GeneralError(
              f"Invalid test export format '{format_}'.")
  
  
  @tests.command(name="id")
  @click.pass_context
  @name_filter_condition
  @verbose_debug_quiet
  @force_dry
  def tests_id(context: click.core.Context, **kwargs: Any) -> None:
      """
      Generate a unique id for each selected test.
  
      A new UUID is generated for each test matching the provided
      filter and the value is stored to disk. Existing identifiers
      are kept intact.
      """
      tmt.Test._save_context(context)
      for test in context.obj.tree.tests():
          tmt.identifier.id_command(test.node, "test", dry=kwargs["dry"])
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Plan
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  @main.group(invoke_without_command=True, cls=CustomGroup)
  @click.pass_context
  @verbose_debug_quiet
  def plans(context: click.core.Context, **kwargs: Any) -> None:
      """
      Manage test plans (L2 metadata).
  
      \b
      Search for available plans.
      Explore detailed test step configuration.
      """
      tmt.Plan._save_context(context)
  
      # Show overview of available plans
      if context.invoked_subcommand is None:
          tmt.Plan.overview(context.obj.tree)
  
  
  @plans.command(name='ls')
  @click.pass_context
  @name_filter_condition
  @verbose_debug_quiet
  def plans_ls(context: click.core.Context, **kwargs: Any) -> None:
      """
      List available plans.
  
      Regular expression can be used to filter plans by name.
      Use '.' to select plans under the current working directory.
      """
      tmt.Plan._save_context(context)
      for plan in context.obj.tree.plans():
          plan.ls()
  
  
  @plans.command(name='show')
  @click.pass_context
  @name_filter_condition
  @verbose_debug_quiet
  def plans_show(context: click.core.Context, **kwargs: Any) -> None:
      """
      Show plan details.
  
      Regular expression can be used to filter plans by name.
      Use '.' to select plans under the current working directory.
      """
      tmt.Plan._save_context(context)
      for plan in context.obj.tree.plans():
          plan.show()
          echo()
  
  
  @plans.command(name='lint')
  @click.pass_context
  @name_filter_condition
  @source
  @verbose_debug_quiet
  def plans_lint(context: click.core.Context, **kwargs: Any) -> None:
      """
      Check plans against the L2 metadata specification.
  
      Regular expression can be used to filter plans by name.
      Use '.' to select plans under the current working directory.
      """
      # FIXME: Workaround https://github.com/pallets/click/pull/1840 for click 7
      context.params.update(**kwargs)
      tmt.Plan._save_context(context)
      exit_code = 0
      for plan in context.obj.tree.plans():
          if not plan.lint():
              exit_code = 1
          echo()
      raise SystemExit(exit_code)
  
  
  _plan_templates = listed(tmt.templates.PLAN, join='or')
  
  
  @plans.command(name='create')
  @click.pass_context
  @click.argument('name')
  @click.option(
      '-t', '--template', metavar='TEMPLATE',
      help='Plan template ({}).'.format(_plan_templates),
      prompt='Template ({})'.format(_plan_templates))
  @click.option(
      '--discover', metavar='YAML', multiple=True,
      help='Discover phase content in yaml format.')
  @click.option(
      '--provision', metavar='YAML', multiple=True,
      help='Provision phase content in yaml format.')
  @click.option(
      '--prepare', metavar='YAML', multiple=True,
      help='Prepare phase content in yaml format.')
  @click.option(
      '--execute', metavar='YAML', multiple=True,
      help='Execute phase content in yaml format.')
  @click.option(
      '--report', metavar='YAML', multiple=True,
      help='Report phase content in yaml format.')
  @click.option(
      '--finish', metavar='YAML', multiple=True,
      help='Finish phase content in yaml format.')
  @verbose_debug_quiet
  @force_dry
  def plans_create(
          context: click.core.Context,
          name: str,
          template: str,
          force: bool,
          **kwargs: Any) -> None:
      """ Create a new plan based on given template. """
      tmt.Plan._save_context(context)
      tmt.Plan.create(name, template, context.obj.tree.root, force)
  
  
  @plans.command(name='export')
  @click.pass_context
  @name_filter_condition_long
  @click.option(
      '--format', 'format_', default='yaml', show_default=True, metavar='FORMAT',
      help='Output format.')
  @click.option(
      '-d', '--debug', is_flag=True,
      help='Provide as much debugging details as possible.')
  def plans_export(context: click.core.Context, format_: str, **kwargs: Any) -> None:
      """
      Export plans into desired format.
  
      Regular expression can be used to filter plans by name.
      Use '.' to select plans under the current working directory.
      """
      tmt.Plan._save_context(context)
      plans = [plan.export(format_='dict') for plan in context.obj.tree.plans()]
  
      # Choose proper format
      if format_ == 'dict':
          echo(plans)
      elif format_ == 'yaml':
          echo(tmt.utils.dict_to_yaml(plans))
      else:
          raise tmt.utils.GeneralError(
              f"Invalid plan export format '{format_}'.")
  
  
  @plans.command(name="id")
  @click.pass_context
  @name_filter_condition
  @verbose_debug_quiet
  @force_dry
  def plans_id(context: click.core.Context, **kwargs: Any) -> None:
      """
      Generate a unique id for each selected plan.
  
      A new UUID is generated for each plan matching the provided
      filter and the value is stored to disk. Existing identifiers
      are kept intact.
      """
      tmt.Plan._save_context(context)
      for plan in context.obj.tree.plans():
          tmt.identifier.id_command(plan.node, "plan", dry=kwargs["dry"])
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Story
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  
  @main.group(invoke_without_command=True, cls=CustomGroup)
  @click.pass_context
  @verbose_debug_quiet
  def stories(context: click.core.Context, **kwargs: Any) -> None:
      """
      Manage user stories.
  
      \b
      Check available user stories.
      Explore coverage (test, implementation, documentation).
      """
      tmt.Story._save_context(context)
  
      # Show overview of available stories
      if context.invoked_subcommand is None:
          tmt.Story.overview(context.obj.tree)
  
  
  @stories.command(name='ls')
  @click.pass_context
  @name_filter_condition_long
  @implemented_verified_documented
  @verbose_debug_quiet
  def stories_ls(
          context: click.core.Context,
          implemented: bool,
          verified: bool,
          documented: bool,
          covered: bool,
          unimplemented: bool,
          unverified: bool,
          undocumented: bool,
          uncovered: bool,
          **kwargs: Any) -> None:
      """
      List available stories.
  
      Regular expression can be used to filter stories by name.
      Use '.' to select stories under the current working directory.
      """
      tmt.Story._save_context(context)
      for story in context.obj.tree.stories():
          if story._match(implemented, verified, documented, covered,
                          unimplemented, unverified, undocumented, uncovered):
              story.ls()
  
  
  @stories.command(name='show')
  @click.pass_context
  @name_filter_condition_long
  @implemented_verified_documented
  @verbose_debug_quiet
  def stories_show(
          context: click.core.Context,
          implemented: bool,
          verified: bool,
          documented: bool,
          covered: bool,
          unimplemented: bool,
          unverified: bool,
          undocumented: bool,
          uncovered: bool,
          **kwargs: Any) -> None:
      """
      Show story details.
  
      Regular expression can be used to filter stories by name.
      Use '.' to select stories under the current working directory.
      """
      tmt.Story._save_context(context)
      for story in context.obj.tree.stories():
          if story._match(implemented, verified, documented, covered,
                          unimplemented, unverified, undocumented, uncovered):
              story.show()
              echo()
  
  
  _story_templates = listed(tmt.templates.STORY, join='or')
  
  
  @stories.command(name='create')
  @click.pass_context
  @click.argument('name')
  @click.option(
      '-t', '--template', metavar='TEMPLATE',
      prompt='Template ({})'.format(_story_templates),
      help='Story template ({}).'.format(_story_templates))
  @verbose_debug_quiet
  @force_dry
  def stories_create(
          context: click.core.Context,
          name: str,
          template: str,
          force: bool,
          **kwargs: Any) -> None:
      """ Create a new story based on given template. """
      tmt.Story._save_context(context)
      tmt.base.Story.create(name, template, context.obj.tree.root, force)
  
  
  @stories.command(name='coverage')
  @click.option(
      '--docs', is_flag=True, help='Show docs coverage.')
  @click.option(
      '--test', is_flag=True, help='Show test coverage.')
  @click.option(
      '--code', is_flag=True, help='Show code coverage.')
  @click.pass_context
  @name_filter_condition_long
  @implemented_verified_documented
  @verbose_debug_quiet
  def stories_coverage(
          context: click.core.Context,
          code: bool,
          test: bool,
          docs: bool,
          implemented: bool,
          verified: bool,
          documented: bool,
          covered: bool,
          unimplemented: bool,
          unverified: bool,
          undocumented: bool,
          uncovered: bool,
          **kwargs: Any) -> None:
      """
      Show code, test and docs coverage for given stories.
  
      Regular expression can be used to filter stories by name.
      Use '.' to select stories under the current working directory.
      """
      tmt.Story._save_context(context)
  
      def headfoot(text: str) -> None:
          """ Format simple header/footer """
          echo(style(text.rjust(4) + ' ', fg='blue'), nl=False)
  
      header = False
      total = code_coverage = test_coverage = docs_coverage = 0
      if not any([code, test, docs]):
          code = test = docs = True
      for story in context.obj.tree.stories():
          # Check conditions
          if not story._match(
                  implemented, verified, documented, covered, unimplemented,
                  unverified, undocumented, uncovered):
              continue
          # Show header once
          if not header:
              if code:
                  headfoot('code')
              if test:
                  headfoot('test')
              if docs:
                  headfoot('docs')
              headfoot('story')
              echo()
              header = True
          # Show individual stats
          status = story.coverage(code, test, docs)
          total += 1
          code_coverage += status[0]
          test_coverage += status[1]
          docs_coverage += status[2]
      # Summary
      if not total:
          return
      if code:
          headfoot('{}%'.format(round(100 * code_coverage / total)))
      if test:
          headfoot('{}%'.format(round(100 * test_coverage / total)))
      if docs:
          headfoot('{}%'.format(round(100 * docs_coverage / total)))
      headfoot('from {}'.format(listed(total, 'story')))
      echo()
  
  
  @stories.command(name='export')
  @click.pass_context
  @name_filter_condition_long
  @implemented_verified_documented
  @click.option(
      '--format', 'format_', default='rst', show_default=True, metavar='FORMAT',
      help='Output format.')
  @click.option(
      '-d', '--debug', is_flag=True,
      help='Provide as much debugging details as possible.')
  def stories_export(
          context: click.core.Context,
          format_: str,
          implemented: bool,
          verified: bool,
          documented: bool,
          covered: bool,
          unimplemented: bool,
          unverified: bool,
          undocumented: bool,
          uncovered: bool,
          **kwargs: Any) -> None:
      """
      Export selected stories into desired format.
  
      Regular expression can be used to filter stories by name.
      Use '.' to select stories under the current working directory.
      """
      tmt.Story._save_context(context)
  
      for story in context.obj.tree.stories(whole=True):
          if story._match(implemented, verified, documented, covered,
                          unimplemented, unverified, undocumented, uncovered):
              echo(story.export(format_))
  
  
  @stories.command(name='lint')
  @click.pass_context
  @name_filter_condition
  @source
  @verbose_debug_quiet
  def stories_lint(context: click.core.Context, **kwargs: Any) -> None:
      """
      Check stories against the L3 metadata specification.
  
      Regular expression can be used to filter stories by name.
      Use '.' to select stories under the current working directory.
      """
      # FIXME: Workaround https://github.com/pallets/click/pull/1840 for click 7
      context.params.update(**kwargs)
      tmt.Story._save_context(context)
      exit_code = 0
      for story in context.obj.tree.stories():
          if not story.lint():
              exit_code = 1
          echo()
      raise SystemExit(exit_code)
  
  
  @stories.command(name="id")
  @click.pass_context
  @name_filter_condition_long
  @implemented_verified_documented
  @verbose_debug_quiet
  @force_dry
  def stories_id(
          context: click.core.Context,
          implemented: bool,
          verified: bool,
          documented: bool,
          covered: bool,
          unimplemented: bool,
          unverified: bool,
          undocumented: bool,
          uncovered: bool,
          **kwargs: Any) -> None:
      """
      Generate a unique id for each selected story.
  
      A new UUID is generated for each story matching the provided
      filter and the value is stored to disk. Existing identifiers
      are kept intact.
      """
      tmt.Story._save_context(context)
      for story in context.obj.tree.stories():
          if story._match(implemented, verified, documented, covered,
                          unimplemented, unverified, undocumented, uncovered):
              tmt.identifier.id_command(story.node, "story", dry=kwargs["dry"])
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Init
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  @main.command()
  @click.pass_context
  @click.argument('path', default='.')
  @click.option(
      '-t', '--template', default='empty', metavar='TEMPLATE',
      type=click.Choice(['empty'] + tmt.templates.INIT_TEMPLATES),
      help='Template ({}).'.format(
          listed(tmt.templates.INIT_TEMPLATES, join='or')))
  @verbose_debug_quiet
  @force_dry
  def init(
          context: click.core.Context,
          path: str,
          template: str,
          force: bool,
          **kwargs: Any) -> None:
      """
      Initialize a new tmt tree.
  
      By default tree is created in the current directory.
      Provide a PATH to create it in a different location.
  
      \b
      A tree can be optionally populated with example metadata:
      * 'mini' template contains a minimal plan and no tests,
      * 'base' template contains a plan and a beakerlib test,
      * 'full' template contains a 'full' story, an 'full' plan and a shell test.
      """
  
      tmt.base.Tree._save_context(context)
      tmt.base.Tree.init(path, template, force, **kwargs)
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Status
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  @main.command()
  @click.pass_context
  @workdir_root
  @click.option(
      '-i', '--id', metavar="ID",
      help='Run id (name or directory path) to show status of.')
  @click.option(
      '--abandoned', is_flag=True, default=False,
      help='List runs which have provision step completed but finish step '
           'not yet done.')
  @click.option(
      '--active', is_flag=True, default=False,
      help='List runs where at least one of the enabled steps has not '
           'been finished.')
  @click.option(
      '--finished', is_flag=True, default=False,
      help='List all runs which have all enabled steps completed.')
  @verbose_debug_quiet
  def status(
          context: click.core.Context,
          workdir_root: str,
          abandoned: bool,
          active: bool,
          finished: bool,
          **kwargs: Any) -> None:
      """
      Show status of runs.
  
      Lists past runs in the given directory filtered using options.
      /var/tmp/tmt is used by default.
  
      By default, status of the whole runs is listed. With more
      verbosity (-v), status of every plan is shown. By default,
      the last completed step is displayed, 'done' is used when
      all enabled steps are completed. Status of every step is
      displayed with the most verbosity (-vv).
  
      """
      if [abandoned, active, finished].count(True) > 1:
          raise tmt.utils.GeneralError(
              "Options --abandoned, --active and --finished cannot be "
              "used together.")
      if not os.path.exists(workdir_root):
          raise tmt.utils.GeneralError(f"Path '{workdir_root}' doesn't exist.")
      status_obj = tmt.Status(context=context)
      status_obj.show()
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Clean
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  dry = tmt.options.force_dry[1]
  
  
  @main.group(chain=True, invoke_without_command=True, cls=CustomGroup)
  @click.pass_context
  @verbose_debug_quiet
  @dry
  def clean(context: click.core.Context, **kwargs: Any) -> None:
      """
      Clean workdirs, guests or images.
  
      Without any command, clean everything, stop the guests, remove
      all runs and then remove all images. Search for runs in
      /var/tmp/tmt, if runs are stored elsewhere, the path to them can
      be set using a subcommand (either runs or guests subcommand).
  
      The subcommands can be chained, the order of cleaning is always
      the same, irrespective of the order on the command line. First, all
      the guests are cleaned, followed by runs and images.
      """
      echo(style('clean', fg='red'))
      clean_obj = tmt.Clean(parent=context.obj.common, context=context)
      context.obj.clean = clean_obj
      context.obj.clean_partials = collections.defaultdict(list)
      exit_code = 0
      if context.invoked_subcommand is None:
          # Set path to default
          context.params['workdir_root'] = tmt.utils.WORKDIR_ROOT
          # Create another level to the hierarchy so that logging indent is
          # consistent between the command and subcommands
          clean_obj = tmt.Clean(parent=clean_obj, context=context)
          if os.path.exists(tmt.utils.WORKDIR_ROOT):
              if not clean_obj.guests():
                  exit_code = 1
              if not clean_obj.runs():
                  exit_code = 1
          else:
              clean_obj.warn(
                  f"Directory '{tmt.utils.WORKDIR_ROOT}' does not exist, "
                  f"skipping guest and run cleanup.")
          clean_obj.images()
          raise SystemExit(exit_code)
  
  
  # FIXME: Click deprecation, see function finito for more info
  clean_callback = clean.result_callback
  if clean_callback is None:
      clean_callback = clean.resultcallback
  
  
  # TODO: commands is unknown, needs revisit
  @clean_callback()  # type: ignore[misc]
  @click.pass_context
  def perform_clean(
          click_context: click.core.Context,
          commands: Any,
          *args: Any,
          **kwargs: Any) -> None:
      """
      Perform clean actions in the correct order.
  
      We need to ensure that guests are always cleaned before the run workdirs
      even if the user specified them in reverse order.
      """
      clean_order = ("guests", "runs", "images")
      exit_code = 0
      for phase in clean_order:
          for partial in click_context.obj.clean_partials[phase]:
              if not partial():
                  exit_code = 1
      raise SystemExit(exit_code)
  
  
  @clean.command(name='runs')
  @click.pass_context
  @workdir_root
  @click.option(
      '-l', '--last', is_flag=True, help='Clean the workdir of the last run.')
  @click.option(
      '-i', '--id', 'id_', metavar="ID",
      help='Run id (name or directory path) to clean workdir of.')
  @click.option(
      '-k', '--keep', type=int,
      help='The number of latest workdirs to keep, clean the rest.')
  @verbose_debug_quiet
  @dry
  def clean_runs(
          context: click.core.Context,
          workdir_root: str,
          last: bool,
          id_: str,
          keep: int,
          **kwargs: Any) -> None:
      """
      Clean workdirs of past runs.
  
      Remove all runs in '/var/tmp/tmt' by default.
      """
      defined = [last is True, id_ is not None, keep is not None]
      if defined.count(True) > 1:
          raise tmt.utils.GeneralError(
              "Options --last, --id and --keep cannot be used together.")
      if keep is not None and keep < 0:
          raise tmt.utils.GeneralError("--keep must not be a negative number.")
      if not os.path.exists(workdir_root):
          raise tmt.utils.GeneralError(f"Path '{workdir_root}' doesn't exist.")
      clean_obj = tmt.Clean(parent=context.obj.clean, context=context)
      context.obj.clean_partials["runs"].append(clean_obj.runs)
  
  
  @clean.command(name='guests')
  @click.pass_context
  @workdir_root
  @click.option(
      '-l', '--last', is_flag=True, help='Stop the guest of the last run.')
  @click.option(
      '-i', '--id', 'id_', metavar="ID",
      help='Run id (name or directory path) to stop the guest of.')
  @click.option(
      '-h', '--how', metavar='METHOD',
      help='Stop guests of the specified provision method.')
  @verbose_debug_quiet
  @dry
  def clean_guests(
          context: click.core.Context,
          workdir_root: str,
          last: bool,
          id_: int,
          **kwargs: Any) -> None:
      """
      Stop running guests of runs.
  
      Stop guests of all runs in '/var/tmp/tmt' by default.
      """
      if last and id_ is not None:
          raise tmt.utils.GeneralError(
              "Options --last and --id cannot be used together.")
      if not os.path.exists(workdir_root):
          raise tmt.utils.GeneralError(f"Path '{workdir_root}' doesn't exist.")
      clean_obj = tmt.Clean(parent=context.obj.clean, context=context)
      context.obj.clean_partials["guests"].append(clean_obj.guests)
  
  
  @clean.command(name='images')
  @click.pass_context
  @verbose_debug_quiet
  @dry
  def clean_images(context: click.core.Context, **kwargs: Any) -> None:
      """
      Remove images of supported provision methods.
  
      Currently supported methods are:
       - testcloud
      """
      # FIXME: If there are more provision methods supporting this,
      #        we should add options to specify which provision should be
      #        cleaned, similarly to guests.
      clean_obj = tmt.Clean(parent=context.obj.clean, context=context)
      context.obj.clean_partials["images"].append(clean_obj.images)
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Lint
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  
  @main.command(name='lint')
  @click.pass_context
  @name_filter_condition
  @source
  @fix
  @verbose_debug_quiet
  def lint(context: click.core.Context, **kwargs: Any) -> None:
      """
      Check all the present metadata against the specification.
  
      Combines all the partial linting (tests, plans and stories)
      into one command. Options are applied to all parts of the lint.
  
      Regular expression can be used to filter metadata by name.
      Use '.' to select tests, plans and stories under the current
      working directory.
      """
      exit_code = 0
      for command in (tests_lint, plans_lint, stories_lint):
          try:
              context.forward(command)
          except SystemExit as e:
              exit_code |= e.code
      raise SystemExit(exit_code)
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Setup
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  
  @main.group(cls=CustomGroup)
  @click.pass_context
  def setup(context: click.core.Context, **kwargs: Any) -> None:
      """
      Setup the environment for working with tmt.
      """
  
  
  @setup.group(cls=CustomGroup)
  @click.pass_context
  def completion(context: click.core.Context, **kwargs: Any) -> None:
      """
      Setup shell completions.
  
      By default, these commands only write a shell script to the output
      which can then be sourced from the shell's configuration file. Use
      the '--install' option to store and enable the configuration
      permanently.
      """
  
  
  COMPLETE_VARIABLE = '_TMT_COMPLETE'
  COMPLETE_SCRIPT = 'tmt-complete'
  
  
  def setup_completion(shell: str, install: bool) -> None:
      """ Setup completion based on the shell """
      config = tmt.utils.Config()
      # Fish gets installed into its special location where it is automatically
      # loaded.
      if shell == 'fish':
          script = os.path.expanduser('~/.config/fish/completions/tmt.fish')
      # Bash and zsh get installed to tmt's config directory.
      else:
          script = os.path.join(config.path, f'{COMPLETE_SCRIPT}.{shell}')
  
      out = open(script, 'w') if install else sys.stdout
      subprocess.run(f'{COMPLETE_VARIABLE}={shell}_source tmt',
                     shell=True, stdout=out)
  
      if install:
          out.close()
          # If requested, modify .bashrc or .zshrc
          if shell != 'fish':
              config_path = os.path.expanduser(f'~/.{shell}rc')
              with open(config_path, 'a') as shell_config:
                  shell_config.write('\n# Generated by tmt\n')
                  shell_config.write(f'source {script}')
  
  
  @completion.command(name='bash')
  @click.pass_context
  @click.option(
      '--install', '-i', 'install', is_flag=True,
      help="Persistently store the script to tmt's configuration directory "
           "and set it up by modifying '~/.bashrc'.")
  def completion_bash(context: click.core.Context, install: bool, **kwargs: Any) -> None:
      """
      Setup shell completions for bash.
      """
      setup_completion('bash', install)
  
  
  @completion.command(name='zsh')
  @click.pass_context
  @click.option(
      '--install', '-i', 'install', is_flag=True,
      help="Persistently store the script to tmt's configuration directory "
           "and set it up by modifying '~/.zshrc'.")
  def completion_zsh(context: click.core.Context, install: bool, **kwargs: Any) -> None:
      """
      Setup shell completions for zsh.
      """
      setup_completion('zsh', install)
  
  
  @completion.command(name='fish')
  @click.pass_context
  @click.option(
      '--install', '-i', 'install', is_flag=True,
      help="Persistently store the script to "
           "'~/.config/fish/completions/tmt.fish'.")
  def completion_fish(context: click.core.Context, install: bool, **kwargs: Any) -> None:
      """
      Setup shell completions for fish.
      """
      setup_completion('fish', install)
  
files: tmt/cli.py
files: # coding: utf-8
  
  """ Command line interface for the Test Management Tool """
  
  import collections
  import dataclasses
  import os
  import subprocess
  import sys
  from typing import TYPE_CHECKING, Any, Callable, List, Optional, Set
  
  import click
  import fmf
  from click import echo, style
  from fmf.utils import listed
  
  import tmt
  import tmt.convert
  import tmt.export
  import tmt.identifier
  import tmt.options
  import tmt.plugins
  import tmt.steps
  import tmt.templates
  import tmt.utils
  
  if TYPE_CHECKING:
      import tmt.steps.discover
      import tmt.steps.execute
  
  # Explore available plugins (need to detect all supported methods first)
  tmt.plugins.explore()
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Click Context Object Container
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  
  @dataclasses.dataclass
  class ContextObject:
      """ Click Context Object Container """
      common: tmt.utils.Common
      fmf_context: tmt.utils.FmfContextType
      steps: Set[tmt.steps.Step]
      tree: tmt.Tree
      clean: Optional[tmt.Clean] = None
      run: Optional[tmt.Run] = None
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Custom Group
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  class CustomGroup(click.Group):
      """ Custom Click Group """
  
      def list_commands(self, context: click.core.Context) -> List[str]:
          """ Prevent alphabetical sorting """
          return list(self.commands.keys())
  
      def get_command(self, context: click.core.Context, cmd_name: str) -> Optional[click.Command]:
          """ Allow command shortening """
          # Backward-compatible 'test convert' (just temporary for now FIXME)
          cmd_name = cmd_name.replace('convert', 'import')
          # Support both story & stories
          cmd_name = cmd_name.replace('story', 'stories')
          found = click.Group.get_command(self, context, cmd_name)
          if found is not None:
              return found
          matches = [command for command in self.list_commands(context)
                     if command.startswith(cmd_name)]
          if not matches:
              return None
          elif len(matches) == 1:
              return click.Group.get_command(self, context, matches[0])
          context.fail('Did you mean {}?'.format(
              listed(sorted(matches), join='or')))
          return None
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Common Options
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  def verbose_debug_quiet(function: Callable[..., Any]) -> Callable[..., Any]:
      """ Verbose, debug and quiet output """
      for option in reversed(tmt.options.verbose_debug_quiet):
          function = option(function)
      return function
  
  
  def force_dry(function: Callable[..., Any]) -> Callable[..., Any]:
      """ Force and dry actions """
      for option in reversed(tmt.options.force_dry):
          function = option(function)
      return function
  
  
  def fix(function: Callable[..., Any]) -> Callable[..., Any]:
      """ Fix action """
      function = tmt.options.fix(function)
      return function
  
  
  def workdir_root(function: Callable[..., Any]) -> Callable[..., Any]:
      """ Where tmt workdirs are to be found """
      return tmt.options.workdir_root(function)
  
  
  def name_filter_condition(function: Callable[..., Any]) -> Callable[..., Any]:
      """ Common filter options (short & long) """
      options = [
          click.argument(
              'names', nargs=-1, metavar='[REGEXP|.]'),
          click.option(
              '-f', '--filter', 'filters', metavar='FILTER', multiple=True,
              help="Apply advanced filter (see 'pydoc fmf.filter')."),
          click.option(
              '-c', '--condition', 'conditions', metavar="EXPR", multiple=True,
              help="Use arbitrary Python expression for filtering."),
          click.option(
              '--enabled', is_flag=True,
              help="Show only enabled tests, plans or stories."),
          click.option(
              '--disabled', is_flag=True,
              help="Show only disabled tests, plans or stories."),
          click.option(
              '--link', 'links', metavar="RELATION:TARGET", multiple=True,
              help="Filter by linked objects (regular expressions are "
                   "supported for both relation and target)."),
          click.option(
              '-x', '--exclude', 'exclude', metavar='[REGEXP]', multiple=True,
              help="Exclude a regular expression from search result."),
          ]
  
      for option in reversed(options):
          function = option(function)
      return function
  
  
  def source(function: Callable[..., Any]) -> Callable[..., Any]:
      """ Option to select fmf objects by their sources """
      return click.option(
          '--source', is_flag=True, help="Select by fmf source file names instead of object names."
          )(function)
  
  
  def name_filter_condition_long(function: Callable[..., Any]) -> Callable[..., Any]:
      """ Common filter options (long only) """
      options = [
          click.argument(
              'names', nargs=-1, metavar='[REGEXP|.]'),
          click.option(
              '--filter', 'filters', metavar='FILTER', multiple=True,
              help="Apply advanced filter (see 'pydoc fmf.filter')."),
          click.option(
              '--condition', 'conditions', metavar="EXPR", multiple=True,
              help="Use arbitrary Python expression for filtering."),
          click.option(
              '--enabled', is_flag=True,
              help="Show only enabled tests, plans or stories."),
          click.option(
              '--disabled', is_flag=True,
              help="Show only disabled tests, plans or stories."),
          click.option(
              '--link', 'links', metavar="RELATION:TARGET", multiple=True,
              help="Filter by linked objects (regular expressions are "
                   "supported for both relation and target)."),
          click.option(
              '--exclude', 'exclude', metavar='[REGEXP]', multiple=True,
              help="Exclude a regular expression from search result."),
          ]
  
      for option in reversed(options):
          function = option(function)
      return function
  
  
  def implemented_verified_documented(function: Callable[..., Any]) -> Callable[..., Any]:
      """ Common story options """
  
      options = [
          click.option(
              '--implemented', is_flag=True,
              help='Implemented stories only.'),
          click.option(
              '--unimplemented', is_flag=True,
              help='Unimplemented stories only.'),
          click.option(
              '--verified', is_flag=True,
              help='Stories verified by tests.'),
          click.option(
              '--unverified', is_flag=True,
              help='Stories not verified by tests.'),
          click.option(
              '--documented', is_flag=True,
              help='Documented stories only.'),
          click.option(
              '--undocumented', is_flag=True,
              help='Undocumented stories only.'),
          click.option(
              '--covered', is_flag=True,
              help='Covered stories only.'),
          click.option(
              '--uncovered', is_flag=True,
              help='Uncovered stories only.'),
          ]
  
      for option in reversed(options):
          function = option(function)
      return function
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Main
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  @click.group(invoke_without_command=True, cls=CustomGroup)
  @click.pass_context
  @click.option(
      '-r', '--root', metavar='PATH', show_default=True,
      help="Path to the tree root, '.' by default.")
  @click.option(
      '-c', '--context', metavar='DATA', multiple=True,
      help='Set the fmf context. Use KEY=VAL or KEY=VAL1,VAL2... format '
           'to define individual dimensions or the @FILE notation to load data '
           'from provided yaml file. Can be specified multiple times. ')
  @verbose_debug_quiet
  @click.option(
      '--version', is_flag=True,
      help='Show tmt version and commit hash.')
  def main(
          click_contex: click.core.Context,
          root: str,
          context: List[str],
          **kwargs: Any) -> None:
      """ Test Management Tool """
      # Show current tmt version and exit
      if kwargs.get('version'):
          print(f"tmt version: {tmt.__version__}")
          raise SystemExit(0)
  
      # Disable coloring if NO_COLOR is set
      if 'NO_COLOR' in os.environ:
          click_contex.color = False
  
      # Save click context and fmf context for future use
      tmt.utils.Common._save_context(click_contex)
  
      # Initialize metadata tree (from given path or current directory)
      tree = tmt.Tree(root or os.curdir)
  
      # TODO: context object details need checks
      click_contex.obj = ContextObject(
          common=tmt.utils.Common(),
          fmf_context=tmt.utils.context_to_dict(context),
          steps=set(),
          tree=tree
          )
  
      # Show overview of available tests, plans and stories
      if click_contex.invoked_subcommand is None:
          tmt.Test.overview(tree)
          tmt.Plan.overview(tree)
          tmt.Story.overview(tree)
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Run
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  @main.group(chain=True, invoke_without_command=True, cls=CustomGroup)
  @click.pass_context
  @click.option(
      '-i', '--id', 'id_', help='Run id (name or directory path).', metavar="ID")
  @click.option(
      '-l', '--last', help='Execute the last run once again.', is_flag=True)
  @click.option(
      '-r', '--rm', '--remove', 'remove', is_flag=True,
      help='Remove the workdir when test run is finished.')
  @click.option(
      '--scratch', is_flag=True,
      help='Remove the run workdir before executing to start from scratch.')
  @click.option(
      '--follow', is_flag=True,
      help='Output the logfile as it grows.')
  @click.option(
      '-a', '--all', help='Run all steps, customize some.', is_flag=True)
  @click.option(
      '-u', '--until', type=click.Choice(tmt.steps.STEPS), metavar='STEP',
      help='Enable given step and all preceding steps.')
  @click.option(
      '-s', '--since', type=click.Choice(tmt.steps.STEPS), metavar='STEP',
      help='Enable given step and all following steps.')
  @click.option(
      '-A', '--after', type=click.Choice(tmt.steps.STEPS), metavar='STEP',
      help='Enable all steps after the given one.')
  @click.option(
      '-B', '--before', type=click.Choice(tmt.steps.STEPS), metavar='STEP',
      help='Enable all steps before the given one.')
  @click.option(
      '-S', '--skip', type=click.Choice(tmt.steps.STEPS), metavar='STEP',
      help='Skip given step(s) during test run execution.', multiple=True)
  @click.option(
      '-e', '--environment', metavar='KEY=VALUE|@FILE', multiple=True,
      help='Set environment variable. Can be specified multiple times. The '
           '"@" prefix marks a file to load (yaml or dotenv formats supported).')
  @click.option(
      '--environment-file', metavar='FILE|URL', multiple=True,
      help='Set environment variables from file or url (yaml or dotenv formats '
           'are supported). Can be specified multiple times.')
  @verbose_debug_quiet
  @force_dry
  def run(context: click.core.Context, id_: str, **kwargs: Any) -> None:
      """ Run test steps. """
      # Initialize
      run = tmt.Run(id_, context.obj.tree, context=context)
      context.obj.run = run
  
  
  # Steps options
  run.add_command(tmt.steps.discover.DiscoverPlugin.command())
  run.add_command(tmt.steps.provision.ProvisionPlugin.command())
  run.add_command(tmt.steps.prepare.PreparePlugin.command())
  run.add_command(tmt.steps.execute.ExecutePlugin.command())
  run.add_command(tmt.steps.report.ReportPlugin.command())
  run.add_command(tmt.steps.finish.FinishPlugin.command())
  run.add_command(tmt.steps.Login.command())
  run.add_command(tmt.steps.Reboot.command())
  
  
  @run.command(name='plans')
  @click.pass_context
  @click.option(
      '-n', '--name', 'names', metavar='[REGEXP|.]', multiple=True,
      help="Regular expression to match plan name or '.' for current directory.")
  @click.option(
      '-f', '--filter', 'filters', metavar='FILTER', multiple=True,
      help="Apply advanced filter (see 'pydoc fmf.filter').")
  @click.option(
      '-c', '--condition', 'conditions', metavar="EXPR", multiple=True,
      help="Use arbitrary Python expression for filtering.")
  @click.option(
      '--link', 'links', metavar="RELATION:TARGET", multiple=True,
      help="Filter by linked objects (regular expressions are "
           "supported for both relation and target).")
  @click.option(
      '--default', is_flag=True,
      help="Use default plans even if others are available.")
  @verbose_debug_quiet
  def run_plans(context: click.core.Context, **kwargs: Any) -> None:
      """
      Select plans which should be executed.
  
      Regular expression can be used to filter plans by name.
      Use '.' to select plans under the current working directory.
      """
      tmt.base.Plan._save_context(context)
  
  
  @run.command(name='tests')
  @click.pass_context
  @click.option(
      '-n', '--name', 'names', metavar='[REGEXP|.]', multiple=True,
      help="Regular expression to match test name or '.' for current directory.")
  @click.option(
      '-f', '--filter', 'filters', metavar='FILTER', multiple=True,
      help="Apply advanced filter (see 'pydoc fmf.filter').")
  @click.option(
      '-c', '--condition', 'conditions', metavar="EXPR", multiple=True,
      help="Use arbitrary Python expression for filtering.")
  @click.option(
      '--link', 'links', metavar="RELATION:TARGET", multiple=True,
      help="Filter by linked objects (regular expressions are "
           "supported for both relation and target).")
  @verbose_debug_quiet
  def run_tests(context: click.core.Context, **kwargs: Any) -> None:
      """
      Select tests which should be executed.
  
      Regular expression can be used to filter tests by name.
      Use '.' to select tests under the current working directory.
      """
      tmt.base.Test._save_context(context)
  
  
  # FIXME: click 8.0 renamed resultcallback to result_callback. The former
  #        name will be removed in click 8.1. However, click 8.0 will not
  #        be added to F33 and F34. Get rid of this workaround once
  #        all Fedora + EPEL releases have click 8.0 or newer available.
  run_callback = run.result_callback
  if run_callback is None:
      run_callback = run.resultcallback
  
  
  # TODO: commands is unknown, needs revisit
  @run_callback()  # type: ignore
  @click.pass_context
  def finito(
          click_context: click.core.Context,
          commands: Any,
          *args: Any,
          **kwargs: Any) -> None:
      """ Run tests if run defined """
      if hasattr(click_context.obj, 'run'):
          click_context.obj.run.go()
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Test
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  @main.group(invoke_without_command=True, cls=CustomGroup)
  @click.pass_context
  @verbose_debug_quiet
  def tests(context: click.core.Context, **kwargs: Any) -> None:
      """
      Manage tests (L1 metadata).
  
      Check available tests, inspect their metadata.
      Convert old metadata into the new fmf format.
      """
  
      # Show overview of available tests
      if context.invoked_subcommand is None:
          tmt.Test.overview(context.obj.tree)
  
  
  @tests.command(name='ls')
  @click.pass_context
  @name_filter_condition
  @verbose_debug_quiet
  def tests_ls(context: click.core.Context, **kwargs: Any) -> None:
      """
      List available tests.
  
      Regular expression can be used to filter tests by name.
      Use '.' to select tests under the current working directory.
      """
      tmt.Test._save_context(context)
      for test in context.obj.tree.tests():
          test.ls()
  
  
  @tests.command(name='show')
  @click.pass_context
  @name_filter_condition
  @verbose_debug_quiet
  def tests_show(context: click.core.Context, **kwargs: Any) -> None:
      """
      Show test details.
  
      Regular expression can be used to filter tests by name.
      Use '.' to select tests under the current working directory.
      """
      tmt.Test._save_context(context)
      for test in context.obj.tree.tests():
          test.show()
          echo()
  
  
  @tests.command(name='lint')
  @click.pass_context
  @name_filter_condition
  @source
  @fix
  @verbose_debug_quiet
  def tests_lint(context: click.core.Context, **kwargs: Any) -> None:
      """
      Check tests against the L1 metadata specification.
  
      Regular expression can be used to filter tests for linting.
      Use '.' to select tests under the current working directory.
      """
      # FIXME: Workaround https://github.com/pallets/click/pull/1840 for click 7
      context.params.update(**kwargs)
      tmt.Test._save_context(context)
      exit_code = 0
      for test in context.obj.tree.tests():
          if not test.lint():
              exit_code = 1
          echo()
      raise SystemExit(exit_code)
  
  
  _test_templates = listed(tmt.templates.TEST, join='or')
  
  
  @tests.command(name='create')
  @click.pass_context
  @click.argument('name')
  @click.option(
      '-t', '--template', metavar='TEMPLATE',
      help='Test template ({}).'.format(_test_templates),
      prompt='Template ({})'.format(_test_templates))
  @verbose_debug_quiet
  @force_dry
  def tests_create(
          context: click.core.Context,
          name: str,
          template: str,
          force: bool,
          **kwargs: Any) -> None:
      """
      Create a new test based on given template.
  
      Specify directory name or use '.' to create tests under the
      current working directory.
      """
      tmt.Test._save_context(context)
      tmt.Test.create(name, template, context.obj.tree.root, force)
  
  
  @tests.command(name='import')
  @click.pass_context
  @click.argument('paths', nargs=-1, metavar='[PATH]...')
  @click.option(
      '--nitrate / --no-nitrate', default=True,
      help='Import test metadata from Nitrate')
  @click.option(
      '--purpose / --no-purpose', default=True,
      help='Migrate description from PURPOSE file')
  @click.option(
      '--makefile / --no-makefile', default=True,
      help='Convert Beaker Makefile metadata')
  @click.option(
      '--restraint / --no-restraint', default=False,
      help='Convert restraint metadata file')
  @click.option(
      '--general / --no-general', default=True,
      help='Detect components from linked nitrate general plans '
           '(overrides Makefile/restraint component).')
  @click.option(
      '--type', 'types', metavar='TYPE', default=['multihost'], multiple=True,
      show_default=True,
      help="Convert selected types from Makefile into tags. "
           "Use 'all' to convert all detected types.")
  @click.option(
      '--disabled', default=False, is_flag=True,
      help='Import disabled test cases from Nitrate as well.')
  @click.option(
      '--manual', default=False, is_flag=True,
      help='Import manual test cases from Nitrate.')
  @click.option(
      '--plan', metavar='PLAN', type=int,
      help='Identifier of test plan from which to import manual test cases.')
  @click.option(
      '--case', metavar='CASE', type=int,
      help='Identifier of manual test case to be imported.')
  @click.option(
      '--with-script', default=False, is_flag=True,
      help='Import manual cases with non-empty script field in Nitrate.')
  @verbose_debug_quiet
  @force_dry
  def tests_import(
          context: click.core.Context,
          paths: List[str],
          makefile: bool,
          restraint: bool,
          general: bool,
          types: List[str],
          nitrate: bool,
          purpose: bool,
          disabled: bool,
          manual: bool,
          plan: int,
          case: int,
          with_script: bool,
          **kwargs: Any) -> None:
      """
      Import old test metadata into the new fmf format.
  
      Accepts one or more directories where old metadata are stored.
      By default all available sources and current directory are used.
      The following test metadata are converted for each source:
  
      \b
      makefile ..... summary, component, duration, require
      restraint .... name, description, entry_point, owner,
                     max_time, repoRequires
      purpose ...... description
      nitrate ...... contact, component, tag,
                     environment, relevancy, enabled
      """
      tmt.Test._save_context(context)
  
      if manual:
          if not (case or plan):
              raise tmt.utils.GeneralError(
                  "Option --case or --plan is mandatory when using --manual.")
          else:
              tmt.convert.read_manual(plan, case, disabled, with_script)
              return
  
      if not paths:
          paths = ['.']
      for path in paths:
          # Make sure we've got a real directory
          path = os.path.realpath(path)
          if not os.path.isdir(path):
              raise tmt.utils.GeneralError(
                  "Path '{0}' is not a directory.".format(path))
          # Gather old metadata and store them as fmf
          common, individual = tmt.convert.read(
              path, makefile, restraint, nitrate, purpose, disabled, types,
              general)
          # Add path to common metadata if there are virtual test cases
          if individual:
              root = fmf.Tree(path).root
              common['path'] = os.path.join('/', os.path.relpath(path, root))
          # Store common metadata
          common_path = os.path.join(path, 'main.fmf')
          tmt.convert.write(common_path, common)
          # Store individual data (as virtual tests)
          for testcase in individual:
              testcase_path = os.path.join(
                  path, str(testcase['extra-nitrate']) + '.fmf')
              tmt.convert.write(testcase_path, testcase)
          # Adjust runtest.sh content and permission if needed
          tmt.convert.adjust_runtest(os.path.join(path, 'runtest.sh'))
  
  
  @tests.command(name='export')
  @click.pass_context
  @name_filter_condition_long
  @click.option(
      '-h', '--how', metavar='METHOD',
      help='Use specified method for export (nitrate or polarion).')
  @click.option(
      '--nitrate', is_flag=True,
      help="Export test metadata to Nitrate, deprecated by '--how nitrate'.")
  @click.option(
      '--project-id', help='Use specific Polarion project ID.')
  @click.option(
      '--link-polarion / --no-link-polarion', default=True,
      help='Add Polarion link to fmf testcase metadata')
  @click.option(
      '--bugzilla', is_flag=True,
      help="Link Nitrate case to Bugzilla specified in the 'link' attribute "
           "with the relation 'verifies'.")
  @click.option(
      '--ignore-git-validation', is_flag=True,
      help="Ignore unpublished git changes and export to Nitrate. "
      "The case might not be able to be scheduled!")
  @click.option(
      '--create', is_flag=True,
      help="Create test cases in nitrate if they don't exist.")
  @click.option(
      '--general / --no-general', default=False,
      help="Link Nitrate case to component's General plan. Disabled by default. "
           "Note that this will unlink any previously connected general plans.")
  @click.option(
      '--link-runs / --no-link-runs', default=False,
      help="Link Nitrate case to all open runs of descendant plans of "
           "General plan. Disabled by default. Implies --general option.")
  @click.option(
      '--format', 'format_', default='yaml', show_default=True, metavar='FORMAT',
      help='Output format (yaml or dict).')
  @click.option(
      '--fmf-id', is_flag=True,
      help='Show fmf identifiers instead of test metadata.')
  @click.option(
      '--duplicate / --no-duplicate', default=False, show_default=True,
      help='Allow or prevent creating duplicates in Nitrate by searching for '
           'existing test cases with the same fmf identifier.')
  @click.option(
      '-n', '--dry', is_flag=True,
      help="Run in dry mode. No changes, please.")
  @click.option(
      '-d', '--debug', is_flag=True,
      help='Provide as much debugging details as possible.')
  def tests_export(
          context: click.core.Context,
          format_: str,
          how: str,
          nitrate: bool,
          bugzilla: bool,
          **kwargs: Any) -> None:
      """
      Export test data into the desired format.
  
      Regular expression can be used to filter tests by name.
      Use '.' to select tests under the current working directory.
      """
      tmt.Test._save_context(context)
      if nitrate:
          context.obj.common.warn(
              "Option '--nitrate' is deprecated, please use '--how nitrate' instead.")
          how = 'nitrate'
      if bugzilla and not how:
          raise tmt.utils.GeneralError(
              "The --bugzilla option is supported only with --nitrate "
              "or --polarion for now.")
  
      if how == 'nitrate' or how == 'polarion':
          for test in context.obj.tree.tests():
              test.export(format_=how)
      elif format_ in ['dict', 'yaml']:
          keys = None
          if kwargs.get('fmf_id'):
              keys = 'fmf-id'
  
          tests = [test.export(format_='dict', keys=keys) for test in
                   context.obj.tree.tests()]
          if format_ == 'dict':
              echo(tests, nl=False)
          else:
              echo(tmt.utils.dict_to_yaml(tests), nl=False)
      else:
          raise tmt.utils.GeneralError(
              f"Invalid test export format '{format_}'.")
  
  
  @tests.command(name="id")
  @click.pass_context
  @name_filter_condition
  @verbose_debug_quiet
  @force_dry
  def tests_id(context: click.core.Context, **kwargs: Any) -> None:
      """
      Generate a unique id for each selected test.
  
      A new UUID is generated for each test matching the provided
      filter and the value is stored to disk. Existing identifiers
      are kept intact.
      """
      tmt.Test._save_context(context)
      for test in context.obj.tree.tests():
          tmt.identifier.id_command(test.node, "test", dry=kwargs["dry"])
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Plan
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  @main.group(invoke_without_command=True, cls=CustomGroup)
  @click.pass_context
  @verbose_debug_quiet
  def plans(context: click.core.Context, **kwargs: Any) -> None:
      """
      Manage test plans (L2 metadata).
  
      \b
      Search for available plans.
      Explore detailed test step configuration.
      """
      tmt.Plan._save_context(context)
  
      # Show overview of available plans
      if context.invoked_subcommand is None:
          tmt.Plan.overview(context.obj.tree)
  
  
  @plans.command(name='ls')
  @click.pass_context
  @name_filter_condition
  @verbose_debug_quiet
  def plans_ls(context: click.core.Context, **kwargs: Any) -> None:
      """
      List available plans.
  
      Regular expression can be used to filter plans by name.
      Use '.' to select plans under the current working directory.
      """
      tmt.Plan._save_context(context)
      for plan in context.obj.tree.plans():
          plan.ls()
  
  
  @plans.command(name='show')
  @click.pass_context
  @name_filter_condition
  @verbose_debug_quiet
  def plans_show(context: click.core.Context, **kwargs: Any) -> None:
      """
      Show plan details.
  
      Regular expression can be used to filter plans by name.
      Use '.' to select plans under the current working directory.
      """
      tmt.Plan._save_context(context)
      for plan in context.obj.tree.plans():
          plan.show()
          echo()
  
  
  @plans.command(name='lint')
  @click.pass_context
  @name_filter_condition
  @source
  @verbose_debug_quiet
  def plans_lint(context: click.core.Context, **kwargs: Any) -> None:
      """
      Check plans against the L2 metadata specification.
  
      Regular expression can be used to filter plans by name.
      Use '.' to select plans under the current working directory.
      """
      # FIXME: Workaround https://github.com/pallets/click/pull/1840 for click 7
      context.params.update(**kwargs)
      tmt.Plan._save_context(context)
      exit_code = 0
      for plan in context.obj.tree.plans():
          if not plan.lint():
              exit_code = 1
          echo()
      raise SystemExit(exit_code)
  
  
  _plan_templates = listed(tmt.templates.PLAN, join='or')
  
  
  @plans.command(name='create')
  @click.pass_context
  @click.argument('name')
  @click.option(
      '-t', '--template', metavar='TEMPLATE',
      help='Plan template ({}).'.format(_plan_templates),
      prompt='Template ({})'.format(_plan_templates))
  @click.option(
      '--discover', metavar='YAML', multiple=True,
      help='Discover phase content in yaml format.')
  @click.option(
      '--provision', metavar='YAML', multiple=True,
      help='Provision phase content in yaml format.')
  @click.option(
      '--prepare', metavar='YAML', multiple=True,
      help='Prepare phase content in yaml format.')
  @click.option(
      '--execute', metavar='YAML', multiple=True,
      help='Execute phase content in yaml format.')
  @click.option(
      '--report', metavar='YAML', multiple=True,
      help='Report phase content in yaml format.')
  @click.option(
      '--finish', metavar='YAML', multiple=True,
      help='Finish phase content in yaml format.')
  @verbose_debug_quiet
  @force_dry
  def plans_create(
          context: click.core.Context,
          name: str,
          template: str,
          force: bool,
          **kwargs: Any) -> None:
      """ Create a new plan based on given template. """
      tmt.Plan._save_context(context)
      tmt.Plan.create(name, template, context.obj.tree.root, force)
  
  
  @plans.command(name='export')
  @click.pass_context
  @name_filter_condition_long
  @click.option(
      '--format', 'format_', default='yaml', show_default=True, metavar='FORMAT',
      help='Output format.')
  @click.option(
      '-d', '--debug', is_flag=True,
      help='Provide as much debugging details as possible.')
  def plans_export(context: click.core.Context, format_: str, **kwargs: Any) -> None:
      """
      Export plans into desired format.
  
      Regular expression can be used to filter plans by name.
      Use '.' to select plans under the current working directory.
      """
      tmt.Plan._save_context(context)
      plans = [plan.export(format_='dict') for plan in context.obj.tree.plans()]
  
      # Choose proper format
      if format_ == 'dict':
          echo(plans)
      elif format_ == 'yaml':
          echo(tmt.utils.dict_to_yaml(plans))
      else:
          raise tmt.utils.GeneralError(
              f"Invalid plan export format '{format_}'.")
  
  
  @plans.command(name="id")
  @click.pass_context
  @name_filter_condition
  @verbose_debug_quiet
  @force_dry
  def plans_id(context: click.core.Context, **kwargs: Any) -> None:
      """
      Generate a unique id for each selected plan.
  
      A new UUID is generated for each plan matching the provided
      filter and the value is stored to disk. Existing identifiers
      are kept intact.
      """
      tmt.Plan._save_context(context)
      for plan in context.obj.tree.plans():
          tmt.identifier.id_command(plan.node, "plan", dry=kwargs["dry"])
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Story
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  
  @main.group(invoke_without_command=True, cls=CustomGroup)
  @click.pass_context
  @verbose_debug_quiet
  def stories(context: click.core.Context, **kwargs: Any) -> None:
      """
      Manage user stories.
  
      \b
      Check available user stories.
      Explore coverage (test, implementation, documentation).
      """
      tmt.Story._save_context(context)
  
      # Show overview of available stories
      if context.invoked_subcommand is None:
          tmt.Story.overview(context.obj.tree)
  
  
  @stories.command(name='ls')
  @click.pass_context
  @name_filter_condition_long
  @implemented_verified_documented
  @verbose_debug_quiet
  def stories_ls(
          context: click.core.Context,
          implemented: bool,
          verified: bool,
          documented: bool,
          covered: bool,
          unimplemented: bool,
          unverified: bool,
          undocumented: bool,
          uncovered: bool,
          **kwargs: Any) -> None:
      """
      List available stories.
  
      Regular expression can be used to filter stories by name.
      Use '.' to select stories under the current working directory.
      """
      tmt.Story._save_context(context)
      for story in context.obj.tree.stories():
          if story._match(implemented, verified, documented, covered,
                          unimplemented, unverified, undocumented, uncovered):
              story.ls()
  
  
  @stories.command(name='show')
  @click.pass_context
  @name_filter_condition_long
  @implemented_verified_documented
  @verbose_debug_quiet
  def stories_show(
          context: click.core.Context,
          implemented: bool,
          verified: bool,
          documented: bool,
          covered: bool,
          unimplemented: bool,
          unverified: bool,
          undocumented: bool,
          uncovered: bool,
          **kwargs: Any) -> None:
      """
      Show story details.
  
      Regular expression can be used to filter stories by name.
      Use '.' to select stories under the current working directory.
      """
      tmt.Story._save_context(context)
      for story in context.obj.tree.stories():
          if story._match(implemented, verified, documented, covered,
                          unimplemented, unverified, undocumented, uncovered):
              story.show()
              echo()
  
  
  _story_templates = listed(tmt.templates.STORY, join='or')
  
  
  @stories.command(name='create')
  @click.pass_context
  @click.argument('name')
  @click.option(
      '-t', '--template', metavar='TEMPLATE',
      prompt='Template ({})'.format(_story_templates),
      help='Story template ({}).'.format(_story_templates))
  @verbose_debug_quiet
  @force_dry
  def stories_create(
          context: click.core.Context,
          name: str,
          template: str,
          force: bool,
          **kwargs: Any) -> None:
      """ Create a new story based on given template. """
      tmt.Story._save_context(context)
      tmt.base.Story.create(name, template, context.obj.tree.root, force)
  
  
  @stories.command(name='coverage')
  @click.option(
      '--docs', is_flag=True, help='Show docs coverage.')
  @click.option(
      '--test', is_flag=True, help='Show test coverage.')
  @click.option(
      '--code', is_flag=True, help='Show code coverage.')
  @click.pass_context
  @name_filter_condition_long
  @implemented_verified_documented
  @verbose_debug_quiet
  def stories_coverage(
          context: click.core.Context,
          code: bool,
          test: bool,
          docs: bool,
          implemented: bool,
          verified: bool,
          documented: bool,
          covered: bool,
          unimplemented: bool,
          unverified: bool,
          undocumented: bool,
          uncovered: bool,
          **kwargs: Any) -> None:
      """
      Show code, test and docs coverage for given stories.
  
      Regular expression can be used to filter stories by name.
      Use '.' to select stories under the current working directory.
      """
      tmt.Story._save_context(context)
  
      def headfoot(text: str) -> None:
          """ Format simple header/footer """
          echo(style(text.rjust(4) + ' ', fg='blue'), nl=False)
  
      header = False
      total = code_coverage = test_coverage = docs_coverage = 0
      if not any([code, test, docs]):
          code = test = docs = True
      for story in context.obj.tree.stories():
          # Check conditions
          if not story._match(
                  implemented, verified, documented, covered, unimplemented,
                  unverified, undocumented, uncovered):
              continue
          # Show header once
          if not header:
              if code:
                  headfoot('code')
              if test:
                  headfoot('test')
              if docs:
                  headfoot('docs')
              headfoot('story')
              echo()
              header = True
          # Show individual stats
          status = story.coverage(code, test, docs)
          total += 1
          code_coverage += status[0]
          test_coverage += status[1]
          docs_coverage += status[2]
      # Summary
      if not total:
          return
      if code:
          headfoot('{}%'.format(round(100 * code_coverage / total)))
      if test:
          headfoot('{}%'.format(round(100 * test_coverage / total)))
      if docs:
          headfoot('{}%'.format(round(100 * docs_coverage / total)))
      headfoot('from {}'.format(listed(total, 'story')))
      echo()
  
  
  @stories.command(name='export')
  @click.pass_context
  @name_filter_condition_long
  @implemented_verified_documented
  @click.option(
      '--format', 'format_', default='rst', show_default=True, metavar='FORMAT',
      help='Output format.')
  @click.option(
      '-d', '--debug', is_flag=True,
      help='Provide as much debugging details as possible.')
  def stories_export(
          context: click.core.Context,
          format_: str,
          implemented: bool,
          verified: bool,
          documented: bool,
          covered: bool,
          unimplemented: bool,
          unverified: bool,
          undocumented: bool,
          uncovered: bool,
          **kwargs: Any) -> None:
      """
      Export selected stories into desired format.
  
      Regular expression can be used to filter stories by name.
      Use '.' to select stories under the current working directory.
      """
      tmt.Story._save_context(context)
  
      for story in context.obj.tree.stories(whole=True):
          if story._match(implemented, verified, documented, covered,
                          unimplemented, unverified, undocumented, uncovered):
              echo(story.export(format_))
  
  
  @stories.command(name='lint')
  @click.pass_context
  @name_filter_condition
  @source
  @verbose_debug_quiet
  def stories_lint(context: click.core.Context, **kwargs: Any) -> None:
      """
      Check stories against the L3 metadata specification.
  
      Regular expression can be used to filter stories by name.
      Use '.' to select stories under the current working directory.
      """
      # FIXME: Workaround https://github.com/pallets/click/pull/1840 for click 7
      context.params.update(**kwargs)
      tmt.Story._save_context(context)
      exit_code = 0
      for story in context.obj.tree.stories():
          if not story.lint():
              exit_code = 1
          echo()
      raise SystemExit(exit_code)
  
  
  @stories.command(name="id")
  @click.pass_context
  @name_filter_condition_long
  @implemented_verified_documented
  @verbose_debug_quiet
  @force_dry
  def stories_id(
          context: click.core.Context,
          implemented: bool,
          verified: bool,
          documented: bool,
          covered: bool,
          unimplemented: bool,
          unverified: bool,
          undocumented: bool,
          uncovered: bool,
          **kwargs: Any) -> None:
      """
      Generate a unique id for each selected story.
  
      A new UUID is generated for each story matching the provided
      filter and the value is stored to disk. Existing identifiers
      are kept intact.
      """
      tmt.Story._save_context(context)
      for story in context.obj.tree.stories():
          if story._match(implemented, verified, documented, covered,
                          unimplemented, unverified, undocumented, uncovered):
              tmt.identifier.id_command(story.node, "story", dry=kwargs["dry"])
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Init
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  @main.command()
  @click.pass_context
  @click.argument('path', default='.')
  @click.option(
      '-t', '--template', default='empty', metavar='TEMPLATE',
      type=click.Choice(['empty'] + tmt.templates.INIT_TEMPLATES),
      help='Template ({}).'.format(
          listed(tmt.templates.INIT_TEMPLATES, join='or')))
  @verbose_debug_quiet
  @force_dry
  def init(
          context: click.core.Context,
          path: str,
          template: str,
          force: bool,
          **kwargs: Any) -> None:
      """
      Initialize a new tmt tree.
  
      By default tree is created in the current directory.
      Provide a PATH to create it in a different location.
  
      \b
      A tree can be optionally populated with example metadata:
      * 'mini' template contains a minimal plan and no tests,
      * 'base' template contains a plan and a beakerlib test,
      * 'full' template contains a 'full' story, an 'full' plan and a shell test.
      """
  
      tmt.base.Tree._save_context(context)
      tmt.base.Tree.init(path, template, force, **kwargs)
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Status
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  @main.command()
  @click.pass_context
  @workdir_root
  @click.option(
      '-i', '--id', metavar="ID",
      help='Run id (name or directory path) to show status of.')
  @click.option(
      '--abandoned', is_flag=True, default=False,
      help='List runs which have provision step completed but finish step '
           'not yet done.')
  @click.option(
      '--active', is_flag=True, default=False,
      help='List runs where at least one of the enabled steps has not '
           'been finished.')
  @click.option(
      '--finished', is_flag=True, default=False,
      help='List all runs which have all enabled steps completed.')
  @verbose_debug_quiet
  def status(
          context: click.core.Context,
          workdir_root: str,
          abandoned: bool,
          active: bool,
          finished: bool,
          **kwargs: Any) -> None:
      """
      Show status of runs.
  
      Lists past runs in the given directory filtered using options.
      /var/tmp/tmt is used by default.
  
      By default, status of the whole runs is listed. With more
      verbosity (-v), status of every plan is shown. By default,
      the last completed step is displayed, 'done' is used when
      all enabled steps are completed. Status of every step is
      displayed with the most verbosity (-vv).
  
      """
      if [abandoned, active, finished].count(True) > 1:
          raise tmt.utils.GeneralError(
              "Options --abandoned, --active and --finished cannot be "
              "used together.")
      if not os.path.exists(workdir_root):
          raise tmt.utils.GeneralError(f"Path '{workdir_root}' doesn't exist.")
      status_obj = tmt.Status(context=context)
      status_obj.show()
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Clean
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  dry = tmt.options.force_dry[1]
  
  
  @main.group(chain=True, invoke_without_command=True, cls=CustomGroup)
  @click.pass_context
  @verbose_debug_quiet
  @dry
  def clean(context: click.core.Context, **kwargs: Any) -> None:
      """
      Clean workdirs, guests or images.
  
      Without any command, clean everything, stop the guests, remove
      all runs and then remove all images. Search for runs in
      /var/tmp/tmt, if runs are stored elsewhere, the path to them can
      be set using a subcommand (either runs or guests subcommand).
  
      The subcommands can be chained, the order of cleaning is always
      the same, irrespective of the order on the command line. First, all
      the guests are cleaned, followed by runs and images.
      """
      echo(style('clean', fg='red'))
      clean_obj = tmt.Clean(parent=context.obj.common, context=context)
      context.obj.clean = clean_obj
      context.obj.clean_partials = collections.defaultdict(list)
      exit_code = 0
      if context.invoked_subcommand is None:
          # Set path to default
          context.params['workdir_root'] = tmt.utils.WORKDIR_ROOT
          # Create another level to the hierarchy so that logging indent is
          # consistent between the command and subcommands
          clean_obj = tmt.Clean(parent=clean_obj, context=context)
          if os.path.exists(tmt.utils.WORKDIR_ROOT):
              if not clean_obj.guests():
                  exit_code = 1
              if not clean_obj.runs():
                  exit_code = 1
          else:
              clean_obj.warn(
                  f"Directory '{tmt.utils.WORKDIR_ROOT}' does not exist, "
                  f"skipping guest and run cleanup.")
          clean_obj.images()
          raise SystemExit(exit_code)
  
  
  # FIXME: Click deprecation, see function finito for more info
  clean_callback = clean.result_callback
  if clean_callback is None:
      clean_callback = clean.resultcallback
  
  
  # TODO: commands is unknown, needs revisit
  @clean_callback()  # type: ignore
  @click.pass_context
  def perform_clean(
          click_context: click.core.Context,
          commands: Any,
          *args: Any,
          **kwargs: Any) -> None:
      """
      Perform clean actions in the correct order.
  
      We need to ensure that guests are always cleaned before the run workdirs
      even if the user specified them in reverse order.
      """
      clean_order = ("guests", "runs", "images")
      exit_code = 0
      for phase in clean_order:
          for partial in click_context.obj.clean_partials[phase]:
              if not partial():
                  exit_code = 1
      raise SystemExit(exit_code)
  
  
  @clean.command(name='runs')
  @click.pass_context
  @workdir_root
  @click.option(
      '-l', '--last', is_flag=True, help='Clean the workdir of the last run.')
  @click.option(
      '-i', '--id', 'id_', metavar="ID",
      help='Run id (name or directory path) to clean workdir of.')
  @click.option(
      '-k', '--keep', type=int,
      help='The number of latest workdirs to keep, clean the rest.')
  @verbose_debug_quiet
  @dry
  def clean_runs(
          context: click.core.Context,
          workdir_root: str,
          last: bool,
          id_: str,
          keep: int,
          **kwargs: Any) -> None:
      """
      Clean workdirs of past runs.
  
      Remove all runs in '/var/tmp/tmt' by default.
      """
      defined = [last is True, id_ is not None, keep is not None]
      if defined.count(True) > 1:
          raise tmt.utils.GeneralError(
              "Options --last, --id and --keep cannot be used together.")
      if keep is not None and keep < 0:
          raise tmt.utils.GeneralError("--keep must not be a negative number.")
      if not os.path.exists(workdir_root):
          raise tmt.utils.GeneralError(f"Path '{workdir_root}' doesn't exist.")
      clean_obj = tmt.Clean(parent=context.obj.clean, context=context)
      context.obj.clean_partials["runs"].append(clean_obj.runs)
  
  
  @clean.command(name='guests')
  @click.pass_context
  @workdir_root
  @click.option(
      '-l', '--last', is_flag=True, help='Stop the guest of the last run.')
  @click.option(
      '-i', '--id', 'id_', metavar="ID",
      help='Run id (name or directory path) to stop the guest of.')
  @click.option(
      '-h', '--how', metavar='METHOD',
      help='Stop guests of the specified provision method.')
  @verbose_debug_quiet
  @dry
  def clean_guests(
          context: click.core.Context,
          workdir_root: str,
          last: bool,
          id_: int,
          **kwargs: Any) -> None:
      """
      Stop running guests of runs.
  
      Stop guests of all runs in '/var/tmp/tmt' by default.
      """
      if last and id_ is not None:
          raise tmt.utils.GeneralError(
              "Options --last and --id cannot be used together.")
      if not os.path.exists(workdir_root):
          raise tmt.utils.GeneralError(f"Path '{workdir_root}' doesn't exist.")
      clean_obj = tmt.Clean(parent=context.obj.clean, context=context)
      context.obj.clean_partials["guests"].append(clean_obj.guests)
  
  
  @clean.command(name='images')
  @click.pass_context
  @verbose_debug_quiet
  @dry
  def clean_images(context: click.core.Context, **kwargs: Any) -> None:
      """
      Remove images of supported provision methods.
  
      Currently supported methods are:
       - testcloud
      """
      # FIXME: If there are more provision methods supporting this,
      #        we should add options to specify which provision should be
      #        cleaned, similarly to guests.
      clean_obj = tmt.Clean(parent=context.obj.clean, context=context)
      context.obj.clean_partials["images"].append(clean_obj.images)
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Lint
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  
  @main.command(name='lint')
  @click.pass_context
  @name_filter_condition
  @source
  @fix
  @verbose_debug_quiet
  def lint(context: click.core.Context, **kwargs: Any) -> None:
      """
      Check all the present metadata against the specification.
  
      Combines all the partial linting (tests, plans and stories)
      into one command. Options are applied to all parts of the lint.
  
      Regular expression can be used to filter metadata by name.
      Use '.' to select tests, plans and stories under the current
      working directory.
      """
      exit_code = 0
      for command in (tests_lint, plans_lint, stories_lint):
          try:
              context.forward(command)
          except SystemExit as e:
              exit_code |= e.code
      raise SystemExit(exit_code)
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Setup
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  
  @main.group(cls=CustomGroup)
  @click.pass_context
  def setup(context: click.core.Context, **kwargs: Any) -> None:
      """
      Setup the environment for working with tmt.
      """
  
  
  @setup.group(cls=CustomGroup)
  @click.pass_context
  def completion(context: click.core.Context, **kwargs: Any) -> None:
      """
      Setup shell completions.
  
      By default, these commands only write a shell script to the output
      which can then be sourced from the shell's configuration file. Use
      the '--install' option to store and enable the configuration
      permanently.
      """
  
  
  COMPLETE_VARIABLE = '_TMT_COMPLETE'
  COMPLETE_SCRIPT = 'tmt-complete'
  
  
  def setup_completion(shell: str, install: bool) -> None:
      """ Setup completion based on the shell """
      config = tmt.utils.Config()
      # Fish gets installed into its special location where it is automatically
      # loaded.
      if shell == 'fish':
          script = os.path.expanduser('~/.config/fish/completions/tmt.fish')
      # Bash and zsh get installed to tmt's config directory.
      else:
          script = os.path.join(config.path, f'{COMPLETE_SCRIPT}.{shell}')
  
      out = open(script, 'w') if install else sys.stdout
      subprocess.run(f'{COMPLETE_VARIABLE}={shell}_source tmt',
                     shell=True, stdout=out)
  
      if install:
          out.close()
          # If requested, modify .bashrc or .zshrc
          if shell != 'fish':
              config_path = os.path.expanduser(f'~/.{shell}rc')
              with open(config_path, 'a') as shell_config:
                  shell_config.write('\n# Generated by tmt\n')
                  shell_config.write(f'source {script}')
  
  
  @completion.command(name='bash')
  @click.pass_context
  @click.option(
      '--install', '-i', 'install', is_flag=True,
      help="Persistently store the script to tmt's configuration directory "
           "and set it up by modifying '~/.bashrc'.")
  def completion_bash(context: click.core.Context, install: bool, **kwargs: Any) -> None:
      """
      Setup shell completions for bash.
      """
      setup_completion('bash', install)
  
  
  @completion.command(name='zsh')
  @click.pass_context
  @click.option(
      '--install', '-i', 'install', is_flag=True,
      help="Persistently store the script to tmt's configuration directory "
           "and set it up by modifying '~/.zshrc'.")
  def completion_zsh(context: click.core.Context, install: bool, **kwargs: Any) -> None:
      """
      Setup shell completions for zsh.
      """
      setup_completion('zsh', install)
  
  
  @completion.command(name='fish')
  @click.pass_context
  @click.option(
      '--install', '-i', 'install', is_flag=True,
      help="Persistently store the script to "
           "'~/.config/fish/completions/tmt.fish'.")
  def completion_fish(context: click.core.Context, install: bool, **kwargs: Any) -> None:
      """
      Setup shell completions for fish.
      """
      setup_completion('fish', install)
  
files: tmt/cli.py
files: # coding: utf-8
  
  """ Export metadata into nitrate """
  
  
  import email.utils
  import os
  import re
  import traceback
  import types
  import xmlrpc.client
  from functools import lru_cache
  from typing import Any, Dict, Generator, List, Optional, Tuple, Union, cast
  
  import fmf
  from click import echo, secho, style
  
  import tmt
  import tmt.utils
  from tmt.convert import SYSTEM_OTHER, add_link
  from tmt.identifier import ID_KEY, add_uuid_if_not_defined
  from tmt.utils import ConvertError, check_git_url, markdown_to_html
  
  bugzilla: Optional[types.ModuleType] = None
  gssapi: Optional[types.ModuleType] = None
  nitrate: Optional[types.ModuleType] = None
  
  PolarionException: Any = None
  PolarionTestCase: Any = None
  PolarionWorkItem: Any = None
  
  DEFAULT_PRODUCT: Any = None
  
  SectionsReturnType = Tuple[str, str, str, str]
  HeadingsType = List[List[Union[int, str]]]
  SectionsHeadingsType = Dict[str, HeadingsType]
  
  log = fmf.utils.Logging('tmt').logger
  
  WARNING = """
  Test case has been migrated to git. Any changes made here might be overwritten.
  See: https://tmt.readthedocs.io/en/latest/questions.html#nitrate-migration
  """.lstrip()
  
  # For linking bugs
  BUGZILLA_XMLRPC_URL = "https://bugzilla.redhat.com/xmlrpc.cgi"
  NITRATE_TRACKER_ID = 69  # ID of nitrate in RH's bugzilla
  POLARION_TRACKER_ID = 117  # ID of polarion in RH's bugzilla
  RE_BUGZILLA_URL = r'bugzilla.redhat.com/show_bug.cgi\?id=(\d+)'
  RE_POLARION_URL = r'.*/polarion/#/project/.*/workitem\?id=(.*)'
  LEGACY_POLARION_PROJECTS = set(['RedHatEnterpriseLinux7'])
  
  
  def import_nitrate() -> Any:
      """ Conditionally import the nitrate module """
      # Need to import nitrate only when really needed. Otherwise we get
      # traceback when nitrate not installed or config file not available.
      # And we want to keep the core tmt package with minimal dependencies.
      try:
          global nitrate, DEFAULT_PRODUCT, gssapi
          import gssapi
          import nitrate
          assert nitrate
          DEFAULT_PRODUCT = nitrate.Product(name='RHEL Tests')
          return nitrate
      except ImportError:
          raise ConvertError(
              "Install tmt-test-convert to export tests to nitrate.")
      except nitrate.NitrateError as error:  # type: ignore[union-attr]  # nitrate is no longer None
          raise ConvertError(error)
  
  
  def import_polarion() -> None:
      """ Import polarion python api - pylero """
      try:
          global PolarionException, PolarionTestCase, PolarionWorkItem
          from pylero.exceptions import PyleroLibException as PolarionException
          from pylero.work_item import TestCase as PolarionTestCase
          from pylero.work_item import _WorkItem as PolarionWorkItem
      except ImportError:
          raise ConvertError(
              "Run 'pip install tmt[export-polarion]' so pylero is installed.")
      except PolarionException as exc:
          log.debug(traceback.format_exc())
          raise ConvertError("Failed to login with pylero", original=exc)
  
  
  def get_bz_instance() -> Any:
      """ Import the bugzilla module and return BZ instance """
      try:
          import bugzilla
      except ImportError:
          raise ConvertError(
              "Install 'tmt-test-convert' to link test to the bugzilla.")
  
      try:
          bz_instance = bugzilla.Bugzilla(url=BUGZILLA_XMLRPC_URL)
      except Exception as exc:
          log.debug(traceback.format_exc())
          raise ConvertError(
              "Couldn't initialize the Bugzilla client.", original=exc)
  
      if not bz_instance.logged_in:
          raise ConvertError(
              "Not logged to Bugzilla, check 'man bugzilla' section "
              "'AUTHENTICATION CACHE AND API KEYS'.")
      return bz_instance
  
  
  def _nitrate_find_fmf_testcases(test: 'tmt.Test') -> Generator[Any, None, None]:
      """
      Find all Nitrate test cases with the same fmf identifier
  
      All component general plans are explored for possible duplicates.
      """
      assert nitrate
      for component in test.component:
          try:
              for testcase in find_general_plan(component).testcases:
                  struct_field = tmt.utils.StructuredField(testcase.notes)
                  try:
                      fmf_id = tmt.utils.yaml_to_dict(struct_field.get('fmf'))
                      if fmf_id == test.fmf_id:
                          echo(style(
                              f"Existing test case '{testcase.identifier}' "
                              f"found for given fmf id.", fg='magenta'))
                          yield testcase
                  except tmt.utils.StructuredFieldError:
                      pass
          except nitrate.NitrateError:
              pass
  
  
  def convert_manual_to_nitrate(test_md: str) -> SectionsReturnType:
      """
      Convert Markdown document to html sections.
  
      These sections can be exported to nitrate.
      Expects: Markdown document as a file.
      Returns: tuple of (step, expect, setup, cleanup) sections
      as html strings.
      """
  
      sections_headings: SectionsHeadingsType = {
          heading: []
          for heading_list in tmt.base.SECTIONS_HEADINGS.values()
          for heading in heading_list
          }
  
      html = markdown_to_html(test_md)
      html_splitlines = html.splitlines()
  
      for key in sections_headings.keys():
          result: HeadingsType = []
          i = 0
          while html_splitlines:
              try:
                  if re.search("^" + key + "$", html_splitlines[i]):
                      html_content = str()
                      if key.startswith('<h1>Test'):
                          html_content = html_splitlines[i].\
                              replace('<h1>', '<b>').\
                              replace('</h1>', '</b>')
                      for j in range(i + 1, len(html_splitlines)):
                          if re.search("^<h[1-4]>(.+?)</h[1-4]>$",
                                       html_splitlines[j]):
                              result.append([i, html_content])
                              i = j - 1
                              break
                          html_content += html_splitlines[j] + "\n"
                          # Check end of the file
                          if j + 1 == len(html_splitlines):
                              result.append([i, html_content])
              except IndexError:
                  sections_headings[key] = result
                  break
              i += 1
              if i >= len(html_splitlines):
                  sections_headings[key] = result
                  break
  
      def concatenate_headings_content(headings: Tuple[str, ...]) -> HeadingsType:
          content = list()
          for v in headings:
              content += sections_headings[v]
          return content
  
      def enumerate_content(content: HeadingsType) -> HeadingsType:
          # for sorting convert the index to integer, but keep whole list as list of strings
          content.sort(key=lambda a: int(a[0]))
          for c in range(len(content)):
              content[c][1] = f"<p>Step {c + 1}.</p>" + str(content[c][1])
          return content
  
      sorted_test = sorted(concatenate_headings_content((
          '<h1>Test</h1>',
          '<h1>Test .*</h1>')))
  
      sorted_step = sorted(enumerate_content(concatenate_headings_content((
          '<h2>Step</h2>',
          '<h2>Test Step</h2>'))) + sorted_test)
      step = ''.join([f"{v[1]}" for v in sorted_step])
  
      sorted_expect = sorted(enumerate_content(concatenate_headings_content((
          '<h2>Expect</h2>',
          '<h2>Result</h2>',
          '<h2>Expected Result</h2>'))) + sorted_test)
      expect = ''.join([f"{v[1]}" for v in sorted_expect])
  
      def check_section_exists(text: str) -> str:
          try:
              return str(sections_headings[text][0][1])
          except (IndexError, KeyError):
              return ''
  
      setup = check_section_exists('<h1>Setup</h1>')
      cleanup = check_section_exists('<h1>Cleanup</h1>')
  
      return step, expect, setup, cleanup
  
  
  def bz_set_coverage(bug_ids: List[int], case_id: str, tracker_id: int) -> None:
      """ Set coverage in Bugzilla """
      bz_instance = get_bz_instance()
  
      overall_pass = True
      no_email = 1  # Do not send emails about the change
      get_bz_dict = {
          'ids': bug_ids,
          'include_fields': ['id', 'external_bugs', 'flags']}
      bugs_data = bz_instance._proxy.Bug.get(get_bz_dict)
      for bug in bugs_data['bugs']:
          # Process flag (might fail for some types)
          bug_id = bug['id']
          if 'qe_test_coverage+' not in set(
                  [x['name'] + x['status'] for x in bug['flags']]):
              try:
                  bz_instance._proxy.Flag.update({
                      'ids': [bug_id],
                      'nomail': no_email,
                      'updates': [{
                          'name': 'qe_test_coverage',
                          'status': '+'
                          }]
                      })
              except xmlrpc.client.Fault as err:
                  # TODO: Fix missing overall_result = False, breaks tests
                  # if bug used for testing is not changed
                  # BZ#1925518 can't have qe_test_coverage flag
                  log.debug(f"Update flag failed: {err}")
                  echo(style(
                      f"Failed to set qe_test_coverage+ flag for BZ#{bug_id}",
                      fg='red'))
          # Process external tracker - should succeed
          current = set(
              [b['ext_bz_bug_id'] for b in bug['external_bugs']
               if b['ext_bz_id'] == tracker_id])
          if case_id not in current:
              query = {
                  'bug_ids': [bug_id],
                  'nomail': no_email,
                  'external_bugs': [{
                      'ext_type_id': tracker_id,
                      'ext_bz_bug_id': case_id,
                      'ext_description': '',
                      }]
                  }
              try:
                  bz_instance._proxy.ExternalBugs.add_external_bug(query)
              except Exception as err:
                  log.debug(f"Link case failed: {err}")
                  echo(style(f"Failed to link to BZ#{bug_id}", fg='red'))
                  overall_pass = False
      if not overall_pass:
          raise ConvertError("Failed to link the case to bugs.")
  
      echo(style("Linked to Bugzilla: {}.".format(
          " ".join([f"BZ#{bz_id}" for bz_id in bug_ids])), fg='magenta'))
  
  
  def get_polarion_ids(
          query_result: List[Any],
          preferred_project: Optional[str] = None) -> Tuple[str, Optional[str]]:
      """ Return case and project ids from query results """
      if not query_result:
          return 'None', None
      if len(query_result) == 1:
          return query_result[0].work_item_id, query_result[0].project_id
  
      if preferred_project:
          try:
              return next(
                  item.work_item_id for item in query_result
                  if item.project_id == preferred_project), preferred_project
          except StopIteration:
              pass
  
      for result in query_result:
          # If multiple cases are found prefer cases from other projects
          # than these legacy ones
          if str(result.project_id) not in LEGACY_POLARION_PROJECTS:
              return result.work_item_id, result.project_id
  
      return query_result[0].work_item_id, query_result[0].project_id
  
  
  def find_polarion_case_ids(
          data: Dict[str, str],
          preferred_project: Optional[str] = None) -> Tuple[str, Optional[str]]:
      """ Find IDs for Polarion case from data dictionary """
      assert PolarionWorkItem
  
      # Search by UUID
      if data.get(ID_KEY):
          query_result = PolarionWorkItem.query(
              data.get(ID_KEY), fields=['work_item_id', 'project_id'])
          return get_polarion_ids(query_result, preferred_project)
      # Search by TCMS Case ID
      extra_nitrate = data.get('extra-nitrate')
      if extra_nitrate:
          nitrate_case_id_search = re.search(r'\d+', extra_nitrate)
          if not nitrate_case_id_search:
              raise ConvertError(
                  "Could not find a valid nitrate testcase ID in 'extra-nitrate' attribute")
          nitrate_case_id = str(int(nitrate_case_id_search.group()))
          query_result = PolarionWorkItem.query(
              f"tcmscaseid:{nitrate_case_id}", fields=['work_item_id', 'project_id'])
          return get_polarion_ids(query_result, preferred_project)
      # Search by extra task
      if data.get('extra-task'):
          query_result = PolarionWorkItem.query(
              data.get('extra-task'), fields=['work_item_id', 'project_id'])
          return get_polarion_ids(query_result, preferred_project)
      return 'None', None
  
  
  def get_polarion_case(data: Dict[str, str], preferred_project: Optional[str] = None) -> Any:
      """ Get Polarion case through couple different methods """
      import_polarion()
  
      assert PolarionTestCase
      assert PolarionException
  
      polarion_id, project_id = find_polarion_case_ids(data, preferred_project)
  
      try:
          polarion_case = PolarionTestCase(
              project_id=project_id, work_item_id=polarion_id)
          echo(style(
              f"Test case '{str(polarion_case.work_item_id)}' found.",
              fg='blue'))
          return polarion_case
      except PolarionException:
          return None
  
  
  def enabled_somewhere(test: 'tmt.Test') -> bool:
      """ True if the test is enabled for some context (adjust rules) """
      # node.original_data are fmf data _before_ adjust was processed
      try:
          if test.node.original_data['enabled']:
              return True
      except KeyError:
          # Key defaults to True so we already know the outcome
          return True
  
      # Some rule in adjust enables the test
      try:
          adjust_rules = test.node.original_data['adjust']
          # TODO: Should not be necessary once we normalize data
          if isinstance(adjust_rules, dict):
              adjust_rules = [adjust_rules]
          for rule in adjust_rules:
              try:
                  if rule['enabled']:
                      return True
              except KeyError:
                  pass
      except KeyError:
          pass
      # At this point nothing enables the test
      return False
  
  
  def export_to_nitrate(test: 'tmt.Test') -> None:
      """ Export fmf metadata to nitrate test cases """
      import_nitrate()
      assert nitrate
      assert gssapi
  
      # Check command line options
      create = test.opt('create')
      general = test.opt('general')
      link_runs = test.opt('link_runs')
      duplicate = test.opt('duplicate')
      link_bugzilla = test.opt('bugzilla')
      ignore_git_validation = test.opt('ignore_git_validation')
      dry_mode = test.opt('dry')
  
      if link_runs:
          general = True
  
      # Check git is already correct
      valid, error_msg = tmt.utils.validate_git_status(test)
      if not valid:
          if ignore_git_validation:
              secho(f"Exporting regardless '{error_msg}'.", fg='red')
          else:
              raise ConvertError(
                  f"Can't export due '{error_msg}'.\n"
                  "Use --ignore-git-validation on your own risk to export regardless.")
  
      # Check nitrate test case
      try:
          nitrate_id = test.node.get('extra-nitrate')[3:]
          nitrate_case = nitrate.TestCase(int(nitrate_id))
          nitrate_case.summary  # Make sure we connect to the server now
          echo(style(f"Test case '{nitrate_case.identifier}' found.", fg='blue'))
      except TypeError:
          # Create a new nitrate test case
          if create:
              nitrate_case = None
              # Check for existing Nitrate tests with the same fmf id
              if not duplicate:
                  testcases = _nitrate_find_fmf_testcases(test)
                  try:
                      # Select the first found testcase if any
                      nitrate_case = next(testcases)
                  except StopIteration:
                      pass
              if not nitrate_case:
                  # Summary for TCMS case
                  extra_summary = prepare_extra_summary(test)
                  if not dry_mode:
                      nitrate_case = create_nitrate_case(extra_summary)
                  else:
                      echo(style(
                          f"Test case '{extra_summary}' created.", fg='blue'))
                  test._metadata['extra-summary'] = extra_summary
              # Either newly created or duplicate with missing extra-nitrate
              if nitrate_case:
                  echo(style("Append the nitrate test case id.", fg='green'))
                  if not dry_mode:
                      with test.node as data:
                          data["extra-nitrate"] = nitrate_case.identifier
          else:
              raise ConvertError(f"Nitrate test case id not found for {test}"
                                 " (You can use --create option to enforce"
                                 " creating testcases)")
      except (nitrate.NitrateError, gssapi.raw.misc.GSSError) as error:
          raise ConvertError(error)
  
      # Check if URL is accessible, to be able to reach from nitrate
      check_git_url(test.fmf_id['url'])
  
      # Summary
      try:
          summary = (test._metadata.get('extra-summary')
                     or test._metadata.get('extra-task')
                     or prepare_extra_summary(test))
      except ConvertError:
          summary = test.name
      if not dry_mode:
          nitrate_case.summary = summary
      echo(style('summary: ', fg='green') + summary)
  
      # Script
      if test.node.get('extra-task'):
          if not dry_mode:
              nitrate_case.script = test.node.get('extra-task')
          echo(style('script: ', fg='green') + test.node.get('extra-task'))
  
      # Components and General plan
      # First remove any components that are already there
      if not dry_mode:
          nitrate_case.components.clear()
      # Only these general plans should stay
      expected_general_plans = set()
      # Then add fmf ones
      if test.component:
          echo(style('components: ', fg='green') + ' '.join(test.component))
          for component in test.component:
              try:
                  nitrate_component = nitrate.Component(
                      name=component, product=DEFAULT_PRODUCT.id)
                  if not dry_mode:
                      nitrate_case.components.add(nitrate_component)
              except nitrate.xmlrpc_driver.NitrateError as error:
                  log.debug(error)
                  echo(style(
                      f"Failed to add component '{component}'.", fg='red'))
              if general:
                  try:
                      general_plan = find_general_plan(component)
                      expected_general_plans.add(general_plan)
                      echo(style(
                          f"Linked to general plan '{general_plan}'.",
                          fg='magenta'))
                      if not dry_mode:
                          nitrate_case.testplans.add(general_plan)
                      if link_runs:
                          add_to_nitrate_runs(
                              nitrate_case, general_plan, test, dry_mode)
                  except nitrate.NitrateError as error:
                      log.debug(error)
                      echo(style(
                          f"Failed to find general test plan for '{component}'.",
                          fg='red'))
      # Remove unexpected general plans
      if general and nitrate_case:
          # Remove also all general plans linked to testcase
          for nitrate_plan in [plan for plan in nitrate_case.testplans]:
              if (nitrate_plan.type.name == "General"
                      and nitrate_plan not in expected_general_plans):
                  echo(style(
                      f"Removed general plan '{nitrate_plan}'.", fg='red'))
                  if not dry_mode:
                      nitrate_case.testplans.remove(nitrate_plan)
  
      # Tags
      # Convert 'tier' attribute into a Tier tag
      if test.tier is not None:
          test.tag.append(f"Tier{test.tier}")
      # Add special fmf-export tag
      test.tag.append('fmf-export')
      if not dry_mode:
          nitrate_case.tags.clear()
          nitrate_case.tags.add([nitrate.Tag(tag) for tag in test.tag])
      echo(style('tags: ', fg='green') + ' '.join(set(test.tag)))
  
      # Default tester
      if test.contact:
          try:
              # Need to pick one value, so picking the first contact
              email_address = email.utils.parseaddr(test.contact[0])[1]
              nitrate_user = nitrate.User(email_address)
              nitrate_user._fetch()  # To check that user exists
              if not dry_mode:
                  nitrate_case.tester = nitrate_user
              echo(style('default tester: ', fg='green') + email_address)
          except nitrate.NitrateError as error:
              log.debug(error)
              raise ConvertError(f"Nitrate issue: {error}")
  
      # Duration
      if not dry_mode:
          nitrate_case.time = test.duration
      echo(style('estimated time: ', fg='green') + test.duration)
  
      # Manual
      if not dry_mode:
          nitrate_case.automated = not test.manual
      echo(style('automated: ', fg='green') + ['auto', 'manual'][test.manual])
  
      # Status
      current_status = nitrate_case.status if nitrate_case else nitrate.CaseStatus('CONFIRMED')
      # Enable enabled tests
      if enabled_somewhere(test):
          if not dry_mode:
              nitrate_case.status = nitrate.CaseStatus('CONFIRMED')
          echo(style('status: ', fg='green') + 'CONFIRMED')
      # Disable disabled tests which are CONFIRMED
      elif current_status == nitrate.CaseStatus('CONFIRMED'):
          if not dry_mode:
              nitrate_case.status = nitrate.CaseStatus('DISABLED')
          echo(style('status: ', fg='green') + 'DISABLED')
      # Keep disabled tests in their states
      else:
          echo(style('status: ', fg='green') + str(current_status))
  
      # Environment
      if test.environment:
          environment = ' '.join(tmt.utils.shell_variables(test.environment))
          if not dry_mode:
              nitrate_case.arguments = environment
          echo(style('arguments: ', fg='green') + environment)
      else:
          # FIXME unable clear to set empty arguments
          # (possibly error in xmlrpc, BZ#1805687)
          if not dry_mode:
              nitrate_case.arguments = ' '
          echo(style('arguments: ', fg='green') + "' '")
  
      # Structured Field
      struct_field = tmt.utils.StructuredField(
          nitrate_case.notes if nitrate_case else '')
      echo(style('Structured Field: ', fg='green'))
  
      # Mapping of structured field sections to fmf case attributes
      section_to_attr = {
          'description': test.summary,
          'purpose-file': test.description,
          'hardware': test.node.get('extra-hardware'),
          'pepa': test.node.get('extra-pepa'),
          }
      for section, attribute in section_to_attr.items():
          if attribute is None:
              try:
                  struct_field.remove(section)
              except tmt.utils.StructuredFieldError:
                  pass
          else:
              struct_field.set(section, attribute)
              echo(style(section + ': ', fg='green') + attribute.strip())
  
      # fmf identifer
      fmf_id = tmt.utils.dict_to_yaml(test.fmf_id)
      struct_field.set('fmf', fmf_id)
      echo(style('fmf id:\n', fg='green') + fmf_id.strip())
  
      # Warning
      if WARNING not in struct_field.header():
          struct_field.header(WARNING + struct_field.header())
          echo(style(
              'Add migration warning to the test case notes.', fg='green'))
  
      # ID
      uuid = add_uuid_if_not_defined(test.node, dry=dry_mode)
      if not uuid:
          uuid = test.node.get(ID_KEY)
      struct_field.set(ID_KEY, uuid)
      echo(style(f"Append the ID {uuid}.", fg='green'))
  
      # Saving case.notes with edited StructField
      if not dry_mode:
          nitrate_case.notes = struct_field.save()
  
      # Export manual test instructions from *.md file to nitrate as html
      md_path = return_markdown_file()
      if os.path.exists(md_path):
          step, expect, setup, cleanup = convert_manual_to_nitrate(md_path)
          if not dry_mode:
              nitrate.User()._server.TestCase.store_text(
                  nitrate_case.id, step, expect, setup, cleanup)
          echo(style("manual steps:", fg='green') + f" found in {md_path}")
  
      # List of bugs test verifies
      verifies_bug_ids = []
      for link in test.link.get():
          try:
              bug_id_search = re.search(RE_BUGZILLA_URL, link['verifies'])
              if not bug_id_search:
                  log.debug(f"Did not find bugzila URL in {link['verifies']}")
                  continue
              bug_id = int(bug_id_search.group(1))
              verifies_bug_ids.append(bug_id)
          except Exception as err:
              log.debug(err)
  
      # Add bugs to the Nitrate case
      if verifies_bug_ids:
          echo(style('Verifies bugs: ', fg='green') +
               ', '.join([f"BZ#{b}" for b in verifies_bug_ids]))
      for bug_id in verifies_bug_ids:
          if not dry_mode:
              nitrate_case.bugs.add(nitrate.Bug(bug=int(bug_id)))
  
      # Update nitrate test case
      if not dry_mode:
          nitrate_case.update()
          echo(style("Test case '{0}' successfully exported to nitrate.".format(
              nitrate_case.identifier), fg='magenta'))
  
      # Optionally link Bugzilla to Nitrate case
      if link_bugzilla and verifies_bug_ids:
          if not dry_mode:
              bz_set_coverage(verifies_bug_ids, nitrate_case.id, NITRATE_TRACKER_ID)
  
  
  def export_to_polarion(test: 'tmt.Test') -> None:
      """ Export fmf metadata to a Polarion test case """
      import_polarion()
  
      # Check command line options
      create = test.opt('create')
      link_bugzilla = test.opt('bugzilla')
      project_id = test.opt('project_id')
      dry_mode = test.opt('dry')
      duplicate = test.opt('duplicate')
      link_polarion = test.opt('link_polarion')
  
      polarion_case = None
      if not duplicate:
          polarion_case = get_polarion_case(test.node, project_id)
      summary = prepare_extra_summary(test)
  
      if not polarion_case:
          if create:
              if not project_id:
                  raise ConvertError(
                      "Please provide project_id so tmt knows which "
                      "Polarion project to use for this test case.")
              if not dry_mode:
                  polarion_case = create_polarion_case(
                      summary, project_id=project_id)
              else:
                  echo(style(
                      f"Test case '{summary}' created.", fg='blue'))
              test._metadata['extra-summary'] = summary
          else:
              raise ConvertError(
                  f"Polarion test case id not found for '{test}'. "
                  f"(You can use --create option to enforce creating testcases.)")
  
      # Title
      if not dry_mode and polarion_case.title != test.summary:
          polarion_case.title = test.summary
      echo(style('title: ', fg='green') + test.summary)
  
      # Description
      description = summary
      if test.description:
          description += ' - ' + test.description
      if not dry_mode:
          polarion_case.description = description
      echo(style('description: ', fg='green') + description)
  
      # Automation
      if test.node.get('extra-task'):
          automation_script = test.node.get('extra-task')
          automation_script += ' ' + test.fmf_id['url']
      else:
          automation_script = test.fmf_id['url']
      if not dry_mode:
          polarion_case.caseautomation = 'automated'
          polarion_case.automation_script = automation_script
      echo(style('script: ', fg='green') + automation_script)
  
      # Components
      if not dry_mode:
          polarion_case.caselevel = 'component'
          polarion_case.testtype = 'functional'
          if test.component:
              polarion_case.casecomponent = [test.component[0]]
      echo(style('components: ', fg='green') + ' '.join(test.component))
  
      # Tags and Importance
      if not dry_mode:
          if test.tier is not None:
              if int(test.tier) <= 1:
                  polarion_case.caseimportance = 'high'
              elif int(test.tier) == 2:
                  polarion_case.caseimportance = 'medium'
              else:
                  polarion_case.caseimportance = 'low'
              test.tag.append(f"Tier{test.tier}")
          else:
              polarion_case.caseimportance = 'medium'
  
      test.tag.append('fmf-export')
      if not dry_mode:
          polarion_case.tags = ' '.join(test.tag)
      echo(style('tags: ', fg='green') + ' '.join(set(test.tag)))
  
      # Default tester
      if test.contact:
          # Need to pick one value, so picking the first contact
          email_address = email.utils.parseaddr(test.contact[0])[1]
          login_name = email_address[:email_address.find('@')]
          try:
              if not dry_mode:
                  polarion_case.add_assignee(login_name)
              echo(style('default tester: ', fg='green') + login_name)
          except PolarionException as err:
              log.debug(err)
  
      # Status
      if not dry_mode:
          if test.enabled:
              polarion_case.status = 'approved'
          else:
              polarion_case.status = 'inactive'
      echo(style('enabled: ', fg='green') + str(test.enabled))
  
      echo(style("Append the Polarion test case link.", fg='green'))
      if not dry_mode and link_polarion:
          with test.node as data:
              server_url = str(polarion_case._session._server.url)
              add_link(
                  f'{server_url}{"" if server_url.endswith("/") else "/"}'
                  f'#/project/{polarion_case.project_id}/workitem?id='
                  f'{str(polarion_case.work_item_id)}',
                  data, system=SYSTEM_OTHER, type_='implements')
  
      # List of bugs test verifies
      bug_ids = []
      requirements = []
      for link in test.link.get():
          try:
              bug_ids_search = re.search(RE_BUGZILLA_URL, link['verifies'])
              if bug_ids_search:
                  bug_ids.append(int(bug_ids_search.group(1)))
              else:
                  log.debug('Failed to find bug ID in verifies link')
              polarion_url_search = re.search(RE_POLARION_URL, link['verifies'])
              if polarion_url_search:
                  requirements.append(polarion_url_search.group(1))
              else:
                  log.debug('Failed to find Polarion URL in verifies link')
          except Exception as err:
              log.debug(err)
  
      # Add bugs to the Polarion case
      if not dry_mode:
          polarion_case.tcmsbug = ', '.join(str(bug_ids))
  
      # Add TCMS Case ID to Polarion case
      if test.node.get('extra-nitrate') and not dry_mode:
          tcms_case_id_search = re.search(r'\d+', test.node.get("extra-nitrate"))
          if tcms_case_id_search:
              polarion_case.tcmscaseid = str(int(tcms_case_id_search.group()))
  
      # Add id to Polarion case
      uuid = add_uuid_if_not_defined(test.node, dry=dry_mode)
      if not uuid:
          uuid = test.node.get(ID_KEY)
      if not dry_mode:
          polarion_case.test_case_id = uuid
      echo(style(f"Append the ID {uuid}.", fg='green'))
  
      # Add Requirements to Polarion case
      if not dry_mode:
          for req in requirements:
              polarion_case.add_linked_item(req, 'verifies')
  
      # Update Polarion test case
      if not dry_mode:
          polarion_case.update()
      echo(style(
          f"Test case '{summary}' successfully exported to Polarion.",
          fg='magenta'))
  
      # Optionally link Bugzilla to Polarion case
      if link_bugzilla and bug_ids and not dry_mode:
          case_id = (f"{polarion_case.project_id}/workitem?id="
                     f"{str(polarion_case.work_item_id)}")
          bz_set_coverage(bug_ids, case_id, POLARION_TRACKER_ID)
  
  
  def add_to_nitrate_runs(
          nitrate_case: Any,
          general_plan: Any,
          test: 'tmt.Test',
          dry_mode: bool) -> None:
      """
      Add nitrate test case to all active runs under given general plan
  
      Go down plan tree from general plan, add case and case run to
      all open runs. Try to apply adjust.
      """
      assert nitrate
      for child_plan in nitrate.TestPlan.search(parent=general_plan.id):
          for testrun in child_plan.testruns:
              if testrun.status == nitrate.RunStatus("FINISHED"):
                  continue
              if not enabled_for_environment(test, tcms_notes=testrun.notes):
                  continue
              # nitrate_case is None when --dry and --create are used together
              if not nitrate_case or child_plan not in nitrate_case.testplans:
                  echo(style(f"Link to plan '{child_plan}'.", fg='magenta'))
                  if not dry_mode:
                      nitrate_case.testplans.add(child_plan)
              if not nitrate_case or nitrate_case not in [
                      caserun.testcase for caserun in testrun]:
                  echo(style(f"Link to run '{testrun}'.", fg='magenta'))
                  if not dry_mode:
                      nitrate.CaseRun(testcase=nitrate_case, testrun=testrun)
  
  
  def enabled_for_environment(test: 'tmt.Test', tcms_notes: str) -> bool:
      """ Check whether test is enabled for specified environment """
      field = tmt.utils.StructuredField(tcms_notes)
      context_dict = {}
      try:
          for line in cast(str, field.get('environment')).split('\n'):
              try:
                  dimension, values = line.split('=', maxsplit=2)
                  context_dict[dimension.strip()] = [
                      value.strip() for value in re.split(",|and", values)]
              except ValueError:
                  pass
      except tmt.utils.StructuredFieldError:
          pass
  
      if not context_dict:
          return True
  
      try:
          context = fmf.context.Context(**context_dict)
          test_node = test.node.copy()
          test_node.adjust(context)
          # TODO: remove cast later
          return cast(bool, tmt.Test(node=test_node).enabled)
      except BaseException as exception:
          log.debug(f"Failed to process adjust: {exception}")
          return True
  
  
  def check_md_file_respects_spec(md_path: str) -> List[str]:
      """
      Check that the file respects manual test specification
  
      Return list of warnings, empty list if no problems found.
      """
      warnings_list = []
      # TODO: remove cast lastr
      sections_headings = cast(Dict[str, List[str]], tmt.base.SECTIONS_HEADINGS)
      required_headings = set(sections_headings['Step'] +
                              sections_headings['Expect'])
      values = []
      for _ in list(sections_headings.values()):
          values += _
  
      md_to_html = tmt.utils.markdown_to_html(md_path)
      html_headings_from_file = [i[0] for i in
                                 re.findall('(^<h[1-4]>(.+?)</h[1-4]>$)',
                                            md_to_html,
                                            re.M)]
  
      # No invalid headings in the file w/o headings
      if not html_headings_from_file:
          invalid_headings = []
      else:
          # Find invalid headings in the file
          invalid_headings = [key for key in set(html_headings_from_file)
                              if (key not in values) !=
                              bool(re.search(
                                  sections_headings['Test'][1], key))]
  
      # Remove invalid headings from html_headings_from_file
      for index in invalid_headings:
          warnings_list.append(f'unknown html heading "{index}" is used')
          html_headings_from_file = [i for i in html_headings_from_file
                                     if i != index]
  
      def count_html_headings(heading: str) -> None:
          if html_headings_from_file.count(heading) > 1:
              warnings_list.append(
                  f'{html_headings_from_file.count(heading)}'
                  f' headings "{heading}" are used')
  
      # Warn if 2 or more # Setup or # Cleanup are used
      count_html_headings(sections_headings['Setup'][0])
      count_html_headings(sections_headings['Cleanup'][0])
  
      warn_outside_test_section = 'Heading "{}" from the section "{}" is '\
                                  'used \noutside of Test sections.'
      warn_headings_not_in_pairs = 'The number of headings from the section' \
                                   ' "Step" - {}\ndoesn\'t equal to the ' \
                                   'number of headings from the section \n' \
                                   '"Expect" - {} in the test section "{}"'
      warn_required_section_is_absent = '"{}" section doesn\'t exist in ' \
                                        'the Markdown file'
      warn_unexpected_headings = 'Headings "{}" aren\'t expected in the ' \
                                 'section "{}"'
  
      def required_section_exists(
              section: List[str],
              section_name: str,
              prefix: Union[str, Tuple[str, ...]]) -> int:
          res = list(filter(
              lambda t: t.startswith(prefix), section))
          if not res:
              warnings_list.append(
                  warn_required_section_is_absent.format(section_name))
              return 0
          else:
              return len(res)
  
      # Required sections don't exist
      if not required_section_exists(html_headings_from_file,
                                     'Test',
                                     '<h1>Test'):
          return warnings_list
  
      # Remove Optional heading #Cleanup if it's in the end of document
      if html_headings_from_file[-1] == '<h1>Cleanup</h1>':
          html_headings_from_file.pop()
          # Add # Test heading to close the file
          html_headings_from_file.append(sections_headings['Test'][0])
  
      index = 0
      while html_headings_from_file:
          # # Step cannot be used outside of test sections.
          if html_headings_from_file[index] == \
                  sections_headings['Step'][0] or \
                  html_headings_from_file[index] == \
                  sections_headings['Step'][1]:
              warnings_list.append(warn_outside_test_section.format(
                  html_headings_from_file[index], 'Step'))
  
          # # Expect cannot be used outside of test sections.
          if html_headings_from_file[index] == \
                  sections_headings['Expect'][0] or \
                  html_headings_from_file[index] == \
                  sections_headings['Expect'][1] or \
                  html_headings_from_file[index] == \
                  sections_headings['Expect'][2]:
              warnings_list.append(warn_outside_test_section.format(
                  html_headings_from_file[index], 'Expect'))
  
          if html_headings_from_file[index].startswith('<h1>Test'):
              test_section_name = html_headings_from_file[index]
              try:
                  html_headings_from_file[index + 1]
              except IndexError:
                  break
              for i, v in enumerate(html_headings_from_file[index + 1:]):
                  if re.search('^<h1>(Test .*|Test)</h1>$', v):
                      test_section = html_headings_from_file[index + 1:
                                                             index + 1 + i]
  
                      # Unexpected headings inside Test section
                      unexpected_headings = set(test_section) - \
                          required_headings
                      if unexpected_headings:
                          warnings_list.append(
                              warn_unexpected_headings.
                              format(', '.join(unexpected_headings),
                                     test_section_name))
  
                      amount_of_steps = required_section_exists(
                          test_section,
                          'Step',
                          tuple(sections_headings['Step']))
                      amount_of_expects = required_section_exists(
                          test_section,
                          'Expect',
                          tuple(sections_headings['Expect']))
  
                      # # Step isn't in pair with # Expect
                      if amount_of_steps != amount_of_expects != 0:
                          warnings_list.append(warn_headings_not_in_pairs.
                                               format(amount_of_steps,
                                                      amount_of_expects,
                                                      test_section_name))
                      index += i
                      break
  
          index += 1
          if index >= len(html_headings_from_file) - 1:
              break
      return warnings_list
  
  
  def return_markdown_file() -> str:
      """ Return path to the markdown file """
      files = '\n'.join(os.listdir())
      reg_exp = r'.+\.md$'
      md_files = re.findall(reg_exp, files, re.M)
      fail_message = "in the current working directory.\n" \
                     "Manual steps couldn't be exported"
      if len(md_files) == 1:
          md_path = os.path.join(os.getcwd(), md_files[0])
      elif len(md_files) == 0:
          md_path = ''
          echo((style(f'Markdown file doesn\'t exist {fail_message}',
                      fg='yellow')))
      else:
          md_path = ''
          echo((style(f'{len(md_files)} Markdown files found {fail_message}',
                      fg='yellow')))
      return md_path
  
  
  def get_category() -> str:
      """ Get category from Makefile """
      category = 'Sanity'
      try:
          with open('Makefile', encoding='utf-8') as makefile_file:
              makefile = makefile_file.read()
          category_search = re.search(
              r'echo\s+"Type:\s*(.*)"', makefile, re.M)
          if category_search:
              category = category_search.group(1)
      # Default to 'Sanity' if Makefile or Type not found
      except (IOError, AttributeError):
          pass
      return category
  
  
  def create_nitrate_case(summary: str) -> Any:
      """ Create new nitrate case """
      # Create the new test case
      assert nitrate
      category = nitrate.Category(name=get_category(), product=DEFAULT_PRODUCT)
      testcase = nitrate.TestCase(summary=summary, category=category)
      echo(style(f"Test case '{testcase.identifier}' created.", fg='blue'))
      return testcase
  
  
  def create_polarion_case(summary: str, project_id: str) -> Any:
      """ Create new polarion case """
      # Create the new test case
      testcase = PolarionTestCase.create(project_id, summary, summary)
      testcase.tcmscategory = get_category()
      testcase.update()
      echo(style(f"Test case '{testcase.work_item_id}' created.", fg='blue'))
      return testcase
  
  
  def prepare_extra_summary(test: 'tmt.Test') -> str:
      """ extra-summary for export --create test """
      remote_dirname = re.sub('.git$', '', os.path.basename(test.fmf_id['url']))
      if not remote_dirname:
          raise ConvertError("Unable to find git remote url.")
      generated = f"{remote_dirname} {test.name}"
      if test.summary:
          generated += f" - {test.summary}"
      # TODO: remove cast later
      return cast(str, test.node.get('extra-summary', generated))
  
  
  # avoid multiple searching for general plans (it is expensive)
  @lru_cache(maxsize=None)
  def find_general_plan(component: str) -> Any:
      """ Return single General Test Plan or raise an error """
      assert nitrate
      # At first find by linked components
      found = nitrate.TestPlan.search(
          type__name="General",
          is_active=True,
          component__name=f"{component}")
      # Attempt to find by name if no test plan found
      if not found:
          found = nitrate.TestPlan.search(
              type__name="General",
              is_active=True,
              name=f"{component} / General")
      # No general -> raise error
      if not found:
          raise nitrate.NitrateError(
              f"No general test plan found for '{component}'.")
      # Multiple general plans are fishy -> raise error
      if len(found) != 1:
          nitrate.NitrateError(
              "Multiple general test plans found for '{component}' component.")
      # Finally return the one and only General plan
      return found[0]
  
files: tmt/export.py
files: # coding: utf-8
  
  """ Export metadata into nitrate """
  
  
  import email.utils
  import os
  import re
  import traceback
  import types
  import xmlrpc.client
  from functools import lru_cache
  from typing import Any, Dict, Generator, List, Optional, Tuple, Union, cast
  
  import fmf
  from click import echo, secho, style
  
  import tmt
  import tmt.utils
  from tmt.convert import SYSTEM_OTHER, add_link
  from tmt.identifier import ID_KEY, add_uuid_if_not_defined
  from tmt.utils import ConvertError, check_git_url, markdown_to_html
  
  bugzilla: Optional[types.ModuleType] = None
  gssapi: Optional[types.ModuleType] = None
  nitrate: Optional[types.ModuleType] = None
  
  PolarionException: Any = None
  PolarionTestCase: Any = None
  PolarionWorkItem: Any = None
  
  DEFAULT_PRODUCT: Any = None
  
  SectionsReturnType = Tuple[str, str, str, str]
  HeadingsType = List[List[Union[int, str]]]
  SectionsHeadingsType = Dict[str, HeadingsType]
  
  log = fmf.utils.Logging('tmt').logger
  
  WARNING = """
  Test case has been migrated to git. Any changes made here might be overwritten.
  See: https://tmt.readthedocs.io/en/latest/questions.html#nitrate-migration
  """.lstrip()
  
  # For linking bugs
  BUGZILLA_XMLRPC_URL = "https://bugzilla.redhat.com/xmlrpc.cgi"
  NITRATE_TRACKER_ID = 69  # ID of nitrate in RH's bugzilla
  POLARION_TRACKER_ID = 117  # ID of polarion in RH's bugzilla
  RE_BUGZILLA_URL = r'bugzilla.redhat.com/show_bug.cgi\?id=(\d+)'
  RE_POLARION_URL = r'.*/polarion/#/project/.*/workitem\?id=(.*)'
  LEGACY_POLARION_PROJECTS = set(['RedHatEnterpriseLinux7'])
  
  
  def import_nitrate() -> Any:
      """ Conditionally import the nitrate module """
      # Need to import nitrate only when really needed. Otherwise we get
      # traceback when nitrate not installed or config file not available.
      # And we want to keep the core tmt package with minimal dependencies.
      try:
          global nitrate, DEFAULT_PRODUCT, gssapi
          import gssapi
          import nitrate
          assert nitrate
          DEFAULT_PRODUCT = nitrate.Product(name='RHEL Tests')
          return nitrate
      except ImportError:
          raise ConvertError(
              "Install tmt-test-convert to export tests to nitrate.")
      except nitrate.NitrateError as error:  # type: ignore
          raise ConvertError(error)
  
  
  def import_polarion() -> None:
      """ Import polarion python api - pylero """
      try:
          global PolarionException, PolarionTestCase, PolarionWorkItem
          from pylero.exceptions import PyleroLibException as PolarionException
          from pylero.work_item import TestCase as PolarionTestCase
          from pylero.work_item import _WorkItem as PolarionWorkItem
      except ImportError:
          raise ConvertError(
              "Run 'pip install tmt[export-polarion]' so pylero is installed.")
      except PolarionException as exc:
          log.debug(traceback.format_exc())
          raise ConvertError("Failed to login with pylero", original=exc)
  
  
  def get_bz_instance() -> Any:
      """ Import the bugzilla module and return BZ instance """
      try:
          import bugzilla
      except ImportError:
          raise ConvertError(
              "Install 'tmt-test-convert' to link test to the bugzilla.")
  
      try:
          bz_instance = bugzilla.Bugzilla(url=BUGZILLA_XMLRPC_URL)
      except Exception as exc:
          log.debug(traceback.format_exc())
          raise ConvertError(
              "Couldn't initialize the Bugzilla client.", original=exc)
  
      if not bz_instance.logged_in:
          raise ConvertError(
              "Not logged to Bugzilla, check 'man bugzilla' section "
              "'AUTHENTICATION CACHE AND API KEYS'.")
      return bz_instance
  
  
  def _nitrate_find_fmf_testcases(test: 'tmt.Test') -> Generator[Any, None, None]:
      """
      Find all Nitrate test cases with the same fmf identifier
  
      All component general plans are explored for possible duplicates.
      """
      assert nitrate
      for component in test.component:
          try:
              for testcase in find_general_plan(component).testcases:
                  struct_field = tmt.utils.StructuredField(testcase.notes)
                  try:
                      fmf_id = tmt.utils.yaml_to_dict(struct_field.get('fmf'))
                      if fmf_id == test.fmf_id:
                          echo(style(
                              f"Existing test case '{testcase.identifier}' "
                              f"found for given fmf id.", fg='magenta'))
                          yield testcase
                  except tmt.utils.StructuredFieldError:
                      pass
          except nitrate.NitrateError:
              pass
  
  
  def convert_manual_to_nitrate(test_md: str) -> SectionsReturnType:
      """
      Convert Markdown document to html sections.
  
      These sections can be exported to nitrate.
      Expects: Markdown document as a file.
      Returns: tuple of (step, expect, setup, cleanup) sections
      as html strings.
      """
  
      sections_headings: SectionsHeadingsType = {
          heading: []
          for heading_list in tmt.base.SECTIONS_HEADINGS.values()
          for heading in heading_list
          }
  
      html = markdown_to_html(test_md)
      html_splitlines = html.splitlines()
  
      for key in sections_headings.keys():
          result: HeadingsType = []
          i = 0
          while html_splitlines:
              try:
                  if re.search("^" + key + "$", html_splitlines[i]):
                      html_content = str()
                      if key.startswith('<h1>Test'):
                          html_content = html_splitlines[i].\
                              replace('<h1>', '<b>').\
                              replace('</h1>', '</b>')
                      for j in range(i + 1, len(html_splitlines)):
                          if re.search("^<h[1-4]>(.+?)</h[1-4]>$",
                                       html_splitlines[j]):
                              result.append([i, html_content])
                              i = j - 1
                              break
                          html_content += html_splitlines[j] + "\n"
                          # Check end of the file
                          if j + 1 == len(html_splitlines):
                              result.append([i, html_content])
              except IndexError:
                  sections_headings[key] = result
                  break
              i += 1
              if i >= len(html_splitlines):
                  sections_headings[key] = result
                  break
  
      def concatenate_headings_content(headings: Tuple[str, ...]) -> HeadingsType:
          content = list()
          for v in headings:
              content += sections_headings[v]
          return content
  
      def enumerate_content(content: HeadingsType) -> HeadingsType:
          # for sorting convert the index to integer, but keep whole list as list of strings
          content.sort(key=lambda a: int(a[0]))
          for c in range(len(content)):
              content[c][1] = f"<p>Step {c + 1}.</p>" + str(content[c][1])
          return content
  
      sorted_test = sorted(concatenate_headings_content((
          '<h1>Test</h1>',
          '<h1>Test .*</h1>')))
  
      sorted_step = sorted(enumerate_content(concatenate_headings_content((
          '<h2>Step</h2>',
          '<h2>Test Step</h2>'))) + sorted_test)
      step = ''.join([f"{v[1]}" for v in sorted_step])
  
      sorted_expect = sorted(enumerate_content(concatenate_headings_content((
          '<h2>Expect</h2>',
          '<h2>Result</h2>',
          '<h2>Expected Result</h2>'))) + sorted_test)
      expect = ''.join([f"{v[1]}" for v in sorted_expect])
  
      def check_section_exists(text: str) -> str:
          try:
              return str(sections_headings[text][0][1])
          except (IndexError, KeyError):
              return ''
  
      setup = check_section_exists('<h1>Setup</h1>')
      cleanup = check_section_exists('<h1>Cleanup</h1>')
  
      return step, expect, setup, cleanup
  
  
  def bz_set_coverage(bug_ids: List[int], case_id: str, tracker_id: int) -> None:
      """ Set coverage in Bugzilla """
      bz_instance = get_bz_instance()
  
      overall_pass = True
      no_email = 1  # Do not send emails about the change
      get_bz_dict = {
          'ids': bug_ids,
          'include_fields': ['id', 'external_bugs', 'flags']}
      bugs_data = bz_instance._proxy.Bug.get(get_bz_dict)
      for bug in bugs_data['bugs']:
          # Process flag (might fail for some types)
          bug_id = bug['id']
          if 'qe_test_coverage+' not in set(
                  [x['name'] + x['status'] for x in bug['flags']]):
              try:
                  bz_instance._proxy.Flag.update({
                      'ids': [bug_id],
                      'nomail': no_email,
                      'updates': [{
                          'name': 'qe_test_coverage',
                          'status': '+'
                          }]
                      })
              except xmlrpc.client.Fault as err:
                  # TODO: Fix missing overall_result = False, breaks tests
                  # if bug used for testing is not changed
                  # BZ#1925518 can't have qe_test_coverage flag
                  log.debug(f"Update flag failed: {err}")
                  echo(style(
                      f"Failed to set qe_test_coverage+ flag for BZ#{bug_id}",
                      fg='red'))
          # Process external tracker - should succeed
          current = set(
              [b['ext_bz_bug_id'] for b in bug['external_bugs']
               if b['ext_bz_id'] == tracker_id])
          if case_id not in current:
              query = {
                  'bug_ids': [bug_id],
                  'nomail': no_email,
                  'external_bugs': [{
                      'ext_type_id': tracker_id,
                      'ext_bz_bug_id': case_id,
                      'ext_description': '',
                      }]
                  }
              try:
                  bz_instance._proxy.ExternalBugs.add_external_bug(query)
              except Exception as err:
                  log.debug(f"Link case failed: {err}")
                  echo(style(f"Failed to link to BZ#{bug_id}", fg='red'))
                  overall_pass = False
      if not overall_pass:
          raise ConvertError("Failed to link the case to bugs.")
  
      echo(style("Linked to Bugzilla: {}.".format(
          " ".join([f"BZ#{bz_id}" for bz_id in bug_ids])), fg='magenta'))
  
  
  def get_polarion_ids(
          query_result: List[Any],
          preferred_project: Optional[str] = None) -> Tuple[str, Optional[str]]:
      """ Return case and project ids from query results """
      if not query_result:
          return 'None', None
      if len(query_result) == 1:
          return query_result[0].work_item_id, query_result[0].project_id
  
      if preferred_project:
          try:
              return next(
                  item.work_item_id for item in query_result
                  if item.project_id == preferred_project), preferred_project
          except StopIteration:
              pass
  
      for result in query_result:
          # If multiple cases are found prefer cases from other projects
          # than these legacy ones
          if str(result.project_id) not in LEGACY_POLARION_PROJECTS:
              return result.work_item_id, result.project_id
  
      return query_result[0].work_item_id, query_result[0].project_id
  
  
  def find_polarion_case_ids(
          data: Dict[str, str],
          preferred_project: Optional[str] = None) -> Tuple[str, Optional[str]]:
      """ Find IDs for Polarion case from data dictionary """
      assert PolarionWorkItem
  
      # Search by UUID
      if data.get(ID_KEY):
          query_result = PolarionWorkItem.query(
              data.get(ID_KEY), fields=['work_item_id', 'project_id'])
          return get_polarion_ids(query_result, preferred_project)
      # Search by TCMS Case ID
      extra_nitrate = data.get('extra-nitrate')
      if extra_nitrate:
          nitrate_case_id_search = re.search(r'\d+', extra_nitrate)
          if not nitrate_case_id_search:
              raise ConvertError(
                  "Could not find a valid nitrate testcase ID in 'extra-nitrate' attribute")
          nitrate_case_id = str(int(nitrate_case_id_search.group()))
          query_result = PolarionWorkItem.query(
              f"tcmscaseid:{nitrate_case_id}", fields=['work_item_id', 'project_id'])
          return get_polarion_ids(query_result, preferred_project)
      # Search by extra task
      if data.get('extra-task'):
          query_result = PolarionWorkItem.query(
              data.get('extra-task'), fields=['work_item_id', 'project_id'])
          return get_polarion_ids(query_result, preferred_project)
      return 'None', None
  
  
  def get_polarion_case(data: Dict[str, str], preferred_project: Optional[str] = None) -> Any:
      """ Get Polarion case through couple different methods """
      import_polarion()
  
      assert PolarionTestCase
      assert PolarionException
  
      polarion_id, project_id = find_polarion_case_ids(data, preferred_project)
  
      try:
          polarion_case = PolarionTestCase(
              project_id=project_id, work_item_id=polarion_id)
          echo(style(
              f"Test case '{str(polarion_case.work_item_id)}' found.",
              fg='blue'))
          return polarion_case
      except PolarionException:
          return None
  
  
  def enabled_somewhere(test: 'tmt.Test') -> bool:
      """ True if the test is enabled for some context (adjust rules) """
      # node.original_data are fmf data _before_ adjust was processed
      try:
          if test.node.original_data['enabled']:
              return True
      except KeyError:
          # Key defaults to True so we already know the outcome
          return True
  
      # Some rule in adjust enables the test
      try:
          adjust_rules = test.node.original_data['adjust']
          # TODO: Should not be necessary once we normalize data
          if isinstance(adjust_rules, dict):
              adjust_rules = [adjust_rules]
          for rule in adjust_rules:
              try:
                  if rule['enabled']:
                      return True
              except KeyError:
                  pass
      except KeyError:
          pass
      # At this point nothing enables the test
      return False
  
  
  def export_to_nitrate(test: 'tmt.Test') -> None:
      """ Export fmf metadata to nitrate test cases """
      import_nitrate()
      assert nitrate
      assert gssapi
  
      # Check command line options
      create = test.opt('create')
      general = test.opt('general')
      link_runs = test.opt('link_runs')
      duplicate = test.opt('duplicate')
      link_bugzilla = test.opt('bugzilla')
      ignore_git_validation = test.opt('ignore_git_validation')
      dry_mode = test.opt('dry')
  
      if link_runs:
          general = True
  
      # Check git is already correct
      valid, error_msg = tmt.utils.validate_git_status(test)
      if not valid:
          if ignore_git_validation:
              secho(f"Exporting regardless '{error_msg}'.", fg='red')
          else:
              raise ConvertError(
                  f"Can't export due '{error_msg}'.\n"
                  "Use --ignore-git-validation on your own risk to export regardless.")
  
      # Check nitrate test case
      try:
          nitrate_id = test.node.get('extra-nitrate')[3:]
          nitrate_case = nitrate.TestCase(int(nitrate_id))
          nitrate_case.summary  # Make sure we connect to the server now
          echo(style(f"Test case '{nitrate_case.identifier}' found.", fg='blue'))
      except TypeError:
          # Create a new nitrate test case
          if create:
              nitrate_case = None
              # Check for existing Nitrate tests with the same fmf id
              if not duplicate:
                  testcases = _nitrate_find_fmf_testcases(test)
                  try:
                      # Select the first found testcase if any
                      nitrate_case = next(testcases)
                  except StopIteration:
                      pass
              if not nitrate_case:
                  # Summary for TCMS case
                  extra_summary = prepare_extra_summary(test)
                  if not dry_mode:
                      nitrate_case = create_nitrate_case(extra_summary)
                  else:
                      echo(style(
                          f"Test case '{extra_summary}' created.", fg='blue'))
                  test._metadata['extra-summary'] = extra_summary
              # Either newly created or duplicate with missing extra-nitrate
              if nitrate_case:
                  echo(style("Append the nitrate test case id.", fg='green'))
                  if not dry_mode:
                      with test.node as data:
                          data["extra-nitrate"] = nitrate_case.identifier
          else:
              raise ConvertError(f"Nitrate test case id not found for {test}"
                                 " (You can use --create option to enforce"
                                 " creating testcases)")
      except (nitrate.NitrateError, gssapi.raw.misc.GSSError) as error:
          raise ConvertError(error)
  
      # Check if URL is accessible, to be able to reach from nitrate
      check_git_url(test.fmf_id['url'])
  
      # Summary
      try:
          summary = (test._metadata.get('extra-summary')
                     or test._metadata.get('extra-task')
                     or prepare_extra_summary(test))
      except ConvertError:
          summary = test.name
      if not dry_mode:
          nitrate_case.summary = summary
      echo(style('summary: ', fg='green') + summary)
  
      # Script
      if test.node.get('extra-task'):
          if not dry_mode:
              nitrate_case.script = test.node.get('extra-task')
          echo(style('script: ', fg='green') + test.node.get('extra-task'))
  
      # Components and General plan
      # First remove any components that are already there
      if not dry_mode:
          nitrate_case.components.clear()
      # Only these general plans should stay
      expected_general_plans = set()
      # Then add fmf ones
      if test.component:
          echo(style('components: ', fg='green') + ' '.join(test.component))
          for component in test.component:
              try:
                  nitrate_component = nitrate.Component(
                      name=component, product=DEFAULT_PRODUCT.id)
                  if not dry_mode:
                      nitrate_case.components.add(nitrate_component)
              except nitrate.xmlrpc_driver.NitrateError as error:
                  log.debug(error)
                  echo(style(
                      f"Failed to add component '{component}'.", fg='red'))
              if general:
                  try:
                      general_plan = find_general_plan(component)
                      expected_general_plans.add(general_plan)
                      echo(style(
                          f"Linked to general plan '{general_plan}'.",
                          fg='magenta'))
                      if not dry_mode:
                          nitrate_case.testplans.add(general_plan)
                      if link_runs:
                          add_to_nitrate_runs(
                              nitrate_case, general_plan, test, dry_mode)
                  except nitrate.NitrateError as error:
                      log.debug(error)
                      echo(style(
                          f"Failed to find general test plan for '{component}'.",
                          fg='red'))
      # Remove unexpected general plans
      if general and nitrate_case:
          # Remove also all general plans linked to testcase
          for nitrate_plan in [plan for plan in nitrate_case.testplans]:
              if (nitrate_plan.type.name == "General"
                      and nitrate_plan not in expected_general_plans):
                  echo(style(
                      f"Removed general plan '{nitrate_plan}'.", fg='red'))
                  if not dry_mode:
                      nitrate_case.testplans.remove(nitrate_plan)
  
      # Tags
      # Convert 'tier' attribute into a Tier tag
      if test.tier is not None:
          test.tag.append(f"Tier{test.tier}")
      # Add special fmf-export tag
      test.tag.append('fmf-export')
      if not dry_mode:
          nitrate_case.tags.clear()
          nitrate_case.tags.add([nitrate.Tag(tag) for tag in test.tag])
      echo(style('tags: ', fg='green') + ' '.join(set(test.tag)))
  
      # Default tester
      if test.contact:
          try:
              # Need to pick one value, so picking the first contact
              email_address = email.utils.parseaddr(test.contact[0])[1]
              nitrate_user = nitrate.User(email_address)
              nitrate_user._fetch()  # To check that user exists
              if not dry_mode:
                  nitrate_case.tester = nitrate_user
              echo(style('default tester: ', fg='green') + email_address)
          except nitrate.NitrateError as error:
              log.debug(error)
              raise ConvertError(f"Nitrate issue: {error}")
  
      # Duration
      if not dry_mode:
          nitrate_case.time = test.duration
      echo(style('estimated time: ', fg='green') + test.duration)
  
      # Manual
      if not dry_mode:
          nitrate_case.automated = not test.manual
      echo(style('automated: ', fg='green') + ['auto', 'manual'][test.manual])
  
      # Status
      current_status = nitrate_case.status if nitrate_case else nitrate.CaseStatus('CONFIRMED')
      # Enable enabled tests
      if enabled_somewhere(test):
          if not dry_mode:
              nitrate_case.status = nitrate.CaseStatus('CONFIRMED')
          echo(style('status: ', fg='green') + 'CONFIRMED')
      # Disable disabled tests which are CONFIRMED
      elif current_status == nitrate.CaseStatus('CONFIRMED'):
          if not dry_mode:
              nitrate_case.status = nitrate.CaseStatus('DISABLED')
          echo(style('status: ', fg='green') + 'DISABLED')
      # Keep disabled tests in their states
      else:
          echo(style('status: ', fg='green') + str(current_status))
  
      # Environment
      if test.environment:
          environment = ' '.join(tmt.utils.shell_variables(test.environment))
          if not dry_mode:
              nitrate_case.arguments = environment
          echo(style('arguments: ', fg='green') + environment)
      else:
          # FIXME unable clear to set empty arguments
          # (possibly error in xmlrpc, BZ#1805687)
          if not dry_mode:
              nitrate_case.arguments = ' '
          echo(style('arguments: ', fg='green') + "' '")
  
      # Structured Field
      struct_field = tmt.utils.StructuredField(
          nitrate_case.notes if nitrate_case else '')
      echo(style('Structured Field: ', fg='green'))
  
      # Mapping of structured field sections to fmf case attributes
      section_to_attr = {
          'description': test.summary,
          'purpose-file': test.description,
          'hardware': test.node.get('extra-hardware'),
          'pepa': test.node.get('extra-pepa'),
          }
      for section, attribute in section_to_attr.items():
          if attribute is None:
              try:
                  struct_field.remove(section)
              except tmt.utils.StructuredFieldError:
                  pass
          else:
              struct_field.set(section, attribute)
              echo(style(section + ': ', fg='green') + attribute.strip())
  
      # fmf identifer
      fmf_id = tmt.utils.dict_to_yaml(test.fmf_id)
      struct_field.set('fmf', fmf_id)
      echo(style('fmf id:\n', fg='green') + fmf_id.strip())
  
      # Warning
      if WARNING not in struct_field.header():
          struct_field.header(WARNING + struct_field.header())
          echo(style(
              'Add migration warning to the test case notes.', fg='green'))
  
      # ID
      uuid = add_uuid_if_not_defined(test.node, dry=dry_mode)
      if not uuid:
          uuid = test.node.get(ID_KEY)
      struct_field.set(ID_KEY, uuid)
      echo(style(f"Append the ID {uuid}.", fg='green'))
  
      # Saving case.notes with edited StructField
      if not dry_mode:
          nitrate_case.notes = struct_field.save()
  
      # Export manual test instructions from *.md file to nitrate as html
      md_path = return_markdown_file()
      if os.path.exists(md_path):
          step, expect, setup, cleanup = convert_manual_to_nitrate(md_path)
          if not dry_mode:
              nitrate.User()._server.TestCase.store_text(
                  nitrate_case.id, step, expect, setup, cleanup)
          echo(style("manual steps:", fg='green') + f" found in {md_path}")
  
      # List of bugs test verifies
      verifies_bug_ids = []
      for link in test.link.get():
          try:
              bug_id_search = re.search(RE_BUGZILLA_URL, link['verifies'])
              if not bug_id_search:
                  log.debug(f"Did not find bugzila URL in {link['verifies']}")
                  continue
              bug_id = int(bug_id_search.group(1))
              verifies_bug_ids.append(bug_id)
          except Exception as err:
              log.debug(err)
  
      # Add bugs to the Nitrate case
      if verifies_bug_ids:
          echo(style('Verifies bugs: ', fg='green') +
               ', '.join([f"BZ#{b}" for b in verifies_bug_ids]))
      for bug_id in verifies_bug_ids:
          if not dry_mode:
              nitrate_case.bugs.add(nitrate.Bug(bug=int(bug_id)))
  
      # Update nitrate test case
      if not dry_mode:
          nitrate_case.update()
          echo(style("Test case '{0}' successfully exported to nitrate.".format(
              nitrate_case.identifier), fg='magenta'))
  
      # Optionally link Bugzilla to Nitrate case
      if link_bugzilla and verifies_bug_ids:
          if not dry_mode:
              bz_set_coverage(verifies_bug_ids, nitrate_case.id, NITRATE_TRACKER_ID)
  
  
  def export_to_polarion(test: 'tmt.Test') -> None:
      """ Export fmf metadata to a Polarion test case """
      import_polarion()
  
      # Check command line options
      create = test.opt('create')
      link_bugzilla = test.opt('bugzilla')
      project_id = test.opt('project_id')
      dry_mode = test.opt('dry')
      duplicate = test.opt('duplicate')
      link_polarion = test.opt('link_polarion')
  
      polarion_case = None
      if not duplicate:
          polarion_case = get_polarion_case(test.node, project_id)
      summary = prepare_extra_summary(test)
  
      if not polarion_case:
          if create:
              if not project_id:
                  raise ConvertError(
                      "Please provide project_id so tmt knows which "
                      "Polarion project to use for this test case.")
              if not dry_mode:
                  polarion_case = create_polarion_case(
                      summary, project_id=project_id)
              else:
                  echo(style(
                      f"Test case '{summary}' created.", fg='blue'))
              test._metadata['extra-summary'] = summary
          else:
              raise ConvertError(
                  f"Polarion test case id not found for '{test}'. "
                  f"(You can use --create option to enforce creating testcases.)")
  
      # Title
      if not dry_mode and polarion_case.title != test.summary:
          polarion_case.title = test.summary
      echo(style('title: ', fg='green') + test.summary)
  
      # Description
      description = summary
      if test.description:
          description += ' - ' + test.description
      if not dry_mode:
          polarion_case.description = description
      echo(style('description: ', fg='green') + description)
  
      # Automation
      if test.node.get('extra-task'):
          automation_script = test.node.get('extra-task')
          automation_script += ' ' + test.fmf_id['url']
      else:
          automation_script = test.fmf_id['url']
      if not dry_mode:
          polarion_case.caseautomation = 'automated'
          polarion_case.automation_script = automation_script
      echo(style('script: ', fg='green') + automation_script)
  
      # Components
      if not dry_mode:
          polarion_case.caselevel = 'component'
          polarion_case.testtype = 'functional'
          if test.component:
              polarion_case.casecomponent = [test.component[0]]
      echo(style('components: ', fg='green') + ' '.join(test.component))
  
      # Tags and Importance
      if not dry_mode:
          if test.tier is not None:
              if int(test.tier) <= 1:
                  polarion_case.caseimportance = 'high'
              elif int(test.tier) == 2:
                  polarion_case.caseimportance = 'medium'
              else:
                  polarion_case.caseimportance = 'low'
              test.tag.append(f"Tier{test.tier}")
          else:
              polarion_case.caseimportance = 'medium'
  
      test.tag.append('fmf-export')
      if not dry_mode:
          polarion_case.tags = ' '.join(test.tag)
      echo(style('tags: ', fg='green') + ' '.join(set(test.tag)))
  
      # Default tester
      if test.contact:
          # Need to pick one value, so picking the first contact
          email_address = email.utils.parseaddr(test.contact[0])[1]
          login_name = email_address[:email_address.find('@')]
          try:
              if not dry_mode:
                  polarion_case.add_assignee(login_name)
              echo(style('default tester: ', fg='green') + login_name)
          except PolarionException as err:
              log.debug(err)
  
      # Status
      if not dry_mode:
          if test.enabled:
              polarion_case.status = 'approved'
          else:
              polarion_case.status = 'inactive'
      echo(style('enabled: ', fg='green') + str(test.enabled))
  
      echo(style("Append the Polarion test case link.", fg='green'))
      if not dry_mode and link_polarion:
          with test.node as data:
              server_url = str(polarion_case._session._server.url)
              add_link(
                  f'{server_url}{"" if server_url.endswith("/") else "/"}'
                  f'#/project/{polarion_case.project_id}/workitem?id='
                  f'{str(polarion_case.work_item_id)}',
                  data, system=SYSTEM_OTHER, type_='implements')
  
      # List of bugs test verifies
      bug_ids = []
      requirements = []
      for link in test.link.get():
          try:
              bug_ids_search = re.search(RE_BUGZILLA_URL, link['verifies'])
              if bug_ids_search:
                  bug_ids.append(int(bug_ids_search.group(1)))
              else:
                  log.debug('Failed to find bug ID in verifies link')
              polarion_url_search = re.search(RE_POLARION_URL, link['verifies'])
              if polarion_url_search:
                  requirements.append(polarion_url_search.group(1))
              else:
                  log.debug('Failed to find Polarion URL in verifies link')
          except Exception as err:
              log.debug(err)
  
      # Add bugs to the Polarion case
      if not dry_mode:
          polarion_case.tcmsbug = ', '.join(str(bug_ids))
  
      # Add TCMS Case ID to Polarion case
      if test.node.get('extra-nitrate') and not dry_mode:
          tcms_case_id_search = re.search(r'\d+', test.node.get("extra-nitrate"))
          if tcms_case_id_search:
              polarion_case.tcmscaseid = str(int(tcms_case_id_search.group()))
  
      # Add id to Polarion case
      uuid = add_uuid_if_not_defined(test.node, dry=dry_mode)
      if not uuid:
          uuid = test.node.get(ID_KEY)
      if not dry_mode:
          polarion_case.test_case_id = uuid
      echo(style(f"Append the ID {uuid}.", fg='green'))
  
      # Add Requirements to Polarion case
      if not dry_mode:
          for req in requirements:
              polarion_case.add_linked_item(req, 'verifies')
  
      # Update Polarion test case
      if not dry_mode:
          polarion_case.update()
      echo(style(
          f"Test case '{summary}' successfully exported to Polarion.",
          fg='magenta'))
  
      # Optionally link Bugzilla to Polarion case
      if link_bugzilla and bug_ids and not dry_mode:
          case_id = (f"{polarion_case.project_id}/workitem?id="
                     f"{str(polarion_case.work_item_id)}")
          bz_set_coverage(bug_ids, case_id, POLARION_TRACKER_ID)
  
  
  def add_to_nitrate_runs(
          nitrate_case: Any,
          general_plan: Any,
          test: 'tmt.Test',
          dry_mode: bool) -> None:
      """
      Add nitrate test case to all active runs under given general plan
  
      Go down plan tree from general plan, add case and case run to
      all open runs. Try to apply adjust.
      """
      assert nitrate
      for child_plan in nitrate.TestPlan.search(parent=general_plan.id):
          for testrun in child_plan.testruns:
              if testrun.status == nitrate.RunStatus("FINISHED"):
                  continue
              if not enabled_for_environment(test, tcms_notes=testrun.notes):
                  continue
              # nitrate_case is None when --dry and --create are used together
              if not nitrate_case or child_plan not in nitrate_case.testplans:
                  echo(style(f"Link to plan '{child_plan}'.", fg='magenta'))
                  if not dry_mode:
                      nitrate_case.testplans.add(child_plan)
              if not nitrate_case or nitrate_case not in [
                      caserun.testcase for caserun in testrun]:
                  echo(style(f"Link to run '{testrun}'.", fg='magenta'))
                  if not dry_mode:
                      nitrate.CaseRun(testcase=nitrate_case, testrun=testrun)
  
  
  def enabled_for_environment(test: 'tmt.Test', tcms_notes: str) -> bool:
      """ Check whether test is enabled for specified environment """
      field = tmt.utils.StructuredField(tcms_notes)
      context_dict = {}
      try:
          for line in cast(str, field.get('environment')).split('\n'):
              try:
                  dimension, values = line.split('=', maxsplit=2)
                  context_dict[dimension.strip()] = [
                      value.strip() for value in re.split(",|and", values)]
              except ValueError:
                  pass
      except tmt.utils.StructuredFieldError:
          pass
  
      if not context_dict:
          return True
  
      try:
          context = fmf.context.Context(**context_dict)
          test_node = test.node.copy()
          test_node.adjust(context)
          # TODO: remove cast later
          return cast(bool, tmt.Test(node=test_node).enabled)
      except BaseException as exception:
          log.debug(f"Failed to process adjust: {exception}")
          return True
  
  
  def check_md_file_respects_spec(md_path: str) -> List[str]:
      """
      Check that the file respects manual test specification
  
      Return list of warnings, empty list if no problems found.
      """
      warnings_list = []
      # TODO: remove cast lastr
      sections_headings = cast(Dict[str, List[str]], tmt.base.SECTIONS_HEADINGS)
      required_headings = set(sections_headings['Step'] +
                              sections_headings['Expect'])
      values = []
      for _ in list(sections_headings.values()):
          values += _
  
      md_to_html = tmt.utils.markdown_to_html(md_path)
      html_headings_from_file = [i[0] for i in
                                 re.findall('(^<h[1-4]>(.+?)</h[1-4]>$)',
                                            md_to_html,
                                            re.M)]
  
      # No invalid headings in the file w/o headings
      if not html_headings_from_file:
          invalid_headings = []
      else:
          # Find invalid headings in the file
          invalid_headings = [key for key in set(html_headings_from_file)
                              if (key not in values) !=
                              bool(re.search(
                                  sections_headings['Test'][1], key))]
  
      # Remove invalid headings from html_headings_from_file
      for index in invalid_headings:
          warnings_list.append(f'unknown html heading "{index}" is used')
          html_headings_from_file = [i for i in html_headings_from_file
                                     if i != index]
  
      def count_html_headings(heading: str) -> None:
          if html_headings_from_file.count(heading) > 1:
              warnings_list.append(
                  f'{html_headings_from_file.count(heading)}'
                  f' headings "{heading}" are used')
  
      # Warn if 2 or more # Setup or # Cleanup are used
      count_html_headings(sections_headings['Setup'][0])
      count_html_headings(sections_headings['Cleanup'][0])
  
      warn_outside_test_section = 'Heading "{}" from the section "{}" is '\
                                  'used \noutside of Test sections.'
      warn_headings_not_in_pairs = 'The number of headings from the section' \
                                   ' "Step" - {}\ndoesn\'t equal to the ' \
                                   'number of headings from the section \n' \
                                   '"Expect" - {} in the test section "{}"'
      warn_required_section_is_absent = '"{}" section doesn\'t exist in ' \
                                        'the Markdown file'
      warn_unexpected_headings = 'Headings "{}" aren\'t expected in the ' \
                                 'section "{}"'
  
      def required_section_exists(
              section: List[str],
              section_name: str,
              prefix: Union[str, Tuple[str, ...]]) -> int:
          res = list(filter(
              lambda t: t.startswith(prefix), section))
          if not res:
              warnings_list.append(
                  warn_required_section_is_absent.format(section_name))
              return 0
          else:
              return len(res)
  
      # Required sections don't exist
      if not required_section_exists(html_headings_from_file,
                                     'Test',
                                     '<h1>Test'):
          return warnings_list
  
      # Remove Optional heading #Cleanup if it's in the end of document
      if html_headings_from_file[-1] == '<h1>Cleanup</h1>':
          html_headings_from_file.pop()
          # Add # Test heading to close the file
          html_headings_from_file.append(sections_headings['Test'][0])
  
      index = 0
      while html_headings_from_file:
          # # Step cannot be used outside of test sections.
          if html_headings_from_file[index] == \
                  sections_headings['Step'][0] or \
                  html_headings_from_file[index] == \
                  sections_headings['Step'][1]:
              warnings_list.append(warn_outside_test_section.format(
                  html_headings_from_file[index], 'Step'))
  
          # # Expect cannot be used outside of test sections.
          if html_headings_from_file[index] == \
                  sections_headings['Expect'][0] or \
                  html_headings_from_file[index] == \
                  sections_headings['Expect'][1] or \
                  html_headings_from_file[index] == \
                  sections_headings['Expect'][2]:
              warnings_list.append(warn_outside_test_section.format(
                  html_headings_from_file[index], 'Expect'))
  
          if html_headings_from_file[index].startswith('<h1>Test'):
              test_section_name = html_headings_from_file[index]
              try:
                  html_headings_from_file[index + 1]
              except IndexError:
                  break
              for i, v in enumerate(html_headings_from_file[index + 1:]):
                  if re.search('^<h1>(Test .*|Test)</h1>$', v):
                      test_section = html_headings_from_file[index + 1:
                                                             index + 1 + i]
  
                      # Unexpected headings inside Test section
                      unexpected_headings = set(test_section) - \
                          required_headings
                      if unexpected_headings:
                          warnings_list.append(
                              warn_unexpected_headings.
                              format(', '.join(unexpected_headings),
                                     test_section_name))
  
                      amount_of_steps = required_section_exists(
                          test_section,
                          'Step',
                          tuple(sections_headings['Step']))
                      amount_of_expects = required_section_exists(
                          test_section,
                          'Expect',
                          tuple(sections_headings['Expect']))
  
                      # # Step isn't in pair with # Expect
                      if amount_of_steps != amount_of_expects != 0:
                          warnings_list.append(warn_headings_not_in_pairs.
                                               format(amount_of_steps,
                                                      amount_of_expects,
                                                      test_section_name))
                      index += i
                      break
  
          index += 1
          if index >= len(html_headings_from_file) - 1:
              break
      return warnings_list
  
  
  def return_markdown_file() -> str:
      """ Return path to the markdown file """
      files = '\n'.join(os.listdir())
      reg_exp = r'.+\.md$'
      md_files = re.findall(reg_exp, files, re.M)
      fail_message = "in the current working directory.\n" \
                     "Manual steps couldn't be exported"
      if len(md_files) == 1:
          md_path = os.path.join(os.getcwd(), md_files[0])
      elif len(md_files) == 0:
          md_path = ''
          echo((style(f'Markdown file doesn\'t exist {fail_message}',
                      fg='yellow')))
      else:
          md_path = ''
          echo((style(f'{len(md_files)} Markdown files found {fail_message}',
                      fg='yellow')))
      return md_path
  
  
  def get_category() -> str:
      """ Get category from Makefile """
      category = 'Sanity'
      try:
          with open('Makefile', encoding='utf-8') as makefile_file:
              makefile = makefile_file.read()
          category_search = re.search(
              r'echo\s+"Type:\s*(.*)"', makefile, re.M)
          if category_search:
              category = category_search.group(1)
      # Default to 'Sanity' if Makefile or Type not found
      except (IOError, AttributeError):
          pass
      return category
  
  
  def create_nitrate_case(summary: str) -> Any:
      """ Create new nitrate case """
      # Create the new test case
      assert nitrate
      category = nitrate.Category(name=get_category(), product=DEFAULT_PRODUCT)
      testcase = nitrate.TestCase(summary=summary, category=category)
      echo(style(f"Test case '{testcase.identifier}' created.", fg='blue'))
      return testcase
  
  
  def create_polarion_case(summary: str, project_id: str) -> Any:
      """ Create new polarion case """
      # Create the new test case
      testcase = PolarionTestCase.create(project_id, summary, summary)
      testcase.tcmscategory = get_category()
      testcase.update()
      echo(style(f"Test case '{testcase.work_item_id}' created.", fg='blue'))
      return testcase
  
  
  def prepare_extra_summary(test: 'tmt.Test') -> str:
      """ extra-summary for export --create test """
      remote_dirname = re.sub('.git$', '', os.path.basename(test.fmf_id['url']))
      if not remote_dirname:
          raise ConvertError("Unable to find git remote url.")
      generated = f"{remote_dirname} {test.name}"
      if test.summary:
          generated += f" - {test.summary}"
      # TODO: remove cast later
      return cast(str, test.node.get('extra-summary', generated))
  
  
  # avoid multiple searching for general plans (it is expensive)
  @lru_cache(maxsize=None)
  def find_general_plan(component: str) -> Any:
      """ Return single General Test Plan or raise an error """
      assert nitrate
      # At first find by linked components
      found = nitrate.TestPlan.search(
          type__name="General",
          is_active=True,
          component__name=f"{component}")
      # Attempt to find by name if no test plan found
      if not found:
          found = nitrate.TestPlan.search(
              type__name="General",
              is_active=True,
              name=f"{component} / General")
      # No general -> raise error
      if not found:
          raise nitrate.NitrateError(
              f"No general test plan found for '{component}'.")
      # Multiple general plans are fishy -> raise error
      if len(found) != 1:
          nitrate.NitrateError(
              "Multiple general test plans found for '{component}' component.")
      # Finally return the one and only General plan
      return found[0]
  
files: tmt/export.py
files: import tmt
  import tmt.steps
  import tmt.steps.finish
  import tmt.steps.prepare.ansible
  from tmt.steps.prepare.ansible import PrepareAnsible
  
  
  @tmt.steps.provides_method('ansible')
  class FinishAnsible(tmt.steps.finish.FinishPlugin, PrepareAnsible):
      """
      Perform finishing tasks using ansible
  
      Single playbook config:
  
          finish:
              how: ansible
              playbook: ansible/packages.yml
  
      Multiple playbooks config:
  
          finish:
              how: ansible
              playbook:
                - playbook/one.yml
                - playbook/two.yml
                - playbook/three.yml
  
      The playbook path should be relative to the metadata tree root.
      Use 'order' attribute to select in which order finishing tasks
      should happen if there are multiple configs. Default order is '50'.
      """
  
      _data_class = tmt.steps.prepare.ansible.PrepareAnsibleData
  
      # Assigning class methods seems to cause trouble to mypy
      # See also: https://github.com/python/mypy/issues/6700
      base_command = tmt.steps.finish.FinishPlugin.base_command  # type: ignore[assignment]
  
files: tmt/steps/finish/ansible.py
files: import tmt
  import tmt.steps
  import tmt.steps.finish
  import tmt.steps.prepare.ansible
  from tmt.steps.prepare.ansible import PrepareAnsible
  
  
  @tmt.steps.provides_method('ansible')
  class FinishAnsible(tmt.steps.finish.FinishPlugin, PrepareAnsible):
      """
      Perform finishing tasks using ansible
  
      Single playbook config:
  
          finish:
              how: ansible
              playbook: ansible/packages.yml
  
      Multiple playbooks config:
  
          finish:
              how: ansible
              playbook:
                - playbook/one.yml
                - playbook/two.yml
                - playbook/three.yml
  
      The playbook path should be relative to the metadata tree root.
      Use 'order' attribute to select in which order finishing tasks
      should happen if there are multiple configs. Default order is '50'.
      """
  
      _data_class = tmt.steps.prepare.ansible.PrepareAnsibleData
  
      # Assigning class methods seems to cause trouble to mypy
      # See also: https://github.com/python/mypy/issues/6700
      base_command = tmt.steps.finish.FinishPlugin.base_command  # type: ignore
  
files: tmt/steps/finish/ansible.py
files: # coding: utf-8
  
  import dataclasses
  import datetime
  import os
  import platform
  import re
  import time
  import types
  from typing import Any, List, Optional, cast
  
  import click
  import requests
  
  import tmt
  import tmt.steps
  import tmt.steps.provision
  import tmt.utils
  from tmt.utils import WORKDIR_ROOT, ProvisionError, retry_session
  
  libvirt: Optional[types.ModuleType] = None
  testcloud: Optional[types.ModuleType] = None
  
  
  def import_testcloud() -> None:
      """
      Import testcloud module only when needed
  
      Until we have a separate package for each plugin.
      """
      global testcloud
      global libvirt
      try:
          import libvirt
          import testcloud.image
          import testcloud.instance
          import testcloud.util
      except ImportError:
          raise ProvisionError(
              "Install 'testcloud' to provision using this method.")
  
  
  # Testcloud cache to our tmt's workdir root
  TESTCLOUD_DATA = os.path.join(WORKDIR_ROOT, 'testcloud')
  TESTCLOUD_IMAGES = os.path.join(TESTCLOUD_DATA, 'images')
  
  # Userdata for cloud-init
  USER_DATA = """#cloud-config
  chpasswd:
    list: |
      {user_name}:%s
    expire: false
  users:
    - default
    - name: {user_name}
  ssh_authorized_keys:
    - {public_key}
  ssh_pwauth: true
  disable_root: false
  runcmd:
    - sed -i -e '/^.*PermitRootLogin/s/^.*$/PermitRootLogin yes/'
      -e '/^.*UseDNS/s/^.*$/UseDNS no/'
      -e '/^.*GSSAPIAuthentication/s/^.*$/GSSAPIAuthentication no/'
      /etc/ssh/sshd_config
    - systemctl reload sshd
    - [sh, -c, 'if [ ! -f /etc/systemd/network/20-tc-usernet.network ] &&
    systemctl status systemd-networkd | grep -q "enabled;\\svendor\\spreset:\\senabled";
    then mkdir -p /etc/systemd/network/ &&
    echo "[Match]" >> /etc/systemd/network/20-tc-usernet.network &&
    echo "Name=en*" >> /etc/systemd/network/20-tc-usernet.network &&
    echo "[Network]" >> /etc/systemd/network/20-tc-usernet.network &&
    echo "DHCP=yes" >> /etc/systemd/network/20-tc-usernet.network; fi']
    - [sh, -c, 'if systemctl status systemd-networkd |
    grep -q "enabled;\\svendor\\spreset:\\senabled"; then
    systemctl restart systemd-networkd; fi']
    - [sh, -c, 'if cat /etc/os-release |
    grep -q platform:el8; then systemctl restart sshd; fi']
  """
  
  COREOS_DATA = """variant: fcos
  version: 1.4.0
  passwd:
    users:
      - name: {user_name}
        ssh_authorized_keys:
          - {public_key}
  """
  
  # Libvirt domain XML template related variables
  DOMAIN_TEMPLATE_NAME = 'domain-template.jinja'
  DOMAIN_TEMPLATE_FILE = os.path.join(TESTCLOUD_DATA, DOMAIN_TEMPLATE_NAME)
  DOMAIN_TEMPLATE = """<domain type='{{ virt_type }}' xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'>
    <name>{{ domain_name }}</name>
    <uuid>{{ uuid }}</uuid>
    <memory unit='KiB'>{{ memory }}</memory>
    <currentMemory unit='KiB'>{{ memory }}</currentMemory>
    <vcpu placement='static'>2</vcpu>
    <os>
      <type arch='{{ arch }}' machine='{{ model }}'>hvm</type>
      {{ uefi_loader }}
      <boot dev='hd'/>
    </os>
    {{ cpu }}
    {{ extra_specs }}
    <clock offset='utc'>
      <timer name='rtc' tickpolicy='catchup'/>
      <timer name='pit' tickpolicy='delay'/>
      <timer name='hpet' present='no'/>
    </clock>
    <on_poweroff>destroy</on_poweroff>
    <on_reboot>restart</on_reboot>
    <on_crash>restart</on_crash>
    <devices>
      <emulator>{{ emulator_path }}</emulator>
      <disk type='file' device='disk'>
        <driver name='qemu' type='qcow2' cache='unsafe'/>
        <source file="{{ disk }}"/>
        <target dev='vda' bus='virtio'/>
        {{ boot_drive_address }}
      </disk>
      <disk type='file' device='disk'>
        <driver name='qemu' type='raw'/>
        <source file="{{ seed }}"/>
        <target dev='vdb' bus='virtio'/>
        <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
      </disk>
      <interface type='{{ network_type }}'>
        <mac address="{{ mac_address }}"/>
        {{ network_source }}
        {{ ip_setup }}
        <model type='virtio'/>
        <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
      </interface>
      <serial type='pty'>
        <target port='0'/>
      </serial>
      <console type='pty'>
        <target type='serial' port='0'/>
      </console>
      <input type="keyboard" bus="virtio"/>
      <rng model='virtio'>
        <backend model='random'>/dev/urandom</backend>
      </rng>
    </devices>
    {{ qemu_args }}
  </domain>
  """  # noqa: E501
  
  # VM defaults
  DEFAULT_BOOT_TIMEOUT = 60      # seconds
  DEFAULT_CONNECT_TIMEOUT = 60   # seconds
  NON_KVM_ADDITIONAL_WAIT = 10       # seconds
  NON_KVM_TIMEOUT_COEF = 10          # times
  
  # SSH key type, set None for ssh-keygen default one
  SSH_KEYGEN_TYPE = "ecdsa"
  
  DEFAULT_USER = 'root'
  DEFAULT_MEMORY = 2048
  DEFAULT_DISK = 10
  DEFAULT_IMAGE = 'fedora'
  DEFAULT_CONNECTION = 'session'
  DEFAULT_ARCH = platform.machine()
  
  
  @dataclasses.dataclass
  class TestcloudGuestData(tmt.steps.provision.GuestSshData):
      # Override parent class with our defaults
      user: str = DEFAULT_USER
  
      image: str = DEFAULT_IMAGE
      memory: int = DEFAULT_MEMORY
      disk: int = DEFAULT_DISK
      connection: str = DEFAULT_CONNECTION
      arch: str = DEFAULT_ARCH
  
      image_url: Optional[str] = None
      instance_name: Optional[str] = None
  
  
  @dataclasses.dataclass
  class ProvisionTestcloudData(TestcloudGuestData, tmt.steps.StepData):
      pass
  
  
  @tmt.steps.provides_method('virtual.testcloud')
  class ProvisionTestcloud(tmt.steps.provision.ProvisionPlugin):
      """
      Local virtual machine using testcloud
  
      Minimal config which uses the latest fedora image:
  
          provision:
              how: virtual
  
      Here's a full config example:
  
          provision:
              how: virtual
              image: fedora
              user: root
              memory: 2048
  
      As the image use 'fedora' for the latest released Fedora compose,
      'rawhide' for the latest Rawhide compose, short aliases such as
      'fedora-32', 'f-32' or 'f32' for specific release or a full url to
      the qcow2 image for example from:
  
          https://kojipkgs.fedoraproject.org/compose/
  
      Short names are also provided for 'centos', 'centos-stream',
      'debian' and 'ubuntu' (e.g. 'centos-8' or 'c8').
  
      Supported Fedora CoreOS images are:
  
          fedora-coreos
          fedora-coreos-stable
          fedora-coreos-testing
          fedora-coreos-next
  
      Use the full path for images stored on local disk, for example:
  
          /var/tmp/images/Fedora-Cloud-Base-31-1.9.x86_64.qcow2
  
      In addition to the qcow2 format, vagrant boxes can be used as well,
      testcloud will take care of unpacking the image for you.
      """
  
      _data_class = ProvisionTestcloudData
  
      # Guest instance
      _guest = None
  
      @classmethod
      def options(cls, how: Optional[str] = None) -> List[tmt.options.ClickOptionDecoratorType]:
          """ Prepare command line options for testcloud """
          options = super().options(how)
          options[:0] = [
              click.option(
                  '-i', '--image', metavar='IMAGE',
                  help='Select image to be used. Provide a short name, '
                       'full path to a local file or a complete url.'),
              click.option(
                  '-m', '--memory', metavar='MEMORY', type=int,
                  help='Set available memory in MB, 2048 MB by default.'),
              click.option(
                  '-D', '--disk', metavar='MEMORY', type=int,
                  help='Specify disk size in GB, 10 GB by default.'),
              click.option(
                  '-u', '--user', metavar='USER',
                  help='Username to use for all guest operations.'),
              click.option(
                  '-c', '--connection',
                  type=click.Choice(['session', 'system']),
                  help="What session type to use, 'session' by default."),
              click.option(
                  '-a', '--arch',
                  type=click.Choice(['x86_64', 'aarch64', 's390x', 'ppc64le']),
                  help="What architecture to virtualize, host arch by default."),
              ]
          return options
  
      # TODO: Revisit this `type: ignore` once `Guest` becomes a generic type
      def wake(self, data: Optional[TestcloudGuestData] = None) -> None:  # type: ignore[override]
          """ Wake up the plugin, process data, apply options """
          super().wake(data=data)
  
          # Wake up testcloud instance
          if data:
              guest = GuestTestcloud(data, name=self.name, parent=self.step)
              guest.wake()
              self._guest = guest
  
      def go(self) -> None:
          """ Provision the testcloud instance """
          super().go()
  
          # Give info about provided data
          data = TestcloudGuestData(**{
              key: self.get(key)
              for key in TestcloudGuestData.keys()
              })
  
          # Once plan schema is enforced this won't be necessary
          # click enforces int for cmdline and schema validation
          # will make sure 'int' gets from plan data.
          # Another key is 'port' however that is not exposed to the cli
          for int_key in ["memory", "disk", "port"]:
              value = getattr(data, int_key)
              if value is not None:
                  try:
                      setattr(data, int_key, int(value))
                  except ValueError:
                      raise tmt.utils.SpecificationError(
                          f"Value '{value}' cannot be converted to int for '{int_key}' attribute.")
  
          for key, value in data.to_dict().items():
              if key == 'memory':
                  self.info('memory', f"{value} MB", 'green')
              elif key == 'disk':
                  self.info('disk', f"{value} GB", 'green')
              elif key == 'connection':
                  self.verbose('connection', value, 'green')
              elif value is not None:
                  self.info(key, value, 'green')
  
          # Create a new GuestTestcloud instance and start it
          self._guest = GuestTestcloud(data, name=self.name, parent=self.step)
          self._guest.start()
  
      def guest(self) -> Optional[tmt.Guest]:
          """ Return the provisioned guest """
          return self._guest
  
      @classmethod
      def clean_images(cls, clean: Any, dry: Any) -> bool:
          """ Remove the testcloud images """
          clean.info('testcloud', shift=1, color='green')
          if not os.path.exists(TESTCLOUD_IMAGES):
              clean.warn(
                  f"Directory '{TESTCLOUD_IMAGES}' does not exist.", shift=2)
              return True
          successful = True
          for image in os.listdir(TESTCLOUD_IMAGES):
              image = os.path.join(TESTCLOUD_IMAGES, image)
              if dry:
                  clean.verbose(f"Would remove '{image}'.", shift=2)
              else:
                  clean.verbose(f"Removing '{image}'.", shift=2)
                  try:
                      os.remove(image)
                  except OSError:
                      clean.error(f"Failed to remove '{image}'.", shift=2)
                      successful = False
          return successful
  
  
  class GuestTestcloud(tmt.GuestSsh):
      """
      Testcloud Instance
  
      The following keys are expected in the 'data' dictionary::
  
          image ...... qcov image name or url
          user ....... user name to log in
          memory ..... memory size for vm
          disk ....... disk size for vm
          connection . either session (default) or system, to be passed to qemu
          arch ....... architecture for the VM, host arch is the default
      """
  
      # TODO: Revisit this `type: ignore` once `Guest` becomes a generic type
      _data_class = TestcloudGuestData  # type: ignore[assignment]
  
      image: str
      image_url: Optional[str]
      instance_name: Optional[str]
      memory: int
      disk: str
      connection: str
      arch: str
  
      # Not to be saved, recreated from image_url/instance_name/... every
      # time guest is instantiated.
      _image: Optional['testcloud.image.Image'] = None  # type: ignore[name-defined]
      _instance: Optional['testcloud.instance.Instance'] = None  # type: ignore[name-defined]
  
      def _get_url(self, url: str, message: str) -> requests.Response:
          """ Get url, retry when fails, return response """
  
          def try_get_url() -> requests.Response:
              try:
                  with retry_session() as session:
                      response = session.get(url)
  
                  if response.ok:
                      return response
  
              except requests.RequestException:
                  pass
              finally:
                  raise tmt.utils.WaitingIncomplete()
  
          try:
              return tmt.utils.wait(
                  self, try_get_url, datetime.timedelta(
                      seconds=DEFAULT_CONNECT_TIMEOUT), tick=1)
  
          except tmt.utils.WaitingTimedOutError:
              raise ProvisionError(
                  f'Failed to {message} in {DEFAULT_CONNECT_TIMEOUT}s.')
  
      def _guess_image_url(self, name: str) -> str:
          """ Guess image url for given name """
  
          # Try to check if given url is a local file
          if os.path.isabs(name) and os.path.isfile(name):
              return f'file://{name}'
  
          name = name.lower().strip()
          url = None
  
          # Map fedora aliases (e.g. rawhide, fedora, fedora-32, f-32, f32)
          matched_fedora = re.match(r'^f(edora)?-?(\d+)$', name)
          # Map fedora coreos aliases (e.g. stable, next or testing)
          matched_fedora_coreos = re.match(r'^f(edora-coreos)?-?(stable|testing|next)$', name)
          # Map centos aliases (e.g. centos:X, centos, centos-stream:X)
          matched_centos = [re.match(r'^c(entos)?-?(\d+)$', name),
                            re.match(r'^c(entos-stream)?-?(\d+)$', name)]
          matched_ubuntu = re.match(r'^u(buntu)?-?(\w+)$', name)
          matched_debian = re.match(r'^d(ebian)?-?(\w+)$', name)
  
          assert testcloud is not None
          # Plain name match means we want the latest release
          if name == 'fedora':
              url = testcloud.util.get_fedora_image_url("latest", self.arch)
          elif name == 'fedora-coreos':
              url = testcloud.util.get_fedora_image_url("stable", self.arch)
          elif name == 'centos':
              url = testcloud.util.get_centos_image_url("latest", self.arch)
          elif name == 'centos-stream':
              url = testcloud.util.get_centos_image_url(
                  "latest", stream=True, arch=self.arch)
          elif name == 'ubuntu':
              url = testcloud.util.get_ubuntu_image_url("latest", self.arch)
          elif name == 'debian':
              url = testcloud.util.get_debian_image_url("latest", self.arch)
  
          elif matched_fedora:
              url = testcloud.util.get_fedora_image_url(
                  matched_fedora.group(2), self.arch)
          elif matched_fedora_coreos:
              url = testcloud.util.get_fedora_image_url(
                  matched_fedora_coreos.group(2), self.arch)
          elif matched_centos[0]:
              url = testcloud.util.get_centos_image_url(
                  matched_centos[0].group(2), self.arch)
          elif matched_centos[1]:
              url = testcloud.util.get_centos_image_url(
                  matched_centos[1].group(2), stream=True, arch=self.arch)
          elif matched_ubuntu:
              url = testcloud.util.get_ubuntu_image_url(
                  matched_ubuntu.group(2), self.arch)
          elif matched_debian:
              url = testcloud.util.get_debian_image_url(
                  matched_debian.group(2), self.arch)
          elif 'rawhide' in name:
              url = testcloud.util.get_fedora_image_url("rawhide", self.arch)
  
          if not url:
              raise ProvisionError(f"Could not map '{name}' to compose.")
          return cast(str, url)
  
      @staticmethod
      def _create_template() -> None:
          """ Create libvirt domain template """
          # Write always to ovewrite possible outdated version
          with open(DOMAIN_TEMPLATE_FILE, 'w') as template:
              template.write(DOMAIN_TEMPLATE)
  
      def wake(self) -> None:
          """ Wake up the guest """
          self.debug(
              f"Waking up testcloud instance '{self.instance_name}'.",
              level=2, shift=0)
          self.prepare_config()
          assert testcloud is not None
          self._image = testcloud.image.Image(self.image_url)
          self._instance = testcloud.instance.Instance(
              self.instance_name, image=self._image,
              connection=f"qemu:///{self.connection}", desired_arch=self.arch)
  
      def prepare_ssh_key(self, key_type: Optional[str] = None) -> None:
          """ Prepare ssh key for authentication """
          # Create ssh key paths
          key_name = "id_{}".format(key_type if key_type is not None else 'rsa')
          assert self.workdir is not None
          self.key = [os.path.join(self.workdir, key_name)]
          self.pubkey = os.path.join(self.workdir, f'{key_name}.pub')
  
          # Generate ssh key
          self.debug('Generating an ssh key.')
          command = ["ssh-keygen", "-f", self.key[0], "-N", ""]
          if key_type is not None:
              command.extend(["-t", key_type])
          self.run(command)
          with open(self.pubkey, 'r') as pubkey:
              self.config.USER_DATA = USER_DATA.format(
                  user_name=self.user, public_key=pubkey.read())
          with open(self.pubkey, 'r') as pubkey:
              self.config.COREOS_DATA = COREOS_DATA.format(
                  user_name=self.user, public_key=pubkey.read())
  
      def prepare_config(self) -> None:
          """ Prepare common configuration """
          import_testcloud()
  
          # Get configuration
          assert testcloud is not None
          self.config = testcloud.config.get_config()
  
          # Make sure download progress is disabled unless in debug mode,
          # so it does not spoil our logging
          self.config.DOWNLOAD_PROGRESS = self.opt('debug') > 2
          self.config.DOWNLOAD_PROGRESS_VERBOSE = False
  
          # Configure to tmt's storage directories
          self.config.DATA_DIR = TESTCLOUD_DATA
          self.config.STORE_DIR = TESTCLOUD_IMAGES
  
      def start(self) -> None:
          """ Start provisioned guest """
          if self.opt('dry'):
              return
          # Make sure required directories exist
          os.makedirs(TESTCLOUD_DATA, exist_ok=True)
          os.makedirs(TESTCLOUD_IMAGES, exist_ok=True)
  
          # Make sure libvirt domain template exists
          GuestTestcloud._create_template()
  
          # Prepare config
          self.prepare_config()
  
          # Kick off image URL with the given image
          self.image_url = self.image
  
          # If image does not start with http/https/file, consider it a
          # mapping value and try to guess the URL
          if not re.match(r'^(?:https?|file)://.*', self.image_url):
              self.image_url = self._guess_image_url(self.image_url)
              self.debug(f"Guessed image url: '{self.image_url}'", level=3)
  
          # Initialize and prepare testcloud image
          assert testcloud is not None
          self._image = testcloud.image.Image(self.image_url)
          self.verbose('qcow', self._image.name, 'green')
          if not os.path.exists(self._image.local_path):
              self.info('progress', 'downloading...', 'cyan')
          try:
              self._image.prepare()
          except FileNotFoundError as error:
              raise ProvisionError(
                  f"Image '{self._image.local_path}' not found.", original=error)
          except (testcloud.exceptions.TestcloudPermissionsError,
                  PermissionError) as error:
              raise ProvisionError(
                  f"Failed to prepare the image. Check the '{TESTCLOUD_IMAGES}' "
                  f"directory permissions.", original=error)
  
          # Create instance
          self.instance_name = self._tmt_name()
          self._instance = testcloud.instance.Instance(
              name=self.instance_name, image=self._image,
              connection=f"qemu:///{self.connection}", desired_arch=self.arch)
          self.verbose('name', self.instance_name, 'green')
  
          # Decide if we want to multiply timeouts when emulating an architecture
          time_coeff = NON_KVM_TIMEOUT_COEF if not self._instance.kvm else 1
  
          # Decide which networking setup to use
          # Autodetect works with libguestfs python bindings
          # We fall back to basic heuristics based on file name
          # without that installed (eg. from pypi).
          # https://bugzilla.redhat.com/show_bug.cgi?id=1075594
          try:
              import guestfs  # noqa: F401
          except ImportError:
              match_legacy = re.search(
                  r'(rhel|centos).*-7', self.image_url.lower())
              if match_legacy:
                  self._instance.pci_net = "e1000"
              else:
                  self._instance.pci_net = "virtio-net-pci"
  
          # Prepare ssh key
          # TODO: Maybe... some better way to do this?
          if "coreos" in self.image.lower():
              self._instance.coreos = True
              self._instance.ssh_path = self.key[0]
          self.prepare_ssh_key(SSH_KEYGEN_TYPE)
  
          # Boot the virtual machine
          self.info('progress', 'booting...', 'cyan')
          self._instance.ram = self.memory
          self._instance.disk_size = self.disk
          assert libvirt is not None
          try:
              self._instance.prepare()
              self._instance.spawn_vm()
              self._instance.start(DEFAULT_BOOT_TIMEOUT * time_coeff)
          except (testcloud.exceptions.TestcloudInstanceError,
                  libvirt.libvirtError) as error:
              raise ProvisionError(
                  f'Failed to boot testcloud instance ({error}).')
          self.guest = self._instance.get_ip()
          self.port = int(self._instance.get_instance_port())
          self.verbose('ip', self.guest, 'green')
          self.verbose('port', str(self.port), 'green')
          self._instance.create_ip_file(self.guest)
  
          # Wait a bit until the box is up
          if not self.reconnect(
                  timeout=DEFAULT_CONNECT_TIMEOUT *
                  time_coeff,
                  tick=1):
              raise ProvisionError(
                  f"Failed to connect in {DEFAULT_CONNECT_TIMEOUT * time_coeff}s.")
  
          if not self._instance.kvm:
              self.debug(
                  f"Waiting {NON_KVM_ADDITIONAL_WAIT} seconds "
                  f"for non-kvm instance...")
              time.sleep(NON_KVM_ADDITIONAL_WAIT)
  
      def stop(self) -> None:
          """ Stop provisioned guest """
          super().stop()
          # Stop only if the instance successfully booted
          if self._instance and self.guest:
              self.debug(f"Stopping testcloud instance '{self.instance_name}'.")
              assert testcloud is not None
              try:
                  self._instance.stop()
              except testcloud.exceptions.TestcloudInstanceError as error:
                  raise tmt.utils.ProvisionError(
                      f"Failed to stop testcloud instance: {error}")
              self.info('guest', 'stopped', 'green')
  
      def remove(self) -> None:
          """ Remove the guest (disk cleanup) """
          if self._instance:
              self.debug(f"Removing testcloud instance '{self.instance_name}'.")
              try:
                  self._instance.remove(autostop=True)
              except FileNotFoundError as error:
                  raise tmt.utils.ProvisionError(
                      f"Failed to remove testcloud instance: {error}")
              self.info('guest', 'removed', 'green')
  
      def reboot(self,
                 hard: bool = False,
                 command: Optional[str] = None,
                 timeout: Optional[int] = None,
                 tick: float = tmt.utils.DEFAULT_WAIT_TICK,
                 tick_increase: float = tmt.utils.DEFAULT_WAIT_TICK_INCREASE) -> Any:
          """ Reboot the guest, return True if successful """
          # Use custom reboot command if provided
          if command:
              return super().reboot(hard=hard, command=command)
          if not self._instance:
              raise tmt.utils.ProvisionError("No instance initialized.")
          self._instance.reboot(soft=not hard)
          return self.reconnect(timeout=timeout)
  
files: tmt/steps/provision/testcloud.py
files: # coding: utf-8
  
  import dataclasses
  import datetime
  import os
  import platform
  import re
  import time
  import types
  from typing import Any, List, Optional, cast
  
  import click
  import requests
  
  import tmt
  import tmt.steps
  import tmt.steps.provision
  import tmt.utils
  from tmt.utils import WORKDIR_ROOT, ProvisionError, retry_session
  
  libvirt: Optional[types.ModuleType] = None
  testcloud: Optional[types.ModuleType] = None
  
  
  def import_testcloud() -> None:
      """
      Import testcloud module only when needed
  
      Until we have a separate package for each plugin.
      """
      global testcloud
      global libvirt
      try:
          import libvirt
          import testcloud.image
          import testcloud.instance
          import testcloud.util
      except ImportError:
          raise ProvisionError(
              "Install 'testcloud' to provision using this method.")
  
  
  # Testcloud cache to our tmt's workdir root
  TESTCLOUD_DATA = os.path.join(WORKDIR_ROOT, 'testcloud')
  TESTCLOUD_IMAGES = os.path.join(TESTCLOUD_DATA, 'images')
  
  # Userdata for cloud-init
  USER_DATA = """#cloud-config
  chpasswd:
    list: |
      {user_name}:%s
    expire: false
  users:
    - default
    - name: {user_name}
  ssh_authorized_keys:
    - {public_key}
  ssh_pwauth: true
  disable_root: false
  runcmd:
    - sed -i -e '/^.*PermitRootLogin/s/^.*$/PermitRootLogin yes/'
      -e '/^.*UseDNS/s/^.*$/UseDNS no/'
      -e '/^.*GSSAPIAuthentication/s/^.*$/GSSAPIAuthentication no/'
      /etc/ssh/sshd_config
    - systemctl reload sshd
    - [sh, -c, 'if [ ! -f /etc/systemd/network/20-tc-usernet.network ] &&
    systemctl status systemd-networkd | grep -q "enabled;\\svendor\\spreset:\\senabled";
    then mkdir -p /etc/systemd/network/ &&
    echo "[Match]" >> /etc/systemd/network/20-tc-usernet.network &&
    echo "Name=en*" >> /etc/systemd/network/20-tc-usernet.network &&
    echo "[Network]" >> /etc/systemd/network/20-tc-usernet.network &&
    echo "DHCP=yes" >> /etc/systemd/network/20-tc-usernet.network; fi']
    - [sh, -c, 'if systemctl status systemd-networkd |
    grep -q "enabled;\\svendor\\spreset:\\senabled"; then
    systemctl restart systemd-networkd; fi']
    - [sh, -c, 'if cat /etc/os-release |
    grep -q platform:el8; then systemctl restart sshd; fi']
  """
  
  COREOS_DATA = """variant: fcos
  version: 1.4.0
  passwd:
    users:
      - name: {user_name}
        ssh_authorized_keys:
          - {public_key}
  """
  
  # Libvirt domain XML template related variables
  DOMAIN_TEMPLATE_NAME = 'domain-template.jinja'
  DOMAIN_TEMPLATE_FILE = os.path.join(TESTCLOUD_DATA, DOMAIN_TEMPLATE_NAME)
  DOMAIN_TEMPLATE = """<domain type='{{ virt_type }}' xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'>
    <name>{{ domain_name }}</name>
    <uuid>{{ uuid }}</uuid>
    <memory unit='KiB'>{{ memory }}</memory>
    <currentMemory unit='KiB'>{{ memory }}</currentMemory>
    <vcpu placement='static'>2</vcpu>
    <os>
      <type arch='{{ arch }}' machine='{{ model }}'>hvm</type>
      {{ uefi_loader }}
      <boot dev='hd'/>
    </os>
    {{ cpu }}
    {{ extra_specs }}
    <clock offset='utc'>
      <timer name='rtc' tickpolicy='catchup'/>
      <timer name='pit' tickpolicy='delay'/>
      <timer name='hpet' present='no'/>
    </clock>
    <on_poweroff>destroy</on_poweroff>
    <on_reboot>restart</on_reboot>
    <on_crash>restart</on_crash>
    <devices>
      <emulator>{{ emulator_path }}</emulator>
      <disk type='file' device='disk'>
        <driver name='qemu' type='qcow2' cache='unsafe'/>
        <source file="{{ disk }}"/>
        <target dev='vda' bus='virtio'/>
        {{ boot_drive_address }}
      </disk>
      <disk type='file' device='disk'>
        <driver name='qemu' type='raw'/>
        <source file="{{ seed }}"/>
        <target dev='vdb' bus='virtio'/>
        <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
      </disk>
      <interface type='{{ network_type }}'>
        <mac address="{{ mac_address }}"/>
        {{ network_source }}
        {{ ip_setup }}
        <model type='virtio'/>
        <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
      </interface>
      <serial type='pty'>
        <target port='0'/>
      </serial>
      <console type='pty'>
        <target type='serial' port='0'/>
      </console>
      <input type="keyboard" bus="virtio"/>
      <rng model='virtio'>
        <backend model='random'>/dev/urandom</backend>
      </rng>
    </devices>
    {{ qemu_args }}
  </domain>
  """  # noqa: E501
  
  # VM defaults
  DEFAULT_BOOT_TIMEOUT = 60      # seconds
  DEFAULT_CONNECT_TIMEOUT = 60   # seconds
  NON_KVM_ADDITIONAL_WAIT = 10       # seconds
  NON_KVM_TIMEOUT_COEF = 10          # times
  
  # SSH key type, set None for ssh-keygen default one
  SSH_KEYGEN_TYPE = "ecdsa"
  
  DEFAULT_USER = 'root'
  DEFAULT_MEMORY = 2048
  DEFAULT_DISK = 10
  DEFAULT_IMAGE = 'fedora'
  DEFAULT_CONNECTION = 'session'
  DEFAULT_ARCH = platform.machine()
  
  
  @dataclasses.dataclass
  class TestcloudGuestData(tmt.steps.provision.GuestSshData):
      # Override parent class with our defaults
      user: str = DEFAULT_USER
  
      image: str = DEFAULT_IMAGE
      memory: int = DEFAULT_MEMORY
      disk: int = DEFAULT_DISK
      connection: str = DEFAULT_CONNECTION
      arch: str = DEFAULT_ARCH
  
      image_url: Optional[str] = None
      instance_name: Optional[str] = None
  
  
  @dataclasses.dataclass
  class ProvisionTestcloudData(TestcloudGuestData, tmt.steps.StepData):
      pass
  
  
  @tmt.steps.provides_method('virtual.testcloud')
  class ProvisionTestcloud(tmt.steps.provision.ProvisionPlugin):
      """
      Local virtual machine using testcloud
  
      Minimal config which uses the latest fedora image:
  
          provision:
              how: virtual
  
      Here's a full config example:
  
          provision:
              how: virtual
              image: fedora
              user: root
              memory: 2048
  
      As the image use 'fedora' for the latest released Fedora compose,
      'rawhide' for the latest Rawhide compose, short aliases such as
      'fedora-32', 'f-32' or 'f32' for specific release or a full url to
      the qcow2 image for example from:
  
          https://kojipkgs.fedoraproject.org/compose/
  
      Short names are also provided for 'centos', 'centos-stream',
      'debian' and 'ubuntu' (e.g. 'centos-8' or 'c8').
  
      Supported Fedora CoreOS images are:
  
          fedora-coreos
          fedora-coreos-stable
          fedora-coreos-testing
          fedora-coreos-next
  
      Use the full path for images stored on local disk, for example:
  
          /var/tmp/images/Fedora-Cloud-Base-31-1.9.x86_64.qcow2
  
      In addition to the qcow2 format, vagrant boxes can be used as well,
      testcloud will take care of unpacking the image for you.
      """
  
      _data_class = ProvisionTestcloudData
  
      # Guest instance
      _guest = None
  
      @classmethod
      def options(cls, how: Optional[str] = None) -> List[tmt.options.ClickOptionDecoratorType]:
          """ Prepare command line options for testcloud """
          options = super().options(how)
          options[:0] = [
              click.option(
                  '-i', '--image', metavar='IMAGE',
                  help='Select image to be used. Provide a short name, '
                       'full path to a local file or a complete url.'),
              click.option(
                  '-m', '--memory', metavar='MEMORY', type=int,
                  help='Set available memory in MB, 2048 MB by default.'),
              click.option(
                  '-D', '--disk', metavar='MEMORY', type=int,
                  help='Specify disk size in GB, 10 GB by default.'),
              click.option(
                  '-u', '--user', metavar='USER',
                  help='Username to use for all guest operations.'),
              click.option(
                  '-c', '--connection',
                  type=click.Choice(['session', 'system']),
                  help="What session type to use, 'session' by default."),
              click.option(
                  '-a', '--arch',
                  type=click.Choice(['x86_64', 'aarch64', 's390x', 'ppc64le']),
                  help="What architecture to virtualize, host arch by default."),
              ]
          return options
  
      # TODO: Revisit this `type: ignore` once `Guest` becomes a generic type
      def wake(self, data: Optional[TestcloudGuestData] = None) -> None:  # type: ignore[override]
          """ Wake up the plugin, process data, apply options """
          super().wake(data=data)
  
          # Wake up testcloud instance
          if data:
              guest = GuestTestcloud(data, name=self.name, parent=self.step)
              guest.wake()
              self._guest = guest
  
      def go(self) -> None:
          """ Provision the testcloud instance """
          super().go()
  
          # Give info about provided data
          data = TestcloudGuestData(**{
              key: self.get(key)
              for key in TestcloudGuestData.keys()
              })
  
          # Once plan schema is enforced this won't be necessary
          # click enforces int for cmdline and schema validation
          # will make sure 'int' gets from plan data.
          # Another key is 'port' however that is not exposed to the cli
          for int_key in ["memory", "disk", "port"]:
              value = getattr(data, int_key)
              if value is not None:
                  try:
                      setattr(data, int_key, int(value))
                  except ValueError:
                      raise tmt.utils.SpecificationError(
                          f"Value '{value}' cannot be converted to int for '{int_key}' attribute.")
  
          for key, value in data.to_dict().items():
              if key == 'memory':
                  self.info('memory', f"{value} MB", 'green')
              elif key == 'disk':
                  self.info('disk', f"{value} GB", 'green')
              elif key == 'connection':
                  self.verbose('connection', value, 'green')
              elif value is not None:
                  self.info(key, value, 'green')
  
          # Create a new GuestTestcloud instance and start it
          self._guest = GuestTestcloud(data, name=self.name, parent=self.step)
          self._guest.start()
  
      def guest(self) -> Optional[tmt.Guest]:
          """ Return the provisioned guest """
          return self._guest
  
      @classmethod
      def clean_images(cls, clean: Any, dry: Any) -> bool:
          """ Remove the testcloud images """
          clean.info('testcloud', shift=1, color='green')
          if not os.path.exists(TESTCLOUD_IMAGES):
              clean.warn(
                  f"Directory '{TESTCLOUD_IMAGES}' does not exist.", shift=2)
              return True
          successful = True
          for image in os.listdir(TESTCLOUD_IMAGES):
              image = os.path.join(TESTCLOUD_IMAGES, image)
              if dry:
                  clean.verbose(f"Would remove '{image}'.", shift=2)
              else:
                  clean.verbose(f"Removing '{image}'.", shift=2)
                  try:
                      os.remove(image)
                  except OSError:
                      clean.error(f"Failed to remove '{image}'.", shift=2)
                      successful = False
          return successful
  
  
  class GuestTestcloud(tmt.GuestSsh):
      """
      Testcloud Instance
  
      The following keys are expected in the 'data' dictionary::
  
          image ...... qcov image name or url
          user ....... user name to log in
          memory ..... memory size for vm
          disk ....... disk size for vm
          connection . either session (default) or system, to be passed to qemu
          arch ....... architecture for the VM, host arch is the default
      """
  
      # TODO: Revisit this `type: ignore` once `Guest` becomes a generic type
      _data_class = TestcloudGuestData  # type: ignore
  
      image: str
      image_url: Optional[str]
      instance_name: Optional[str]
      memory: int
      disk: str
      connection: str
      arch: str
  
      # Not to be saved, recreated from image_url/instance_name/... every
      # time guest is instantiated.
      _image: Optional['testcloud.image.Image'] = None  # type: ignore
      _instance: Optional['testcloud.instance.Instance'] = None  # type: ignore
  
      def _get_url(self, url: str, message: str) -> requests.Response:
          """ Get url, retry when fails, return response """
  
          def try_get_url() -> requests.Response:
              try:
                  with retry_session() as session:
                      response = session.get(url)
  
                  if response.ok:
                      return response
  
              except requests.RequestException:
                  pass
              finally:
                  raise tmt.utils.WaitingIncomplete()
  
          try:
              return tmt.utils.wait(
                  self, try_get_url, datetime.timedelta(
                      seconds=DEFAULT_CONNECT_TIMEOUT), tick=1)
  
          except tmt.utils.WaitingTimedOutError:
              raise ProvisionError(
                  f'Failed to {message} in {DEFAULT_CONNECT_TIMEOUT}s.')
  
      def _guess_image_url(self, name: str) -> str:
          """ Guess image url for given name """
  
          # Try to check if given url is a local file
          if os.path.isabs(name) and os.path.isfile(name):
              return f'file://{name}'
  
          name = name.lower().strip()
          url = None
  
          # Map fedora aliases (e.g. rawhide, fedora, fedora-32, f-32, f32)
          matched_fedora = re.match(r'^f(edora)?-?(\d+)$', name)
          # Map fedora coreos aliases (e.g. stable, next or testing)
          matched_fedora_coreos = re.match(r'^f(edora-coreos)?-?(stable|testing|next)$', name)
          # Map centos aliases (e.g. centos:X, centos, centos-stream:X)
          matched_centos = [re.match(r'^c(entos)?-?(\d+)$', name),
                            re.match(r'^c(entos-stream)?-?(\d+)$', name)]
          matched_ubuntu = re.match(r'^u(buntu)?-?(\w+)$', name)
          matched_debian = re.match(r'^d(ebian)?-?(\w+)$', name)
  
          assert testcloud is not None
          # Plain name match means we want the latest release
          if name == 'fedora':
              url = testcloud.util.get_fedora_image_url("latest", self.arch)
          elif name == 'fedora-coreos':
              url = testcloud.util.get_fedora_image_url("stable", self.arch)
          elif name == 'centos':
              url = testcloud.util.get_centos_image_url("latest", self.arch)
          elif name == 'centos-stream':
              url = testcloud.util.get_centos_image_url(
                  "latest", stream=True, arch=self.arch)
          elif name == 'ubuntu':
              url = testcloud.util.get_ubuntu_image_url("latest", self.arch)
          elif name == 'debian':
              url = testcloud.util.get_debian_image_url("latest", self.arch)
  
          elif matched_fedora:
              url = testcloud.util.get_fedora_image_url(
                  matched_fedora.group(2), self.arch)
          elif matched_fedora_coreos:
              url = testcloud.util.get_fedora_image_url(
                  matched_fedora_coreos.group(2), self.arch)
          elif matched_centos[0]:
              url = testcloud.util.get_centos_image_url(
                  matched_centos[0].group(2), self.arch)
          elif matched_centos[1]:
              url = testcloud.util.get_centos_image_url(
                  matched_centos[1].group(2), stream=True, arch=self.arch)
          elif matched_ubuntu:
              url = testcloud.util.get_ubuntu_image_url(
                  matched_ubuntu.group(2), self.arch)
          elif matched_debian:
              url = testcloud.util.get_debian_image_url(
                  matched_debian.group(2), self.arch)
          elif 'rawhide' in name:
              url = testcloud.util.get_fedora_image_url("rawhide", self.arch)
  
          if not url:
              raise ProvisionError(f"Could not map '{name}' to compose.")
          return cast(str, url)
  
      @staticmethod
      def _create_template() -> None:
          """ Create libvirt domain template """
          # Write always to ovewrite possible outdated version
          with open(DOMAIN_TEMPLATE_FILE, 'w') as template:
              template.write(DOMAIN_TEMPLATE)
  
      def wake(self) -> None:
          """ Wake up the guest """
          self.debug(
              f"Waking up testcloud instance '{self.instance_name}'.",
              level=2, shift=0)
          self.prepare_config()
          assert testcloud is not None
          self._image = testcloud.image.Image(self.image_url)
          self._instance = testcloud.instance.Instance(
              self.instance_name, image=self._image,
              connection=f"qemu:///{self.connection}", desired_arch=self.arch)
  
      def prepare_ssh_key(self, key_type: Optional[str] = None) -> None:
          """ Prepare ssh key for authentication """
          # Create ssh key paths
          key_name = "id_{}".format(key_type if key_type is not None else 'rsa')
          assert self.workdir is not None
          self.key = [os.path.join(self.workdir, key_name)]
          self.pubkey = os.path.join(self.workdir, f'{key_name}.pub')
  
          # Generate ssh key
          self.debug('Generating an ssh key.')
          command = ["ssh-keygen", "-f", self.key[0], "-N", ""]
          if key_type is not None:
              command.extend(["-t", key_type])
          self.run(command)
          with open(self.pubkey, 'r') as pubkey:
              self.config.USER_DATA = USER_DATA.format(
                  user_name=self.user, public_key=pubkey.read())
          with open(self.pubkey, 'r') as pubkey:
              self.config.COREOS_DATA = COREOS_DATA.format(
                  user_name=self.user, public_key=pubkey.read())
  
      def prepare_config(self) -> None:
          """ Prepare common configuration """
          import_testcloud()
  
          # Get configuration
          assert testcloud is not None
          self.config = testcloud.config.get_config()
  
          # Make sure download progress is disabled unless in debug mode,
          # so it does not spoil our logging
          self.config.DOWNLOAD_PROGRESS = self.opt('debug') > 2
          self.config.DOWNLOAD_PROGRESS_VERBOSE = False
  
          # Configure to tmt's storage directories
          self.config.DATA_DIR = TESTCLOUD_DATA
          self.config.STORE_DIR = TESTCLOUD_IMAGES
  
      def start(self) -> None:
          """ Start provisioned guest """
          if self.opt('dry'):
              return
          # Make sure required directories exist
          os.makedirs(TESTCLOUD_DATA, exist_ok=True)
          os.makedirs(TESTCLOUD_IMAGES, exist_ok=True)
  
          # Make sure libvirt domain template exists
          GuestTestcloud._create_template()
  
          # Prepare config
          self.prepare_config()
  
          # Kick off image URL with the given image
          self.image_url = self.image
  
          # If image does not start with http/https/file, consider it a
          # mapping value and try to guess the URL
          if not re.match(r'^(?:https?|file)://.*', self.image_url):
              self.image_url = self._guess_image_url(self.image_url)
              self.debug(f"Guessed image url: '{self.image_url}'", level=3)
  
          # Initialize and prepare testcloud image
          assert testcloud is not None
          self._image = testcloud.image.Image(self.image_url)
          self.verbose('qcow', self._image.name, 'green')
          if not os.path.exists(self._image.local_path):
              self.info('progress', 'downloading...', 'cyan')
          try:
              self._image.prepare()
          except FileNotFoundError as error:
              raise ProvisionError(
                  f"Image '{self._image.local_path}' not found.", original=error)
          except (testcloud.exceptions.TestcloudPermissionsError,
                  PermissionError) as error:
              raise ProvisionError(
                  f"Failed to prepare the image. Check the '{TESTCLOUD_IMAGES}' "
                  f"directory permissions.", original=error)
  
          # Create instance
          self.instance_name = self._tmt_name()
          self._instance = testcloud.instance.Instance(
              name=self.instance_name, image=self._image,
              connection=f"qemu:///{self.connection}", desired_arch=self.arch)
          self.verbose('name', self.instance_name, 'green')
  
          # Decide if we want to multiply timeouts when emulating an architecture
          time_coeff = NON_KVM_TIMEOUT_COEF if not self._instance.kvm else 1
  
          # Decide which networking setup to use
          # Autodetect works with libguestfs python bindings
          # We fall back to basic heuristics based on file name
          # without that installed (eg. from pypi).
          # https://bugzilla.redhat.com/show_bug.cgi?id=1075594
          try:
              import guestfs  # noqa: F401
          except ImportError:
              match_legacy = re.search(
                  r'(rhel|centos).*-7', self.image_url.lower())
              if match_legacy:
                  self._instance.pci_net = "e1000"
              else:
                  self._instance.pci_net = "virtio-net-pci"
  
          # Prepare ssh key
          # TODO: Maybe... some better way to do this?
          if "coreos" in self.image.lower():
              self._instance.coreos = True
              self._instance.ssh_path = self.key[0]
          self.prepare_ssh_key(SSH_KEYGEN_TYPE)
  
          # Boot the virtual machine
          self.info('progress', 'booting...', 'cyan')
          self._instance.ram = self.memory
          self._instance.disk_size = self.disk
          assert libvirt is not None
          try:
              self._instance.prepare()
              self._instance.spawn_vm()
              self._instance.start(DEFAULT_BOOT_TIMEOUT * time_coeff)
          except (testcloud.exceptions.TestcloudInstanceError,
                  libvirt.libvirtError) as error:
              raise ProvisionError(
                  f'Failed to boot testcloud instance ({error}).')
          self.guest = self._instance.get_ip()
          self.port = int(self._instance.get_instance_port())
          self.verbose('ip', self.guest, 'green')
          self.verbose('port', str(self.port), 'green')
          self._instance.create_ip_file(self.guest)
  
          # Wait a bit until the box is up
          if not self.reconnect(
                  timeout=DEFAULT_CONNECT_TIMEOUT *
                  time_coeff,
                  tick=1):
              raise ProvisionError(
                  f"Failed to connect in {DEFAULT_CONNECT_TIMEOUT * time_coeff}s.")
  
          if not self._instance.kvm:
              self.debug(
                  f"Waiting {NON_KVM_ADDITIONAL_WAIT} seconds "
                  f"for non-kvm instance...")
              time.sleep(NON_KVM_ADDITIONAL_WAIT)
  
      def stop(self) -> None:
          """ Stop provisioned guest """
          super().stop()
          # Stop only if the instance successfully booted
          if self._instance and self.guest:
              self.debug(f"Stopping testcloud instance '{self.instance_name}'.")
              assert testcloud is not None
              try:
                  self._instance.stop()
              except testcloud.exceptions.TestcloudInstanceError as error:
                  raise tmt.utils.ProvisionError(
                      f"Failed to stop testcloud instance: {error}")
              self.info('guest', 'stopped', 'green')
  
      def remove(self) -> None:
          """ Remove the guest (disk cleanup) """
          if self._instance:
              self.debug(f"Removing testcloud instance '{self.instance_name}'.")
              try:
                  self._instance.remove(autostop=True)
              except FileNotFoundError as error:
                  raise tmt.utils.ProvisionError(
                      f"Failed to remove testcloud instance: {error}")
              self.info('guest', 'removed', 'green')
  
      def reboot(self,
                 hard: bool = False,
                 command: Optional[str] = None,
                 timeout: Optional[int] = None,
                 tick: float = tmt.utils.DEFAULT_WAIT_TICK,
                 tick_increase: float = tmt.utils.DEFAULT_WAIT_TICK_INCREASE) -> Any:
          """ Reboot the guest, return True if successful """
          # Use custom reboot command if provided
          if command:
              return super().reboot(hard=hard, command=command)
          if not self._instance:
              raise tmt.utils.ProvisionError("No instance initialized.")
          self._instance.reboot(soft=not hard)
          return self.reconnect(timeout=timeout)
  
files: tmt/steps/provision/testcloud.py
files: import dataclasses
  import os
  import types
  from typing import Any, List, Optional, cast, overload
  
  import click
  
  import tmt
  import tmt.options
  import tmt.steps
  import tmt.steps.report
  
  DEFAULT_NAME = "junit.xml"
  
  junit_xml: Optional[types.ModuleType] = None
  
  # Thanks to import burried in a function, we don't get the regular missing annotations
  # but "name ... is not defined". Define a special type to follow the test suite instance
  # around, and if junit_xml ever gets annotations, we can replace it with provided type.
  JunitTestSuite = Any
  
  
  def import_junit_xml() -> None:
      """
      Import junit_xml module only when needed
  
      Until we have a separate package for each plugin.
      """
      global junit_xml
      try:
          import junit_xml
      except ImportError:
          raise tmt.utils.ReportError(
              "Missing 'junit-xml', fixable by 'pip install tmt[report-junit]'.")
  
  
  @overload
  def duration_to_seconds(duration: str) -> int: pass
  
  
  @overload
  def duration_to_seconds(duration: None) -> None: pass
  
  
  def duration_to_seconds(duration: Optional[str]) -> Optional[int]:
      """ Convert valid duration string in to seconds """
      if duration is None:
          return None
      try:
          h, m, s = duration.split(':')
          return int(h) * 3600 + int(m) * 60 + int(s)
      except Exception as error:
          raise tmt.utils.ReportError(
              f"Malformed duration '{duration}' ({error}).")
  
  
  def make_junit_xml(report: "tmt.steps.report.ReportPlugin") -> JunitTestSuite:
      """ Create junit xml object """
      import_junit_xml()
      assert junit_xml
  
      suite = junit_xml.TestSuite(report.step.plan.name)
  
      for result in report.step.plan.execute.results():
          try:
              main_log = report.step.plan.execute.read(result.log[0])
          except (IndexError, AttributeError):
              main_log = None
          case = junit_xml.TestCase(
              result.name,
              classname=None,
              elapsed_sec=duration_to_seconds(result.duration),
              stdout=main_log)
          # Map tmt OUTCOME to JUnit states
          if result.result == "error":
              case.add_error_info(result.result)
          elif result.result == "fail":
              case.add_failure_info(result.result)
          elif result.result == "info":
              case.add_skipped_info(result.result)
          elif result.result == "warn":
              case.add_error_info(result.result)
          # Passed state is the default
          suite.test_cases.append(case)
  
      return cast(JunitTestSuite, suite)
  
  
  @dataclasses.dataclass
  class ReportJUnitData(tmt.steps.StepData):
      file: Optional[str] = None
  
  
  @tmt.steps.provides_method('junit')
  class ReportJUnit(tmt.steps.report.ReportPlugin):
      """
      Write test results in JUnit format
  
      When FILE is not specified output is written to the 'junit.xml'
      located in the current workdir.
      """
  
      _data_class = ReportJUnitData
  
      @classmethod
      def options(cls, how: Optional[str] = None) -> List[tmt.options.ClickOptionDecoratorType]:
          """ Prepare command line options for connect """
          options = super().options(how)
          options[:0] = [
              click.option(
                  '--file', metavar='FILE',
                  help='Path to the file to store junit to'),
              ]
          return options
  
      def go(self) -> None:
          """ Read executed tests and write junit """
          super().go()
  
          suite = make_junit_xml(self)
  
          assert self.workdir is not None
          f_path = self.get("file", os.path.join(self.workdir, DEFAULT_NAME))
          try:
              with open(f_path, 'w') as fw:
                  if hasattr(junit_xml, 'to_xml_report_file'):
                      junit_xml.to_xml_report_file(fw, [suite])  # type: ignore[union-attr]
                  else:
                      # For older junit-xml
                      junit_xml.TestSuite.to_file(fw, [suite])  # type: ignore[union-attr]
              self.info("output", f_path, 'yellow')
          except Exception as error:
              raise tmt.utils.ReportError(
                  f"Failed to write the output '{f_path}' ({error}).")
  
files: tmt/steps/report/junit.py
files: import dataclasses
  import os
  import types
  from typing import Any, List, Optional, cast, overload
  
  import click
  
  import tmt
  import tmt.options
  import tmt.steps
  import tmt.steps.report
  
  DEFAULT_NAME = "junit.xml"
  
  junit_xml: Optional[types.ModuleType] = None
  
  # Thanks to import burried in a function, we don't get the regular missing annotations
  # but "name ... is not defined". Define a special type to follow the test suite instance
  # around, and if junit_xml ever gets annotations, we can replace it with provided type.
  JunitTestSuite = Any
  
  
  def import_junit_xml() -> None:
      """
      Import junit_xml module only when needed
  
      Until we have a separate package for each plugin.
      """
      global junit_xml
      try:
          import junit_xml
      except ImportError:
          raise tmt.utils.ReportError(
              "Missing 'junit-xml', fixable by 'pip install tmt[report-junit]'.")
  
  
  @overload
  def duration_to_seconds(duration: str) -> int: pass
  
  
  @overload
  def duration_to_seconds(duration: None) -> None: pass
  
  
  def duration_to_seconds(duration: Optional[str]) -> Optional[int]:
      """ Convert valid duration string in to seconds """
      if duration is None:
          return None
      try:
          h, m, s = duration.split(':')
          return int(h) * 3600 + int(m) * 60 + int(s)
      except Exception as error:
          raise tmt.utils.ReportError(
              f"Malformed duration '{duration}' ({error}).")
  
  
  def make_junit_xml(report: "tmt.steps.report.ReportPlugin") -> JunitTestSuite:
      """ Create junit xml object """
      import_junit_xml()
      assert junit_xml
  
      suite = junit_xml.TestSuite(report.step.plan.name)
  
      for result in report.step.plan.execute.results():
          try:
              main_log = report.step.plan.execute.read(result.log[0])
          except (IndexError, AttributeError):
              main_log = None
          case = junit_xml.TestCase(
              result.name,
              classname=None,
              elapsed_sec=duration_to_seconds(result.duration),
              stdout=main_log)
          # Map tmt OUTCOME to JUnit states
          if result.result == "error":
              case.add_error_info(result.result)
          elif result.result == "fail":
              case.add_failure_info(result.result)
          elif result.result == "info":
              case.add_skipped_info(result.result)
          elif result.result == "warn":
              case.add_error_info(result.result)
          # Passed state is the default
          suite.test_cases.append(case)
  
      return cast(JunitTestSuite, suite)
  
  
  @dataclasses.dataclass
  class ReportJUnitData(tmt.steps.StepData):
      file: Optional[str] = None
  
  
  @tmt.steps.provides_method('junit')
  class ReportJUnit(tmt.steps.report.ReportPlugin):
      """
      Write test results in JUnit format
  
      When FILE is not specified output is written to the 'junit.xml'
      located in the current workdir.
      """
  
      _data_class = ReportJUnitData
  
      @classmethod
      def options(cls, how: Optional[str] = None) -> List[tmt.options.ClickOptionDecoratorType]:
          """ Prepare command line options for connect """
          options = super().options(how)
          options[:0] = [
              click.option(
                  '--file', metavar='FILE',
                  help='Path to the file to store junit to'),
              ]
          return options
  
      def go(self) -> None:
          """ Read executed tests and write junit """
          super().go()
  
          suite = make_junit_xml(self)
  
          assert self.workdir is not None
          f_path = self.get("file", os.path.join(self.workdir, DEFAULT_NAME))
          try:
              with open(f_path, 'w') as fw:
                  if hasattr(junit_xml, 'to_xml_report_file'):
                      junit_xml.to_xml_report_file(fw, [suite])  # type: ignore
                  else:
                      # For older junit-xml
                      junit_xml.TestSuite.to_file(fw, [suite])  # type: ignore
              self.info("output", f_path, 'yellow')
          except Exception as error:
              raise tmt.utils.ReportError(
                  f"Failed to write the output '{f_path}' ({error}).")
  
files: tmt/steps/report/junit.py
files: 
  """ Test Metadata Utilities """
  
  import contextlib
  import copy
  import dataclasses
  import datetime
  import functools
  import glob
  import io
  import os
  import pprint
  import re
  import shlex
  import shutil
  import subprocess
  import sys
  import time
  import unicodedata
  from collections import OrderedDict
  from functools import lru_cache
  from pathlib import Path
  from threading import Thread
  from typing import (IO, TYPE_CHECKING, Any, Callable, Dict, Generator,
                      Iterable, List, NamedTuple, Optional, Pattern, Tuple, Type,
                      TypeVar, Union, cast, overload)
  
  import click
  import fmf
  import jsonschema
  import pkg_resources
  import requests
  import requests.adapters
  import requests.packages.urllib3.util.retry
  import urllib3.exceptions
  from click import echo, style, wrap_text
  from ruamel.yaml import YAML, scalarstring
  from ruamel.yaml.comments import CommentedMap
  
  if sys.version_info >= (3, 8):
      from typing import Literal, Protocol
  else:
      from typing_extensions import Literal, Protocol
  
  
  if TYPE_CHECKING:
      import tmt.base
      import tmt.steps
  
  log = fmf.utils.Logging('tmt').logger
  
  # Default workdir root and max
  WORKDIR_ROOT = '/var/tmp/tmt'
  WORKDIR_MAX = 1000
  
  # Log in workdir
  LOG_FILENAME = 'log.txt'
  
  # Maximum number of lines of stdout/stderr to show upon errors
  OUTPUT_LINES = 100
  # Default output width
  OUTPUT_WIDTH = 79
  
  # Hierarchy indent
  INDENT = 4
  
  # Default name and order for step plugins
  DEFAULT_NAME = 'default'
  DEFAULT_PLUGIN_ORDER = 50
  DEFAULT_PLUGIN_ORDER_MULTIHOST = 10
  DEFAULT_PLUGIN_ORDER_REQUIRES = 70
  DEFAULT_PLUGIN_ORDER_RECOMMENDS = 75
  
  # Config directory
  CONFIG_PATH = '~/.config/tmt'
  
  # Special process return code
  PROCESS_TIMEOUT = 124
  
  # Default select.select(timeout) in seconds
  DEFAULT_SELECT_TIMEOUT = 5
  
  # Default shell and options to be set for all shell scripts
  DEFAULT_SHELL = "/bin/bash"
  SHELL_OPTIONS = 'set -eo pipefail'
  
  # Defaults for HTTP/HTTPS retries and timeouts (see `retry_session()`).
  DEFAULT_RETRY_SESSION_RETRIES: int = 3
  DEFAULT_RETRY_SESSION_BACKOFF_FACTOR: float = 0.1
  
  # Defaults for HTTP/HTTPS retries for getting environment file
  # Retry with exponential backoff, maximum duration ~511 seconds
  ENVFILE_RETRY_SESSION_RETRIES: int = 10
  ENVFILE_RETRY_SESSION_BACKOFF_FACTOR: float = 1
  
  # Default for wait()-related options
  DEFAULT_WAIT_TICK: float = 30.0
  DEFAULT_WAIT_TICK_INCREASE: float = 1.0
  
  # A stand-in variable for generic use.
  T = TypeVar('T')
  
  # A FMF context type, representing name/values context.
  FmfContextType = Dict[str, List[str]]
  
  # A "environment" type, representing name/value environment variables.
  EnvironmentType = Dict[str, str]
  
  # Workdir argument type, can be True, a string, a path or None
  WorkdirArgumentType = Union[Literal[True], str, None]
  
  # Workdir type, can be None or a string
  WorkdirType = Optional[str]
  
  # Option to skip to initialize work tree in plan
  PLAN_SKIP_WORKTREE_INIT = 'plan_skip_worktree_init'
  
  
  class BaseLoggerFnType(Protocol):
      def __call__(
              self,
              key: str,
              value: Optional[str] = None,
              color: Optional[str] = None,
              shift: int = 0,
              level: int = 1,
              err: bool = False) -> None:
          pass
  
  
  class LevelessLoggerFnType(Protocol):
      def __call__(
              self,
              key: str,
              value: Optional[str] = None,
              color: Optional[str] = None,
              shift: int = 0,
              err: bool = False) -> None:
          pass
  
  
  class SemanticLoggerFnType(Protocol):
      def __call__(self, message: str, shift: int = 0) -> None:
          pass
  
  
  LoggerFnType = Union[
      BaseLoggerFnType,
      LevelessLoggerFnType,
      SemanticLoggerFnType]
  
  
  def indent(
          key: str,
          value: Optional[str] = None,
          color: Optional[str] = None,
          level: int = 0) -> str:
      """
      Indent a key/value message.
  
      If both ``key`` and ``value`` are specified, ``{key}: {value}``
      message is rendered. Otherwise, just ``key`` is used alone. If
      ``value`` contains multiple lines, each but the very first line is
      indented by one extra level.
  
      :param value: optional value to print at right side of ``key``.
      :param color: optional color to apply on ``key``.
      :param level: number of indentation levels. Each level is indented
                    by :py:data:`INDENT` spaces.
      """
  
      indent = ' ' * INDENT * level
      deeper = ' ' * INDENT * (level + 1)
  
      # Colorize
      if color is not None:
          key = style(key, fg=color)
  
      # Handle key only
      if value is None:
          message = key
  
      # Handle key + value
      else:
          # Multiline content indented deeper
          if isinstance(value, str):
              lines = value.splitlines()
              if len(lines) > 1:
                  value = ''.join([f"\n{deeper}{line}" for line in lines])
  
          message = f'{key}: {value}'
  
      return indent + message
  
  
  class Config:
      """ User configuration """
  
      def __init__(self) -> None:
          """ Initialize config directory path """
          self.path = os.path.expanduser(CONFIG_PATH)
          if not os.path.exists(self.path):
              try:
                  os.makedirs(self.path)
              except OSError as error:
                  raise GeneralError(
                      f"Failed to create config '{self.path}'.\n{error}")
  
      def last_run(self, run_id: Optional[str] = None) -> Optional[str]:
          """ Get and set last run id """
          symlink = os.path.join(self.path, 'last-run')
          if run_id:
              try:
                  os.remove(symlink)
              except OSError:
                  pass
              try:
                  os.symlink(run_id, symlink)
              except FileExistsError:
                  # Race when tmt runs in parallel
                  log.warning(f"Race condition, unable to save last run '{run_id}'.")
              except OSError as error:
                  raise GeneralError(
                      f"Unable to save last run '{self.path}'.\n{error}")
              return run_id
          if os.path.islink(symlink):
              return os.path.realpath(symlink)
          return None
  
  
  class StreamLogger(Thread):
      """
      Reading pipes of running process in threads.
  
      Code based on:
      https://github.com/packit/packit/blob/main/packit/utils/logging.py#L10
      """
  
      def __init__(self,
                   stream: Optional[IO[bytes]],
                   log_header: str,
                   logger: BaseLoggerFnType) -> None:
          super().__init__(daemon=True)
          self.stream = stream
          self.output: List[str] = []
          self.log_header = log_header
          self.logger = logger
  
      def run(self) -> None:
          if self.stream is None:
              return
  
          for _line in self.stream:
              line = _line.decode('utf-8', errors='replace')
              if line != '':
                  self.logger(
                      self.log_header,
                      line.rstrip('\n'),
                      'yellow',
                      level=3)
              self.output.append(line)
  
      def get_output(self) -> str:
          return "".join(self.output)
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Common
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  CommonDerivedType = TypeVar('CommonDerivedType', bound='Common')
  
  
  class CommandOutput(NamedTuple):
      stdout: Optional[str]
      stderr: Optional[str]
  
  
  class Common:
      """
      Common shared stuff
  
      Takes care of command line context, options and workdir handling.
      Provides logging functions info(), verbose() and debug().
      Implements read() and write() for comfortable file access.
      Provides the run() method for easy command execution.
      """
  
      # Command line context, options and workdir
      _context: Optional[click.Context] = None
      # When set to true, _opt will be ignored (default will be returned)
      ignore_class_options: bool = False
      _options: Dict[str, Any] = dict()
      _workdir: WorkdirType = None
  
      # TODO: must be declared outside of __init__(), because it must exist before
      # __init__() gets called to allow logging helpers work correctly when used
      # from mixins. But that's not very clean, is it? :( Maybe decoupling logging
      # from Common class would help, such a class would be able to initialize
      # itself without involving the rest of Common code. On the other hand,
      # Common owns workdir, for example, whose value affects logging too, so no
      # clear solution so far.
      #
      # Note: cannot use CommonDerivedType - it's a TypeVar filled in by the type
      # given to __init__() and therefore the type it's representing *now* is
      # unknown. but we know `parent` will be derived from `Common` class, so it's
      # mostly fine.
      parent: Optional['Common'] = None
  
      def __init__(
              self,
              parent: Optional[CommonDerivedType] = None,
              name: Optional[str] = None,
              workdir: WorkdirArgumentType = None,
              context: Optional[click.Context] = None,
              relative_indent: int = 1,
              **kwargs: Any) -> None:
          """
          Initialize name and relation with the parent object
  
          Prepare the workdir for provided id / directory path
          or generate a new workdir name if workdir=True given.
          Store command line context and options for future use
          if context is provided.
          """
          # Use lowercase class name as the default name
          self.name = name or self.__class__.__name__.lower()
          self.parent = parent
  
          # Relative log indent level shift against the parent
          self._relative_indent = relative_indent
  
          # Store command line context
          if context:
              self._save_context_to_instance(context)
  
          # Initialize the workdir if requested
          self._workdir_load(workdir)
  
      def __str__(self) -> str:
          """ Name is the default string representation """
          return self.name
  
      @classmethod
      def _save_context(cls, context: click.Context) -> None:
          """ Save provided command line context and options for future use """
          cls._context = context
          cls._options = context.params
  
      def _save_context_to_instance(self, context: click.Context) -> None:
          """ Save provided command line context and options to the instance """
          self._context = context
          self._options = context.params
  
      @overload
      @classmethod
      def _opt(cls, option: str) -> Any:
          pass
  
      @overload
      @classmethod
      def _opt(cls, option: str, default: T) -> T:
          pass
  
      @classmethod
      def _opt(cls, option: str, default: Any = None) -> Any:
          """ Get an option from the command line context (class version) """
          if cls.ignore_class_options:
              return default
          return cls._options.get(option, default)
  
      def _fmf_context(self) -> FmfContextType:
          """ Return the current fmf context """
          if self._context is None:
              return dict()
  
          try:
              return cast(FmfContextType, self._context.obj.fmf_context)
          except AttributeError:
              return dict()
  
      def opt(self, option: str, default: Optional[Any] = None) -> Any:
          """
          Get an option from the command line options
  
          Checks also parent options. For flags (boolean values) parent's
          True wins over child's False (e.g. run --quiet enables quiet
          mode for all included plans and steps).
  
          For options that can be used multiple times, the child overrides
          the parent if it was defined (e.g. run -av provision -vvv runs
          all steps except for provision in mildly verbose mode, provision
          is run with the most verbosity).
  
          Environment variables override command line options.
          """
          # Translate dashes to underscores to match click's conversion
          option = option.replace('-', '_')
          # Check the environment first
          if option == 'debug':
              try:
                  debug = os.environ['TMT_DEBUG']
                  return int(debug)
              except ValueError:
                  raise GeneralError(
                      f"Invalid debug level '{debug}', use an integer.")
              except KeyError:
                  pass
  
          # Get local option
          local = self._options.get(option, default)
          # Check parent option
          parent = None
          if self.parent:
              parent = self.parent.opt(option)
          # Special handling for special flags (parent's yes always wins)
          if option in ['quiet', 'force', 'dry']:
              return parent if parent else local
          # Special handling for counting options (child overrides the
          # parent if it was defined)
          elif option in ['debug', 'verbose']:
              winner = local if local else parent
              if winner is None:
                  winner = 0
              return winner
          else:
              return parent if parent is not None else local
  
      def _level(self) -> int:
          """ Hierarchy level """
          if self.parent is None:
              return -1
          else:
              return self.parent._level() + self._relative_indent
  
      def _indent(
              self,
              key: str,
              value: Optional[str] = None,
              color: Optional[str] = None,
              shift: int = 0) -> str:
          """ Indent message according to the object hierarchy """
  
          return indent(
              key,
              value=value,
              color=color,
              level=self._level() + shift)
  
      def _log(self, message: str) -> None:
          """ Append provided message to the current log """
          # Nothing to do if there is no workdir
          if self.workdir is None:
              return
  
          # Store log only in the top parent
          if self.parent:
              self.parent._log(message)
          else:
              with open(os.path.join(self.workdir, LOG_FILENAME), 'a') as log:
                  log.write(datetime.datetime.utcnow().strftime('%H:%M:%S') + ' '
                            + remove_color(message) + '\n')
  
      def print(
              self,
              key: str,
              value: Optional[str] = None,
              color: Optional[str] = None,
              shift: int = 0,
              err: bool = False) -> None:
          """ Print a message regardless the quiet mode """
          self._log(self._indent(key, value, color=None, shift=shift))
          echo(self._indent(key, value, color, shift), err=err)
  
      def info(
              self,
              key: str,
              value: Optional[str] = None,
              color: Optional[str] = None,
              shift: int = 0,
              err: bool = False) -> None:
          """ Show a message unless in quiet mode """
          self._log(self._indent(key, value, color=None, shift=shift))
          if not self.opt('quiet'):
              echo(self._indent(key, value, color, shift), err=err)
  
      def verbose(
              self,
              key: str,
              value: Optional[str] = None,
              color: Optional[str] = None,
              shift: int = 0,
              level: int = 1,
              err: bool = False) -> None:
          """
          Show message if in requested verbose mode level
  
          In quiet mode verbose messages are not displayed.
          """
          self._log(self._indent(key, value, color=None, shift=shift))
          if not self.opt('quiet') and self.opt('verbose') >= level:
              echo(self._indent(key, value, color, shift), err=err)
  
      def debug(
              self,
              key: str,
              value: Optional[str] = None,
              color: Optional[str] = None,
              shift: int = 0,
              level: int = 1,
              err: bool = False) -> None:
          """
          Show message if in requested debug mode level
  
          In quiet mode debug messages are not displayed.
          """
          self._log(self._indent(key, value, color=None, shift=shift))
          if not self.opt('quiet') and self.opt('debug') >= level:
              echo(self._indent(key, value, color, shift), err=err)
  
      def warn(self, message: str, shift: int = 0) -> None:
          """ Show a yellow warning message on info level, send to stderr """
          self.info('warn', message, color='yellow', shift=shift, err=True)
  
      def fail(self, message: str, shift: int = 0) -> None:
          """ Show a red failure message on info level, send to stderr """
          self.info('fail', message, color='red', shift=shift, err=True)
  
      def _run(self,
               command: Union[str, List[str]],
               cwd: Optional[str],
               shell: bool,
               env: Optional[EnvironmentType],
               log: Optional[BaseLoggerFnType],
               join: bool = False,
               interactive: bool = False,
               timeout: Optional[int] = None) -> CommandOutput:
          """
          Run command, capture the output
  
          By default stdout and stderr are captured separately.
          Use join=True to merge stderr into stdout.
          Use timeout=<seconds> to finish process after given time
          """
          # By default command ouput is logged using debug
          if not log:
              log = self.debug
          # Prepare the environment
          if env:
              if not isinstance(env, dict):
                  raise GeneralError(f"Invalid environment '{env}'.")
              # Do not modify current process environment
              environment = os.environ.copy()
              environment.update(env)
          else:
              environment = None
          self.debug('environment', pprint.pformat(environment), level=4)
  
          # Set only for shell=True as it would affect command
          executable = DEFAULT_SHELL if shell else None
  
          # Run the command in interactive mode if requested
          if interactive:
              try:
                  subprocess.run(
                      command, cwd=cwd, shell=shell, env=environment, check=True,
                      executable=executable)
              except subprocess.CalledProcessError:
                  # Interactive mode can return non-zero if the last command
                  # failed, ignore errors here
                  pass
              finally:
                  return CommandOutput(None, None)
  
          # Create the process
          try:
              process = subprocess.Popen(
                  command, cwd=cwd, shell=shell, env=environment,
                  stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
                  stderr=subprocess.STDOUT if join else subprocess.PIPE,
                  executable=executable)
          except FileNotFoundError as error:
              raise RunError(
                  f"File '{error.filename}' not found.", command, 127)
  
          stdout_thread = StreamLogger(
              process.stdout, log_header='out', logger=log)
          stderr_thread = stdout_thread
          if not join:
              stderr_thread = StreamLogger(
                  process.stderr, log_header='err', logger=log)
          stdout_thread.start()
          if not join:
              stderr_thread.start()
          try:
              process.wait(timeout=timeout)
          except subprocess.TimeoutExpired:
              process.kill()
              process.returncode = PROCESS_TIMEOUT
          stdout_thread.join()
          if not join:
              stderr_thread.join()
  
          # Handle the exit code, return output
          if process.returncode != 0:
              raise RunError(
                  message=f"Command returned '{process.returncode}'.",
                  command=command,
                  returncode=process.returncode,
                  stdout=stdout_thread.get_output(),
                  stderr=stderr_thread.get_output())
          if join:
              return CommandOutput(
                  stdout_thread.get_output(), None)
          else:
              return CommandOutput(
                  stdout_thread.get_output(), stderr_thread.get_output())
  
      def run(self,
              command: Union[str, List[str]],
              message: Optional[str] = None,
              cwd: Optional[str] = None,
              dry: bool = False,
              shell: bool = False,
              env: Optional[EnvironmentType] = None,
              interactive: bool = False,
              join: bool = False,
              log: Optional[BaseLoggerFnType] = None,
              timeout: Optional[int] = None) -> CommandOutput:
          """
          Run command, give message, handle errors
  
          Command is run in the workdir be default.
          In dry mode commands are not executed unless dry=True.
          Environment is updated with variables from the 'env' dictionary.
          Output is logged using self.debug() or custom 'log' function.
          Returns stdout if join=True, (stdout, stderr) tuple otherwise.
          """
  
          # A bit of logging - command, default message, error message for later...
          if isinstance(command, (list, tuple)):
              printable_command = ' '.join(shlex.quote(s) for s in command)
          else:
              printable_command = command
  
          if message:
              self.debug(message, level=2)
  
          self.debug(f'Run command: {printable_command}', level=2)
  
          # Nothing more to do in dry mode (unless requested)
          if self.opt('dry') and not dry:
              return CommandOutput(None, None)
  
          # Run the command, handle the exit code
          cwd = cwd or self.workdir
  
          # Fail nicely if the working directory does not exist
          if cwd and not os.path.exists(cwd):
              raise GeneralError(
                  f"The working directory '{cwd}' does not exist.")
  
          try:
              return self._run(
                  command, cwd, shell, env, log, join, interactive, timeout)
          except RunError as error:
              self.debug(error.message, level=3)
              message = f"Failed to run command: {printable_command} Reason: {error.message}"
              raise RunError(
                  message, error.command, error.returncode,
                  error.stdout, error.stderr)
  
      def read(self, path: str, level: int = 2) -> str:
          """ Read a file from the workdir """
          if self.workdir:
              path = os.path.join(self.workdir, path)
          self.debug(f"Read file '{path}'.", level=level)
          try:
              with open(path, encoding='utf-8', errors='replace') as data:
                  return data.read()
          except OSError as error:
              raise FileError(f"Failed to read '{path}'.\n{error}")
  
      def write(
              self,
              path: str,
              data: Any,
              mode: str = 'w',
              level: int = 2) -> None:
          """ Write a file to the workdir """
          if self.workdir:
              path = os.path.join(self.workdir, path)
          action = 'Append to' if mode == 'a' else 'Write'
          self.debug(f"{action} file '{path}'.", level=level)
          # Dry mode
          if self.opt('dry'):
              return
          try:
              with open(path, mode, encoding='utf-8', errors='replace') as file:
                  file.write(data)
          except OSError as error:
              raise FileError(f"Failed to write '{path}'.\n{error}")
  
      def _workdir_init(self, id_: WorkdirArgumentType = None) -> None:
          """
          Initialize the work directory
  
          Workdir under WORKDIR_ROOT is used/created if 'id' is provided.
          If 'id' is a path, that directory is used instead. Otherwise a
          new workdir is created under the WORKDIR_ROOT directory.
          """
          # Prepare the workdir name from given id or path
          if isinstance(id_, str):
              # Use provided directory if full path given
              if '/' in id_:
                  workdir = id_
              # Construct directory name under workdir root
              else:
                  workdir = os.path.join(WORKDIR_ROOT, id_)
          # Weird workdir id
          elif id_ is not None:
              raise GeneralError(
                  f"Invalid workdir '{id_}', expected a string or None.")
  
          def _check_or_create_workdir_root_with_perms() -> None:
              """ If created WORKDIR_ROOT has to be 1777 for multi-user"""
              if not os.path.isdir(WORKDIR_ROOT):
                  try:
                      os.makedirs(WORKDIR_ROOT, exist_ok=True)
                      os.chmod(WORKDIR_ROOT, 0o1777)
                  except OSError as error:
                      raise FileError(f"Failed to prepare workdir '{WORKDIR_ROOT}': {error}")
  
          if id_ is None:
              # Prepare WORKDIR_ROOT first
              _check_or_create_workdir_root_with_perms()
  
              # Generated unique id or fail, has to be atomic call
              for id_bit in range(1, WORKDIR_MAX + 1):
                  directory = 'run-{}'.format(str(id_bit).rjust(3, '0'))
                  workdir = os.path.join(WORKDIR_ROOT, directory)
                  try:
                      # Call is atomic, no race possible
                      os.makedirs(workdir)
                      break
                  except FileExistsError:
                      pass
              else:
                  raise GeneralError(
                      f"Workdir full. Cleanup the '{WORKDIR_ROOT}' directory.")
          else:
              # Cleanup possible old workdir if called with --scratch
              if self.opt('scratch'):
                  self._workdir_cleanup(workdir)
  
              if workdir.startswith(WORKDIR_ROOT):
                  _check_or_create_workdir_root_with_perms()
  
              # Create the workdir
              create_directory(workdir, 'workdir', quiet=True)
          self._workdir = workdir
  
      def _workdir_name(self) -> Optional[str]:
          """ Construct work directory name from parent workdir """
          # Need the parent workdir
          if self.parent is None or self.parent.workdir is None:
              return None
          # Join parent name with self
          return os.path.join(self.parent.workdir, self.name.lstrip('/'))
  
      def _workdir_load(self, workdir: WorkdirArgumentType) -> None:
          """
          Create the given workdir if it is not None
  
          If workdir=True, the directory name is automatically generated.
          """
          if workdir is True:
              self._workdir_init()
          elif workdir is not None:
              self._workdir_init(workdir)
  
      def _workdir_cleanup(self, path: Optional[str] = None) -> None:
          """ Clean up the work directory """
          directory = path or self._workdir_name()
          if directory is not None:
              if os.path.isdir(directory):
                  self.debug(f"Clean up workdir '{directory}'.", level=2)
                  shutil.rmtree(directory)
          self._workdir = None
  
      @property
      def workdir(self) -> Optional[str]:
          """ Get the workdir, create if does not exist """
          if self._workdir is None:
              self._workdir = self._workdir_name()
              # Workdir not enabled, even parent does not have one
              if self._workdir is None:
                  return None
              # Create a child workdir under the parent workdir
              create_directory(self._workdir, 'workdir', quiet=True)
          return self._workdir
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Exceptions
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  
  class GeneralError(Exception):
      """ General error """
  
      def __init__(self, *args: Any, **kwargs: Any) -> None:
          # Store the original exception for future use
          self.original = kwargs.get('original')
  
  
  class GitUrlError(GeneralError):
      """ Remote git url is not reachable """
  
  
  class FileError(GeneralError):
      """ File operation error """
  
  
  class RunError(GeneralError):
      """ Command execution error """
  
      def __init__(
              self,
              message: str,
              command: Union[str, List[str]],
              returncode: int,
              stdout: Optional[str] = None,
              stderr: Optional[str] = None,
              *args: Any,
              **kwargs: Any) -> None:
          super().__init__(*args, **kwargs)
          self.message = message
          self.command = command
          self.returncode = returncode
          self.stdout = stdout
          self.stderr = stderr
  
  
  class MetadataError(GeneralError):
      """ General metadata error """
  
  
  class SpecificationError(MetadataError):
      """ Metadata specification error """
  
      def __init__(
              self,
              message: str,
              validation_errors: Optional[List[Tuple[jsonschema.ValidationError, str]]] = None,
              *args: Any,
              **kwargs: Any) -> None:
          super().__init__(*args, **kwargs)
  
          self.message = message
          self.validation_errors = validation_errors
  
  
  class ConvertError(MetadataError):
      """ Metadata conversion error """
  
  
  class StructuredFieldError(GeneralError):
      """ StructuredField parsing error """
  
  
  class WaitingIncomplete(GeneralError):
      """ Waiting incomplete """
  
  
  class WaitingTimedOutError(GeneralError):
      """ Waiting ran out of time """
  
  
  # Step exceptions
  
  
  class DiscoverError(GeneralError):
      """ Discover step error """
  
  
  class ProvisionError(GeneralError):
      """ Provision step error """
  
  
  class PrepareError(GeneralError):
      """ Prepare step error """
  
  
  class ExecuteError(GeneralError):
      """ Execute step error """
  
  
  class RebootTimeoutError(ExecuteError):
      """ Reboot failed due to a timeout """
  
  
  class ReportError(GeneralError):
      """ Report step error """
  
  
  class FinishError(GeneralError):
      """ Finish step error """
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Utilities
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  def quote(string: str) -> str:
      """ Surround a string with double quotes """
      return f'"{string}"'
  
  
  def ascii(text: Any) -> bytes:
      """ Transliterate special unicode characters into pure ascii """
      if not isinstance(text, str):
          text = str(text)
      return unicodedata.normalize('NFKD', text).encode('ascii', 'ignore')
  
  
  def listify(
          data: Union[Tuple[Any, ...], List[Any], str, Dict[Any, Any]],
          split: bool = False,
          keys: Optional[List[str]] = None) -> Union[List[Any], Dict[Any, Any]]:
      """
      Ensure that variable is a list, convert if necessary
      For dictionaries check all items or only those with provided keys.
      Also split strings on white-space/comma if split=True.
      """
      separator = re.compile(r'[\s,]+')
      if isinstance(data, tuple):
          data = list(data)
      if isinstance(data, list):
          return fmf.utils.split(data, separator) if split else data
      if isinstance(data, str):
          return fmf.utils.split(data, separator) if split else [data]
      if isinstance(data, dict):
          for key in keys or data:
              if key in data:
                  data[key] = listify(data[key], split=split)
          return data
      return [data]
  
  
  def copytree(
          src: str,
          dst: str,
          symlinks: bool = False,
          dirs_exist_ok: bool = False,
          ) -> Any:
      """ Similar to shutil.copytree but with dirs_exist_ok for Python < 3.8 """
      # No need to reimplement for newer python or if argument is not requested
      if not dirs_exist_ok or sys.version_info >= (3, 8):
          return shutil.copytree(
              src=src, dst=dst, symlinks=symlinks, dirs_exist_ok=dirs_exist_ok)
      # Choice was to either copy python implementation and change ONE line
      # or use rsync (or cp with shell)
      # We need to copy CONTENT of src into dst
      # so src has to end with / and dst cannot
      if src[-1] != '/':
          src += '/'
      if dst[-1] == '/':
          dst = dst[:-1]
  
      command = ["rsync", "-r"]
      if symlinks:
          command.append('-l')
      command.extend([src, dst])
  
      log.debug(f"Calling command '{command}'.")
      outcome = subprocess.run(
          command,
          stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
          stderr=subprocess.STDOUT, universal_newlines=True)
  
      if outcome.returncode != 0:
          raise shutil.Error(
              [f"Unable to copy '{src}' into '{dst}' using rsync.",
               outcome.returncode, outcome.stdout])
      return dst
  
  
  # These two are helpers for shell_to_dict and environment_to_dict -
  # there is some overlap of their functionality.
  def _add_simple_var(result: EnvironmentType, var: str) -> None:
      """
      Add a single NAME=VALUE pair into result dictionary
  
      Parse given string VAR to its constituents, NAME and VALUE, and add
      them to the provided dict.
      """
  
      matched = re.match("([^=]+)=(.*)", var)
      if not matched:
          raise GeneralError(f"Invalid variable specification '{var}'.")
      name, value = matched.groups()
      result[name] = value
  
  
  def _add_file_vars(result: EnvironmentType, filepath: str) -> None:
      """
      Add variables loaded from file into the result dictionary
  
      Load mapping from a YAML file 'filepath', and add its content -
      "name: value" entries - to the provided dict.
      """
  
      if not filepath[1:]:
          raise GeneralError(
              f"Invalid variable file specification '{filepath}'.")
  
      try:
          with open(filepath[1:], 'r') as file:
              # Handle empty file as an empty environment
              content = file.read()
              if not content:
                  log.warn(f"Empty environment file '{filepath}'.")
                  return
              file_vars = yaml_to_dict(content)
      except Exception as exception:
          raise GeneralError(
              f"Failed to load variables from '{filepath}': {exception}")
  
      for name, value in file_vars.items():
          result[name] = str(value)
  
  
  def shell_to_dict(variables: Union[str, List[str]]) -> EnvironmentType:
      """
      Convert shell-like variables into a dictionary
  
      Accepts single string or list of strings. Allowed forms are:
      'X=1'
      'X=1 Y=2 Z=3'
      ['X=1', 'Y=2', 'Z=3']
      ['X=1 Y=2 Z=3', 'A=1 B=2 C=3']
      'TXT="Some text with spaces in it"'
      """
      if not isinstance(variables, (list, tuple)):
          variables = [variables]
      result: EnvironmentType = dict()
      for variable in variables:
          if variable is None:
              continue
          for var in shlex.split(variable):
              _add_simple_var(result, var)
  
      return result
  
  
  def environment_to_dict(variables: Union[str, List[str]]) -> EnvironmentType:
      """
      Convert environment variables into a dictionary
  
      Variables may be specified in the following two ways:
  
      * NAME=VALUE pairs
      * @foo.yaml
  
      If "variable" starts with "@" character, it is treated as a path to
      a YAML file that contains "key: value" pairs which are then
      transparently loaded and added to the final dictionary.
  
      In general, allowed inputs are the same as in "shell_to_dict"
      function, with the addition of "@foo.yaml" form:
      'X=1'
      'X=1 Y=2 Z=3'
      ['X=1', 'Y=2', 'Z=3']
      ['X=1 Y=2 Z=3', 'A=1 B=2 C=3']
      'TXT="Some text with spaces in it"'
      @foo.yaml
      @../../bar.yaml
      """
  
      if not isinstance(variables, (list, tuple)):
          variables = [variables]
      result: EnvironmentType = dict()
  
      for variable in variables:
          if variable is None:
              continue
          for var in shlex.split(variable):
              if var.startswith('@'):
                  _add_file_vars(result, var)
              else:
                  _add_simple_var(result, var)
  
      return result
  
  
  @lru_cache(maxsize=None)
  def environment_file_to_dict(env_file: str, root: str = ".") -> EnvironmentType:
      """
      Read environment variables from the given file.
  
      File should be in YAML format (``.yaml`` or ``.yml`` suffixes), or in dotenv format.
  
      .. code-block:: bash
         :caption: dotenv file example
  
         A=B
         C=D
  
      .. code-block:: yaml
         :caption: YAML file example
  
         A: B
         C: D
  
      Path to each file should be relative to the metadata tree root.
  
      .. note::
  
         For loading environment variables from multiple files, see
         :py:func:`environment_files_to_dict`.
      """
  
      env_file = env_file.strip()
  
      # Fetch a remote file
      if env_file.startswith("http"):
          # Create retry session for longer retries, see #1229
          session = retry_session.create(
              retries=ENVFILE_RETRY_SESSION_RETRIES,
              backoff_factor=ENVFILE_RETRY_SESSION_BACKOFF_FACTOR,
              allowed_methods=('GET',),
              status_forcelist=(
                  429,  # Too Many Requests
                  500,  # Internal Server Error
                  502,  # Bad Gateway
                  503,  # Service Unavailable
                  504   # Gateway Timeout
                  ),
              )
          try:
              response = session.get(env_file)
              response.raise_for_status()
              content = response.text
          except requests.RequestException as error:
              raise GeneralError(
                  f"Failed to fetch the environment file from '{env_file}'. "
                  f"The problem was: '{error}'")
  
      # Read a local file
      else:
          # Ensure we don't escape from the metadata tree root
          try:
              root_path = Path(root).resolve()
              full_path = (Path(root_path) / Path(env_file)).resolve()
              full_path.relative_to(root_path)
          except ValueError:
              raise GeneralError(
                  f"The 'environment-file' path '{full_path}' is outside "
                  f"of the metadata tree root '{root}'.")
          if not Path(full_path).is_file():
              raise GeneralError(f"File '{full_path}' doesn't exist.")
  
          content = Path(full_path).read_text()
  
      # Parse yaml file
      if os.path.splitext(env_file)[1].lower() in ('.yaml', '.yml'):
          environment = parse_yaml(content)
  
      else:
          try:
              environment = parse_dotenv(content)
  
          except ValueError:
              raise GeneralError(
                  f"Failed to extract variables from environment file "
                  f"'{full_path}'. Ensure it has the proper format "
                  f"(i.e. A=B).")
  
      if not environment:
          log.warn(f"Empty environment file '{env_file}'.")
  
          return {}
  
      return environment
  
  
  def environment_files_to_dict(env_files: Iterable[str], root: str = ".") -> EnvironmentType:
      """
      Read environment variables from the given list of files.
  
      Files should be in YAML format (``.yaml`` or ``.yml`` suffixes), or in dotenv format.
  
      .. code-block:: bash
         :caption: dotenv file example
  
         A=B
         C=D
  
      .. code-block:: yaml
         :caption: YAML file example
  
         A: B
         C: D
  
      Path to each file should be relative to the metadata tree root.
  
      .. note::
  
         For loading environment variables from a single file, see
         :py:func:`environment_file_to_dict`, which is a function
         ``environment_files_to_dict()`` calls for each file,
         accumulating data from all input files.
      """
  
      result: EnvironmentType = {}
  
      for env_file in env_files:
          result.update(environment_file_to_dict(env_file, root=root))
  
      return result
  
  
  @contextlib.contextmanager
  def modify_environ(
          new_elements: EnvironmentType) -> Generator[None, None, None]:
      """ A context manager for os.environ that restores the initial state """
      environ_backup = os.environ.copy()
      os.environ.clear()
      os.environ.update(new_elements)
      try:
          yield
      finally:
          os.environ.clear()
          os.environ.update(environ_backup)
  
  
  def context_to_dict(context: List[str]) -> FmfContextType:
      """
      Convert command line context definition into a dictionary
  
      Does the same as environment_to_dict() plus separates possible
      comma-separated values into lists. Here's a couple of examples:
  
      distro=fedora-33 ---> {'distro': ['fedora']}
      arch=x86_64,ppc64 ---> {'arch': ['x86_64', 'ppc64']}
      """
      return {
          key: value.split(',')
          for key, value in environment_to_dict(context).items()}
  
  
  def dict_to_yaml(
          data: Union[Dict[str, Any], List[Any]],
          width: Optional[int] = None,
          sort: bool = False,
          start: bool = False) -> str:
      """ Convert dictionary into yaml """
      output = io.StringIO()
      yaml = YAML()
      yaml.indent(mapping=4, sequence=4, offset=2)
      yaml.default_flow_style = False
      yaml.allow_unicode = True
      yaml.encoding = 'utf-8'
      yaml.width = width
      yaml.explicit_start = start
      # Convert multiline strings
      scalarstring.walk_tree(data)
      if sort:
          # Sort the data https://stackoverflow.com/a/40227545
          sorted_data = CommentedMap()
          for key in sorted(data):
              sorted_data[key] = data[key]
          data = sorted_data
      yaml.dump(data, output)
      return output.getvalue()
  
  
  YamlTypType = Literal['rt', 'safe', 'unsafe', 'base']
  
  
  def yaml_to_dict(data: Any,
                   yaml_type: Optional[YamlTypType] = None) -> Dict[Any, Any]:
      """ Convert yaml into dictionary """
      yaml = YAML(typ=yaml_type)
      loaded_data = yaml.load(data)
      if loaded_data is None:
          return dict()
      if not isinstance(loaded_data, dict):
          raise GeneralError(
              f"Expected dictionary in yaml data, "
              f"got '{type(loaded_data).__name__}'.")
      return loaded_data
  
  
  def key_to_option(key: str) -> str:
      """ Convert a key name to corresponding option name """
  
      return key.replace('_', '-')
  
  
  def option_to_key(option: str) -> str:
      """ Convert an option name to corresponding key name """
  
      return option.replace('-', '_')
  
  
  SerializableContainerDerivedType = TypeVar(
      'SerializableContainerDerivedType',
      bound='SerializableContainer')
  
  
  @dataclasses.dataclass
  class SerializableContainer:
      """
      A mixin class for objects that may be saved in files and restored later
      """
  
      def to_dict(self) -> Dict[str, Any]:
          """ Return keys and values in the form of a dictionary """
  
          return dataclasses.asdict(self)
  
      # This method should remain a class-method: 1. list of keys is known
      # already, therefore it's not necessary to create an instance, and
      # 2. some functionality makes use of this knowledge.
      @classmethod
      def keys(cls) -> Generator[str, None, None]:
          """ Iterate over key names """
  
          for field in dataclasses.fields(cls):
              yield field.name
  
      def values(self) -> Generator[Any, None, None]:
          """ Iterate over key values """
  
          yield from self.to_dict().values()
  
      def items(self) -> Generator[Tuple[str, Any], None, None]:
          """ Iterate over key/value pairs """
  
          yield from self.to_dict().items()
  
      @classmethod
      def default(cls, key: str, default: Any = None) -> Any:
          """
          Return a default value for a given key.
  
          Keys may have a default value, or a default *factory* has been specified.
  
          :param key: key to look for.
          :param default: when key has no default value, ``default`` is returned.
          :returns: a default value defined for the key, or its ``default_factory``'s
              return value of ``default_factory``, or ``default`` when key has no
              default value.
          """
  
          for field in dataclasses.fields(cls):
              if key != field.name:
                  continue
  
              if not isinstance(field.default_factory, dataclasses._MISSING_TYPE):
                  return field.default_factory()
  
              if not isinstance(field.default, dataclasses._MISSING_TYPE):
                  return field.default
  
          else:
              return default
  
      @property
      def is_bare(self) -> bool:
          """
          Check whether all keys are either unset or have their default value.
  
          :returns: ``True`` if all keys either hold their default value
              or are not set at all, ``False`` otherwise.
          """
  
          for field in dataclasses.fields(self):
              value = getattr(self, field.name)
  
              if not isinstance(field.default_factory, dataclasses._MISSING_TYPE):
                  if value != field.default_factory():
                      return False
  
              elif not isinstance(field.default, dataclasses._MISSING_TYPE):
                  if value != field.default:
                      return False
  
              else:
                  pass
  
          return True
  
      #
      # Moving data between containers and objects owning them
      #
  
      def inject_to(self, obj: Any) -> None:
          """
          Inject keys from this container into attributes of a given object
          """
  
          for name, value in self.items():
              setattr(obj, name, value)
  
      @classmethod
      def extract_from(cls: Type[SerializableContainerDerivedType],
                       obj: Any) -> SerializableContainerDerivedType:
          """ Extract keys from given object, and save them in a container """
  
          data = cls()
  
          for key in cls.keys():
              value = getattr(obj, key)
              if value is not None:
                  setattr(data, key, value)
  
          return data
  
      #
      # Serialization - writing containers into YAML files, and restoring
      # them later.
      #
  
      def to_serialized(self) -> Dict[str, Any]:
          """
          Return keys and values in the form allowing later reconstruction.
  
          Used to transform container into a structure one can save in a
          YAML file, and restore it later.
  
          See :py:meth:`from_serialized` for its counterpart.
          """
  
          fields = self.to_dict()
  
          # Add a special field tracking what class we just shattered to pieces.
          fields.update({
              '__class__': {
                  'module': self.__class__.__module__,
                  'name': self.__class__.__name__
                  }
              })
  
          return fields
  
      @classmethod
      def from_serialized(
              cls: Type[SerializableContainerDerivedType],
              serialized: Dict[str, Any]) -> SerializableContainerDerivedType:
          """
          Recreate container from its serialized form.
  
          Used to transform data read from a YAML file into the original
          container.
  
          See :py:meth:`to_serialized` for its counterpart.
          """
  
          # Our special key may or may not be present, depending on who
          # calls this method.  In any case, it is not needed, because we
          # already know what class to restore: this one.
          serialized.pop('__class__', None)
  
          return cls(**serialized)
  
      @staticmethod
      def unserialize(serialized: Dict[str, Any]
                      ) -> SerializableContainerDerivedType:
          """
          Recreate container from its serialized form.
  
          Similar to :py:meth:`from_serialized`, but this method knows
          nothing about container's class, and will locate the correct
          module and class by inspecting serialized data. Discovered
          class' :py:meth:`from_serialized` is then used to create the
          container.
  
          Used to transform data read from a YAML file into original
          containers when their classes are not know to the code.
          Restoring such containers requires inspection of serialized data
          and dynamic imports of modules as needed.
          """
  
          from tmt.plugins import import_
  
          # Unpack class info, to get nicer variable names
          if "__class__" not in serialized:
              raise GeneralError(
                  "Failed to load saved state, probably because of old data format.\n"
                  "Use 'tmt clean runs' to clean up old runs.")
  
          klass_info = serialized.pop('__class__')
          klass_module_name = klass_info['module']
          klass_name = klass_info['name']
  
          # Make sure the module is imported. It probably is, but really,
          # make sure of it.
          import_(klass_module_name)
  
          # Now the module should be available in `sys.modules` like any
          # other, and we can go and grab the class we need from it.
          klass_module = sys.modules[klass_module_name]
          klass = getattr(klass_module, klass_name)
  
          if klass is None:
              raise SystemExit(
                  f"Failed to import '{klass_name}' "
                  f"from '{klass_module_name}' module.")
  
          # Stay away from classes that are not derived from this one, to
          # honor promise given by return value annotation.
          assert issubclass(klass, SerializableContainer)
  
          # Apparently, the issubclass() check above is not good enough for mypy.
          return cast(
              SerializableContainerDerivedType,
              klass.from_serialized(serialized))
  
  
  def markdown_to_html(filename: str) -> str:
      """
      Convert markdown to html
  
      Expects: Markdown document as a file.
      Returns: An HTML document as a string.
      """
      try:
          import markdown
      except ImportError:
          raise ConvertError("Install tmt-test-convert to export tests.")
  
      try:
          with open(filename, 'r') as file:
              try:
                  text = file.read()
              except UnicodeError:
                  raise MetadataError(f"Unable to read '{filename}'.")
              return markdown.markdown(text)
      except IOError:
          raise ConvertError(f"Unable to open '{filename}'.")
  
  
  def shell_variables(
          data: Union[List[str], Tuple[str, ...], Dict[str, Any]]) -> List[str]:
      """
      Prepare variables to be consumed by shell
  
      Convert dictionary or list/tuple of key=value pairs to list of
      key=value pairs where value is quoted with shlex.quote().
      """
  
      # Convert from list/tuple
      if isinstance(data, list) or isinstance(data, tuple):
          converted_data = []
          for item in data:
              splitted_item = item.split('=')
              key = splitted_item[0]
              value = shlex.quote('='.join(splitted_item[1:]))
              converted_data.append(f'{key}={value}')
          return converted_data
  
      # Convert from dictionary
      return [f"{key}={shlex.quote(str(value))}" for key, value in data.items()]
  
  
  def duration_to_seconds(duration: str) -> int:
      """ Convert sleep time format into seconds """
      units = {
          's': 1,
          'm': 60,
          'h': 60 * 60,
          'd': 60 * 60 * 24,
          }
      try:
          match = re.match(r'^(\d+)([smhd]?)$', str(duration))
          if match is None:
              raise SpecificationError(f"Invalid duration '{duration}'.")
          number, suffix = match.groups()
          return int(number) * units.get(suffix, 1)
      except (ValueError, AttributeError):
          raise SpecificationError(f"Invalid duration '{duration}'.")
  
  
  def verdict(
          decision: Optional[bool],
          comment: Optional[str] = None,
          good: str = 'pass',
          bad: str = 'fail',
          problem: str = 'warn',
          **kwargs: Any) -> Optional[bool]:
      """
      Print verdict in green, red or yellow based on the decision
  
      The supported decision values are:
  
          True .... good (green)
          False ... bad (red)
          None .... problem (yellow)
  
      Anything else raises an exception. Additional arguments
      are passed to the `echo` function. Returns back the decision.
      """
  
      if decision is False:
          text = style(bad, fg='red')
      elif decision is True:
          text = style(good, fg='green')
      elif decision is None:
          text = style(problem, fg='yellow')
      else:
          raise GeneralError(
              "Invalid decision value, must be 'True', 'False' or 'None'.")
      if comment:
          text = text + ' ' + comment
      echo(text, **kwargs)
      return decision
  
  
  def format(
          key: str,
          value: Union[None, str, List[Any], Dict[Any, Any]] = None,
          indent: int = 12,
          width: int = 72,
          wrap: Literal[True, False, 'auto'] = 'auto',
          key_color: str = 'green',
          value_color: str = 'black') -> str:
      """
      Nicely format and indent a key-value pair
  
      The following values for 'wrap' are supported:
  
          True .... always reformat text and wrap long lines
          False ... preserve text, no new line changes
          auto .... wrap only if text contains a long line
      """
      indent_string = (indent + 1) * ' '
      # Key
      output = '{} '.format(str(key).rjust(indent, ' '))
      if key_color is not None:
          output = style(output, fg=key_color)
      # Bool
      if isinstance(value, bool):
          output += ('true' if value else 'false')
      # List
      elif isinstance(value, list):
          # Make sure everything is string, prepare list, check for spaces
          value = [str(item) for item in value]
          listed_text = fmf.utils.listed(value)
          has_spaces = any([item.find(' ') > -1 for item in value])
          # Use listed output only for short lists without spaces
          if len(listed_text) < width - indent and not has_spaces:
              output += listed_text
          # Otherwise just place each item on a new line
          else:
              output += ('\n' + indent_string).join(value)
      # Dictionary
      elif isinstance(value, dict):
          # Place each key value pair on a separate line
          output += ('\n' + indent_string).join(
              f'{item[0]}: {item[1]}' for item in value.items())
      # Text
      elif isinstance(value, str):
          # In 'auto' mode enable wrapping when long lines present
          if wrap == 'auto':
              wrap = any(
                  [len(line) + indent - 7 > width
                   for line in value.split('\n')])
          if wrap:
              output += (wrap_text(
                  value, width=width,
                  preserve_paragraphs=True,
                  initial_indent=indent_string,
                  subsequent_indent=indent_string).lstrip())
          else:
              output += (('\n' + indent_string).join(
                  value.rstrip().split('\n')))
      else:
          output += str(value)
      return output
  
  
  def create_directory(
          path: str,
          name: str,
          dry: bool = False,
          quiet: bool = False) -> None:
      """ Create a new directory, handle errors """
      say = log.debug if quiet else echo
      if os.path.isdir(path):
          say("Directory '{}' already exists.".format(path))
          return
      if dry:
          say("Directory '{}' would be created.".format(path))
          return
      try:
          os.makedirs(path, exist_ok=True)
          say("Directory '{}' created.".format(path))
      except OSError as error:
          raise FileError("Failed to create {} '{}' ({})".format(
              name, path, error))
  
  
  def create_file(
          path: str,
          content: str,
          name: str,
          dry: bool = False,
          force: bool = False,
          mode: int = 0o664,
          quiet: bool = False) -> None:
      """ Create a new file, handle errors """
      say = log.debug if quiet else echo
      action = 'would be created' if dry else 'created'
      if os.path.exists(path):
          if force:
              action = 'would be overwritten' if dry else 'overwritten'
          else:
              raise FileError("File '{}' already exists.".format(path))
  
      if dry:
          say("{} '{}' {}.".format(name.capitalize(), path, action))
          return
  
      try:
          with open(path, 'w') as file_:
              file_.write(content)
          say("{} '{}' {}.".format(name.capitalize(), path, action))
          os.chmod(path, mode)
      except OSError as error:
          raise FileError("Failed to create {} '{}' ({})".format(
              name, path, error))
  
  
  # Avoid multiple subprocess calls for the same url
  @lru_cache(maxsize=None)
  def check_git_url(url: str) -> str:
      """ Check that a remote git url is accessible """
      try:
          log.debug(f"Check git url '{url}'.")
          subprocess.check_call(
              ["git", "ls-remote", "--heads", url],
              stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
              env={"GIT_ASKPASS": "echo", "GIT_TERMINAL_PROMPT": "0"})
          return url
      except subprocess.CalledProcessError:
          raise GitUrlError(f"Unable to contact remote git via '{url}'.")
  
  
  def public_git_url(url: str) -> str:
      """
      Convert a git url into a public format
  
      Return url in the format which can be accessed without
      authentication. For now just cover the most common services.
      """
  
      # GitHub, GitLab
      # old: git@github.com:teemtee/tmt.git
      # new: https://github.com/teemtee/tmt.git
      matched = re.match('git@(.*):(.*)', url)
      if matched:
          host, project = matched.groups()
          return f'https://{host}/{project}'
  
      # RHEL packages
      # old: git+ssh://psplicha@pkgs.devel.redhat.com/tests/bash
      # old: ssh://psplicha@pkgs.devel.redhat.com/tests/bash
      # old: ssh://pkgs.devel.redhat.com/tests/bash
      # new: git://pkgs.devel.redhat.com/tests/bash
      matched = re.match(
          r'(git\+)?ssh://(\w+@)?(pkgs\.devel\.redhat\.com)/(.*)', url)
      if matched:
          _, _, host, project = matched.groups()
          return f'git://{host}/{project}'
  
      # Fedora packages, Pagure
      # old: git+ssh://psss@pkgs.fedoraproject.org/tests/shell
      # old: ssh://psss@pkgs.fedoraproject.org/tests/shell
      # new: https://pkgs.fedoraproject.org/tests/shell
      matched = re.match(r'(git\+)?ssh://(\w+@)?([^/]*)/(.*)', url)
      if matched:
          _, _, host, project = matched.groups()
          return f'https://{host}/{project}'
  
      # Otherwise return unmodified
      return url
  
  
  class TimeoutHTTPAdapter(requests.adapters.HTTPAdapter):
      """
      Spice up request's session with custom timeout.
      """
  
      def __init__(self, *args: Any, **kwargs: Any) -> None:
          self.timeout = kwargs.pop('timeout', None)
  
          super().__init__(*args, **kwargs)
  
      def send(  # type: ignore[override] # does not match superclass type on purpose
              self,
              request: requests.PreparedRequest,
              **kwargs: Any) -> requests.Response:
          kwargs.setdefault('timeout', self.timeout)
  
          return super().send(request, **kwargs)
  
  
  class RetryStrategy(requests.packages.urllib3.util.retry.Retry):  # type: ignore[misc]
      def increment(
              self,
              *args: Any,
              **kwargs: Any
              ) -> requests.packages.urllib3.util.retry.Retry:
          error = cast(Optional[Exception], kwargs.get('error', None))
  
          # Detect a subset of exception we do not want to follow with a retry.
          if error is not None:
              # Failed certificate verification - this issue will probably not get any better
              # should we try again.
              if isinstance(error, urllib3.exceptions.SSLError) \
                      and 'certificate verify failed' in str(error):
  
                  # [mpr] I'm not sure how stable this *iternal* API is, but pool seems to be the
                  # only place aware of the remote hostname. Try our best to get the hostname for
                  # a better error message, but don't crash because of a missing attribute or
                  # something as dumb.
  
                  connection_pool = kwargs.get('_pool', None)
  
                  if connection_pool is not None and hasattr(connection_pool, 'host'):
                      message = f"Certificate verify failed for '{connection_pool.host}'."
                  else:
                      message = 'Certificate verify failed.'
  
                  raise GeneralError(message, original=error) from error
  
          return super().increment(*args, **kwargs)
  
  
  class retry_session(contextlib.AbstractContextManager):  # type: ignore[type-arg]
      """
      Context manager for requests.Session() with retries and timeout
      """
      @staticmethod
      def create(
              retries: int = DEFAULT_RETRY_SESSION_RETRIES,
              backoff_factor: float = DEFAULT_RETRY_SESSION_BACKOFF_FACTOR,
              allowed_methods: Optional[Tuple[str, ...]] = None,
              status_forcelist: Optional[Tuple[int, ...]] = None,
              timeout: Optional[int] = None
              ) -> requests.Session:
          retry_strategy = RetryStrategy(
              total=retries,
              status_forcelist=status_forcelist,
              # `method_whitelist`` has been renamed to `allowed_methods` since
              # urllib3 1.26, and it will be removed in urllib3 2.0.
              # `allowed_methods` is therefore the future-proof name, but for the
              # sake of backward compatibility, internally we need to use the
              # deprecated parameter for now. Or request newer urllib3, but that
              # might a problem because of RPM availability.
              method_whitelist=allowed_methods,
              backoff_factor=backoff_factor)
  
          if timeout is not None:
              http_adapter: requests.adapters.HTTPAdapter = TimeoutHTTPAdapter(
                  timeout=timeout, max_retries=retry_strategy)
          else:
              http_adapter = requests.adapters.HTTPAdapter(
                  max_retries=retry_strategy)
  
          session = requests.Session()
          session.mount('http://', http_adapter)
          session.mount('https://', http_adapter)
  
          return session
  
      def __init__(
              self,
              retries: int = DEFAULT_RETRY_SESSION_RETRIES,
              backoff_factor: float = DEFAULT_RETRY_SESSION_BACKOFF_FACTOR,
              allowed_methods: Optional[Tuple[str, ...]] = None,
              status_forcelist: Optional[Tuple[int, ...]] = None,
              timeout: Optional[int] = None
              ) -> None:
          self.retries = retries
          self.backoff_factor = backoff_factor
          self.allowed_methods = allowed_methods
          self.status_forcelist = status_forcelist
          self.timeout = timeout
  
      def __enter__(self) -> requests.Session:
          return self.create(
              retries=self.retries,
              backoff_factor=self.backoff_factor,
              allowed_methods=self.allowed_methods,
              status_forcelist=self.status_forcelist,
              timeout=self.timeout)
  
      def __exit__(self, *args: Any) -> None:
          pass
  
  
  def remove_color(text: str) -> str:
      """ Remove ansi color sequences from the string """
      return re.sub(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])', '', text)
  
  
  def default_branch(repository: str, remote: str = 'origin') -> str:
      """ Detect default branch from given local git repository """
      head = os.path.join(repository, f'.git/refs/remotes/{remote}/HEAD')
      # Make sure the HEAD reference is available
      if not os.path.exists(head):
          subprocess.run(
              f'git remote set-head {remote} --auto'.split(), cwd=repository)
      # The ref format is 'ref: refs/remotes/origin/main'
      with open(head) as ref:
          return ref.read().strip().split('/')[-1]
  
  
  def parse_dotenv(content: str) -> EnvironmentType:
      """ Parse dotenv (shell) format of variables """
      return dict([line.split("=", maxsplit=1)
                  for line in shlex.split(content, comments=True)])
  
  
  def parse_yaml(content: str) -> EnvironmentType:
      """ Parse variables from yaml, ensure flat dictionary format """
      yaml_as_dict = YAML(typ="safe").load(content)
      # Handle empty file as an empty environment
      if yaml_as_dict is None:
          return dict()
      if any(isinstance(val, dict) for val in yaml_as_dict.values()):
          raise GeneralError(
              "Can't set the environment from the nested yaml config. The "
              "config should be just key, value pairs.")
      return {key: str(value) for key, value in yaml_as_dict.items()}
  
  
  def validate_git_status(test: 'tmt.base.Test') -> Tuple[bool, str]:
      """
      Validate that test has current metadata on fmf_id
  
      Return a tuple (boolean, message) as the result of validation.
  
      Checks that sources:
      - all local changes are committed
      - up to date on remote repository
      - .fmf/version marking fmf root is committed as well
  
      When all checks pass returns (True, '').
      """
      sources = test.node.sources + \
          [os.path.join(test.node.root, '.fmf', 'version')]
  
      # Use tmt's run instead of subprocess.run
      run = Common().run
  
      # Check for not committed metadata changes
      cmd = ['git', 'status', '--porcelain', '--'] + sources
      try:
          result = run(cmd, cwd=test.node.root, join=True)
      except RunError as error:
          return (
              False,
              f"Failed to run git status: {error.stdout}"
              )
  
      not_committed = []
      assert result.stdout is not None
      for line in result.stdout.split('\n'):
          if line:
              # XY PATH or XY ORIG -> PATH. XY and PATH are separated by space
              not_committed.append(line[3:])
  
      if not_committed:
          return (False, "Uncommitted changes in " + " ".join(not_committed))
  
      # Check for not pushed changes
      cmd = ["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"]
      try:
          result = run(cmd, cwd=test.node.root)
      except RunError as error:
          return (
              False,
              f'Failed to get remote branch, error raised: "{error.stderr}"'
              )
  
      assert result.stdout is not None
      remote_ref = result.stdout.strip()
  
      cmd = [
          'git',
          'diff',
          f'HEAD..{remote_ref}',
          '--name-status',
          '--'] + sources
      try:
          result = run(cmd, cwd=test.node.root)
      except RunError as error:
          return (
              False,
              f'Failed to diff against remote branch, error raised: "{error.stderr}"')
  
      not_pushed = []
      assert result.stdout is not None
      for line in result.stdout.split('\n'):
          if line:
              _, path = line.strip().split('\t', maxsplit=2)
              not_pushed.append(path)
      if not_pushed:
          return (False, "Not pushed changes in " + " ".join(not_pushed))
  
      return (True, '')
  
  
  def generate_runs(
          path: str, id_: Optional[str] = None) -> Generator[str, None, None]:
      """ Generate absolute paths to runs from path """
      # Prepare absolute workdir path if --id was used
      if id_:
          if '/' not in id_:
              id_ = os.path.join(path, id_)
          if os.path.isabs(id_):
              if os.path.exists(id_):
                  yield id_
              return
      if not os.path.exists(path):
          return
      for filename in os.listdir(path):
          abs_path = os.path.join(path, filename)
          # If id_ is None, the abs_path is considered valid (no filtering
          # is being applied). If it is defined, it has been transformed
          # to absolute path and must be equal to abs_path for the run
          # in abs_path to be generated.
          invalid_id = id_ and abs_path != id_
          invalid_run = not os.path.exists(
              os.path.join(abs_path, 'run.yaml'))
          if not os.path.isdir(abs_path) or invalid_id or invalid_run:
              continue
          yield abs_path
  
  
  def load_run(run: 'tmt.base.Run') -> Tuple[bool, Optional[Exception]]:
      """ Load a run and its steps from the workdir """
      try:
          run.load_from_workdir()
      except GeneralError as error:
          return False, error
      for plan in run.plans:
          for step in plan.steps(disabled=True):
              step.load()
      return True, None
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  StructuredField
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  SFSectionValueType = Union[str, List[str]]
  
  
  class StructuredField:
      """
      Handling multiple text data in a single text field
  
      The StructuredField allows you to easily store and extract several
      sections of text data to/from a single text field. The sections are
      separated by section names in square brackets and can be hosted in
      other text as well.
  
      The section names have to be provided on a separate line and there
      must be no leading/trailing white space before/after the brackets.
      The StructuredField supports two versions of the format:
  
      Version 0: Simple, concise, useful when neither the surrounding text
      or the section data can contain lines which could resemble section
      names. Here's an example of a simple StructuredField::
  
          Note written by human.
  
          [section-one]
          Section one content.
  
          [section-two]
          Section two content.
  
          [section-three]
          Section three content.
  
          [end]
  
          Another note written by human.
  
      Version 1: Includes unique header to prevent collisions with the
      surrounding text and escapes any section-like lines in the content::
  
          Note written by human.
  
          [structured-field-start]
          This is StructuredField version 1. Please, edit with care.
  
          [section-one]
          Section one content.
  
          [section-two]
          Section two content.
          [structured-field-escape][something-resembling-section-name]
  
          [section-three]
          Section three content.
  
          [structured-field-end]
  
          Another note written by human.
  
      Note that an additional empty line is added at the end of each
      section to improve the readability. This line is not considered
      to be part of the section content.
  
      Besides handling the whole section content it's also possible to
      store several key-value pairs in a single section, similarly as in
      the ini config format::
  
          [section]
          key1 = value1
          key2 = value2
          key3 = value3
  
      Provide the key name as the optional argument 'item' when accessing
      these single-line items. Note that the section cannot contain both
      plain text data and key-value pairs.
  
      Example::
  
          field = qe.StructuredField()
          field.set("project", "Project Name")
          field.set("details", "somebody", "owner")
          field.set("details", "2013-05-27", "started")
          field.set("description", "This is a description.\\n"
                  "It spans across multiple lines.\\n")
          print field.save()
  
              [structured-field-start]
              This is StructuredField version 1. Please, edit with care.
  
              [project]
              Project Name
  
              [details]
              owner = somebody
              started = 2013-05-27
  
              [description]
              This is a description.
              It spans across multiple lines.
  
              [structured-field-end]
  
          field.version(0)
          print field.save()
  
              [project]
              Project Name
  
              [details]
              owner = somebody
              started = 2013-05-27
  
              [description]
              This is a description.
              It spans across multiple lines.
  
              [end]
  
      Multiple values for the same key are supported as well. Enable this
      feature with 'multi=True' when initializing the structured field.
      If multiple values are present their list will be returned instead
      of a single string. Similarly use list for setting multiple values::
  
          field = qe.StructuredField(multi=True)
          requirements = ['hypervisor=', 'labcontroller=lab.example.com']
          field.set("hardware", requirements, "hostrequire")
          print field.save()
  
              [structured-field-start]
              This is StructuredField version 1. Please, edit with care.
  
              [hardware]
              hostrequire = hypervisor=
              hostrequire = labcontroller=lab.example.com
  
              [structured-field-end]
  
          print field.get("hardware", "hostrequire")
  
              ['hypervisor=', 'labcontroller=lab.example.com']
      """
  
      # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
      #  StructuredField Special
      # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
      def __init__(
              self,
              text: Optional[str] = None,
              version: int = 1,
              multi: bool = False) -> None:
          """ Initialize the structured field """
          self.version(version)
          self._header: str = ""
          self._footer: str = ""
          # Sections are internally stored in their serialized form, i.e. as
          # strings.
          self._sections: Dict[str, str] = {}
          self._order: List[str] = []
          self._multi = multi
          if text is not None:
              self.load(text)
  
      def __iter__(self) -> Generator[str, None, None]:
          """ By default iterate through all available section names """
          for section in self._order:
              yield section
  
      def __nonzero__(self) -> bool:
          """ True when any section is defined """
          return len(self._order) > 0
  
      __bool__ = __nonzero__
  
      # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
      #  StructuredField Private
      # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
      def _load_version_zero(self, text: str) -> None:
          """ Load version 0 format """
          # Attempt to split the text according to the section tag
          section = re.compile(r"\n?^\[([^\]]+)\]\n", re.MULTILINE)
          parts = section.split(text)
          # If just one part ---> no sections present, just plain text
          if len(parts) == 1:
              self._header = parts[0]
              return
          # Pick header & footer, make sure [end] tag is present
          self._header = parts[0]
          self._footer = re.sub("^\n", "", parts[-1])
          if parts[-2] != "end":
              raise StructuredFieldError("No [end] section tag found")
          # Convert to dictionary and save the order
          keys = parts[1:-2:2]
          values = parts[2:-2:2]
          for key, value in zip(keys, values):
              self.set(key, value)
  
      def _load(self, text: str) -> None:
          """ Load version 1+ format """
          # The text must exactly match the format
          format = re.compile(
              r"(.*)^\[structured-field-start\][ \t]*\n"
              r"(.*)\n\[structured-field-end\][ \t]*\n(.*)",
              re.DOTALL + re.MULTILINE)
          # No match ---> plain text or broken structured field
          matched = format.search(text)
          if not matched:
              if "[structured-field" in text:
                  raise StructuredFieldError("StructuredField parse error")
              self._header = text
              log.debug("StructuredField not found, treating as a plain text")
              return
          # Save header & footer (remove trailing new lines)
          self._header = re.sub("\n\n$", "\n", matched.groups()[0])
          if self._header:
              log.debug(u"Parsed header:\n{0}".format(self._header))
          self._footer = re.sub("^\n", "", matched.groups()[2])
          if self._footer:
              log.debug(u"Parsed footer:\n{0}".format(self._footer))
          # Split the content on the section names
          section = re.compile(r"\n\[([^\]]+)\][ \t]*\n", re.MULTILINE)
          parts = section.split(matched.groups()[1])
          # Detect the version
          version_match = re.search(r"version (\d+)", parts[0])
          if not version_match:
              log.error(parts[0])
              raise StructuredFieldError(
                  "Unable to detect StructuredField version")
          self.version(int(version_match.groups()[0]))
          log.debug(
              "Detected StructuredField version {0}".format(
                  self.version()))
          # Convert to dictionary, remove escapes and save the order
          keys = parts[1::2]
          escape = re.compile(r"^\[structured-field-escape\]", re.MULTILINE)
          values = [escape.sub("", value) for value in parts[2::2]]
          for key, value in zip(keys, values):
              self.set(key, value)
          log.debug(u"Parsed sections:\n{0}".format(
              pprint.pformat(self._sections)))
  
      def _save_version_zero(self) -> str:
          """ Save version 0 format """
          result = []
          if self._header:
              result.append(self._header)
          for section, content in self.iterate():
              result.append(u"[{0}]\n{1}".format(section, content))
          if self:
              result.append(u"[end]\n")
          if self._footer:
              result.append(self._footer)
          return "\n".join(result)
  
      def _save(self) -> str:
          """ Save version 1+ format """
          result = []
          # Regular expression for escaping section-like lines
          escape = re.compile(r"^(\[.+\])$", re.MULTILINE)
          # Header
          if self._header:
              result.append(self._header)
          # Sections
          if self:
              result.append(
                  u"[structured-field-start]\n"
                  u"This is StructuredField version {0}. "
                  u"Please, edit with care.\n".format(self._version))
              for section, content in self.iterate():
                  result.append(u"[{0}]\n{1}".format(section, escape.sub(
                      "[structured-field-escape]\\1", content)))
              result.append(u"[structured-field-end]\n")
          # Footer
          if self._footer:
              result.append(self._footer)
          return "\n".join(result)
  
      def _read_section(self, content: str) -> Dict[str, SFSectionValueType]:
          """ Parse config section and return ordered dictionary """
          dictionary: Dict[str, SFSectionValueType] = OrderedDict()
          for line in content.split("\n"):
              # Remove comments and skip empty lines
              line = re.sub("#.*", "", line)
              if re.match(r"^\s*$", line):
                  continue
              # Parse key and value
              matched = re.search("([^=]+)=(.*)", line)
              if not matched:
                  raise StructuredFieldError(
                      "Invalid key/value line: {0}".format(line))
              key = matched.groups()[0].strip()
              value = matched.groups()[1].strip()
              # Handle multiple values if enabled
              if key in dictionary and self._multi:
                  stored_value = dictionary[key]
                  if isinstance(stored_value, list):
                      stored_value.append(value)
                  else:
                      dictionary[key] = [stored_value, value]
              else:
                  dictionary[key] = value
          return dictionary
  
      def _write_section(self, dictionary: Dict[str, SFSectionValueType]) -> str:
          """ Convert dictionary into a config section format """
          section = ""
          for key in dictionary:
              if isinstance(dictionary[key], list):
                  for value in dictionary[key]:
                      section += "{0} = {1}\n".format(key, value)
              else:
                  section += "{0} = {1}\n".format(key, dictionary[key])
          return section
  
      # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
      #  StructuredField Methods
      # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
      def iterate(self) -> Generator[Tuple[str, str], None, None]:
          """ Return (section, content) tuples for all sections """
          for section in self:
              yield section, self._sections[section]
  
      def version(self, version: Optional[int] = None) -> int:
          """ Get or set the StructuredField version """
          if version is not None:
              if version in [0, 1]:
                  self._version = version
              else:
                  raise StructuredFieldError(
                      "Bad StructuredField version: {0}".format(version))
          return self._version
  
      def load(self, text: str, version: Optional[int] = None) -> None:
          """ Load the StructuredField from a string """
          if version is not None:
              self.version(version)
          # Make sure we got a text, convert from bytes if necessary
          if isinstance(text, bytes):
              text = text.decode("utf8")
          if not isinstance(text, str):
              raise StructuredFieldError(
                  "Invalid StructuredField, expecting string")
          # Remove possible carriage returns
          text = re.sub("\r\n", "\n", text)
          # Make sure the text has a new line at the end
          if text and text[-1] != "\n":
              text += "\n"
          log.debug(u"Parsing StructuredField\n{0}".format(text))
          # Parse respective format version
          if self._version == 0:
              self._load_version_zero(text)
          else:
              self._load(text)
  
      def save(self) -> str:
          """ Convert the StructuredField into a string """
          if self.version() == 0:
              return self._save_version_zero()
          else:
              return self._save()
  
      def header(self, content: Optional[str] = None) -> str:
          """ Get or set the header content """
          if content is not None:
              self._header = content
          return self._header
  
      def footer(self, content: Optional[str] = None) -> str:
          """ Get or set the footer content """
          if content is not None:
              self._footer = content
          return self._footer
  
      def sections(self) -> List[str]:
          """ Get the list of available sections """
          return self._order
  
      def get(
              self,
              section: str,
              item: Optional[str] = None) -> SFSectionValueType:
          """ Return content of given section or section item """
          try:
              content = self._sections[section]
          except KeyError:
              raise StructuredFieldError(
                  "Section [{0!r}] not found".format(ascii(section)))
          # Return the whole section content
          if item is None:
              return content
          # Return only selected item from the section
          try:
              return self._read_section(content)[item]
          except KeyError:
              raise StructuredFieldError(
                  "Unable to read '{0!r}' from section '{1!r}'".format(
                      ascii(item), ascii(section)))
  
      def set(self, section: str, content: Any,
              item: Optional[str] = None) -> None:
          """ Update content of given section or section item """
          # Convert to string if necessary, keep lists untouched
          if isinstance(content, list):
              pass
          elif isinstance(content, bytes):
              content = content.decode("utf8")
          elif not isinstance(content, str):
              content = str(content)
          # Set the whole section content
          if item is None:
              # Add new line if missing
              if content and content[-1] != "\n":
                  content += "\n"
              self._sections[section] = content
          # Set only selected item from the section
          else:
              try:
                  current = self._sections[section]
              except KeyError:
                  current = ""
              dictionary = self._read_section(current)
              dictionary[item] = content
              self._sections[section] = self._write_section(dictionary)
          # Remember the order when adding a new section
          if section not in self._order:
              self._order.append(section)
  
      def remove(self, section: str, item: Optional[str] = None) -> None:
          """ Remove given section or section item """
          # Remove the whole section
          if item is None:
              try:
                  del self._sections[section]
                  del self._order[self._order.index(section)]
              except KeyError:
                  raise StructuredFieldError(
                      "Section [{0!r}] not found".format(ascii(section)))
          # Remove only selected item from the section
          else:
              try:
                  dictionary = self._read_section(self._sections[section])
                  del(dictionary[item])
              except KeyError:
                  raise StructuredFieldError(
                      "Unable to remove '{0!r}' from section '{1!r}'".format(
                          ascii(item), ascii(section)))
              self._sections[section] = self._write_section(dictionary)
  
  
  class DistGitHandler:
      """ Common functionality for DistGit handlers """
      sources_file_name = 'sources'
      uri = "/rpms/{name}/{filename}/{hashtype}/{hash}/{filename}"
  
      usage_name: str  # Name to use for dist-git-type
      re_source: Pattern[str]
      re_ignore_extensions: Pattern[str] = re.compile(r'\.(sign|asc|key)$')
      lookaside_server: str
      remote_substring: Pattern[str]
  
      def url_and_name(self, cwd: str = '.') -> List[Tuple[str, str]]:
          """
          Return list of urls and basenames of the used source
  
          The 'cwd' parameter has to be a DistGit directory.
          """
          # Assumes <package>.spec
          globbed = glob.glob(os.path.join(cwd, '*.spec'))
          if len(globbed) != 1:
              raise GeneralError(f"No .spec file is present in '{cwd}'.")
          package = os.path.basename(globbed[0])[:-len('.spec')]
          ret_values = []
          try:
              with open(os.path.join(cwd, self.sources_file_name)) as f:
                  for line in f.readlines():
                      match = self.re_source.match(line)
                      if match is None:
                          raise GeneralError(
                              f"Couldn't match '{self.sources_file_name}' "
                              f"content with '{self.re_source.pattern}'.")
                      used_hash, source_name, hash_value = match.groups()
                      ret_values.append((self.lookaside_server + self.uri.format(
                          name=package,
                          filename=source_name,
                          hash=hash_value,
                          hashtype=used_hash.lower()
                          ), source_name))
          except Exception as error:
              raise GeneralError(
                  f"Couldn't read '{self.sources_file_name}' file.",
                  original=error)
          if not ret_values:
              raise GeneralError(
                  "No sources found in '{self.sources_file_name}' file.")
          return ret_values
  
      def its_me(self, remotes: List[str]) -> bool:
          """ True if self can work with remotes """
          return any([self.remote_substring.search(item) for item in remotes])
  
  
  class FedoraDistGit(DistGitHandler):
      """ Fedora Handler """
      usage_name = "fedora"
      re_source = re.compile(r"^(\w+) \(([^)]+)\) = ([0-9a-fA-F]+)$")
      lookaside_server = "https://src.fedoraproject.org/repo/pkgs"
      remote_substring = re.compile(r'fedoraproject\.org')
  
  
  class CentOSDistGit(DistGitHandler):
      """ CentOS Handler """
      usage_name = "centos"
      re_source = re.compile(r"^(\w+) \(([^)]+)\) = ([0-9a-fA-F]+)$")
      lookaside_server = "https://sources.stream.centos.org/sources"
      remote_substring = re.compile(r'redhat/centos')
  
  
  def get_distgit_handler(
          remotes: Optional[List[str]] = None,
          usage_name: Optional[str] = None) -> DistGitHandler:
      """
      Return the right DistGitHandler
  
      Pick the DistGitHandler class which understands specified
      remotes or by usage_name.
      """
      for candidate_class in DistGitHandler.__subclasses__():
          if usage_name is not None and usage_name == candidate_class.usage_name:
              return candidate_class()
          if remotes is not None:
              ret_val = candidate_class()
              if ret_val.its_me(remotes):
                  return ret_val
      raise GeneralError(f"No known remote in '{remotes}'.")
  
  
  def get_distgit_handler_names() -> List[str]:
      """ All known distgit handlers """
      return [i.usage_name for i in DistGitHandler.__subclasses__()]
  
  
  def git_clone(
          url: str,
          destination: str,
          common: Common,
          env: Optional[EnvironmentType] = None,
          shallow: bool = False
          ) -> CommandOutput:
      """
      Git clone url to destination, retry without shallow if necessary
  
      For shallow=True attempt to clone repository using --depth=1 option first.
      If not successful attempt to clone whole repo.
  
      Common instance is used to run the command for appropriate logging.
      Environment is updated by 'env' dictionary.
      """
      depth = ['--depth=1'] if shallow else []
      command = ['git', 'clone'] + depth + [url, destination]
      try:
          return common.run(command, env=env)
      except RunError:
          if not shallow:
              # Do not retry if shallow was not used
              raise
          # Git server might not support shallow cloning, try again
          return git_clone(url, destination, common, env, shallow=False)
  
  
  class updatable_message(contextlib.AbstractContextManager):  # type: ignore[type-arg]
      """ Updatable message suitable for progress-bar-like reporting """
  
      def __init__(
              self,
              key: str,
              enabled: bool = True,
              indent_level: int = 0,
              key_color: Optional[str] = None,
              default_value_color: Optional[str] = None
              ) -> None:
          """
          Updatable message suitable for progress-bar-like reporting.
  
          .. code:block:: python3
  
             with updatable_message('foo') as message:
                 while ...:
                     ...
  
                     # check state of remote request, and update message
                     state = remote_api.check()
                     message.update(state)
  
          :param key: a string to use as the left-hand part of logged message.
          :param enabled: if unset, no output would be performed.
          :param indent_level: desired indentation level.
          :param key_color: optional color to apply to ``key``.
          :param default_color: optional color to apply to value when
              :py:meth:`update` is called with ``color`` left out.
          """
  
          self.key = key
          self.enabled = enabled
          self.indent_level = indent_level
          self.key_color = key_color
          self.default_value_color = default_value_color
  
          # No progress if terminal not attached
          if not sys.stdout.isatty():
              self.enabled = False
  
          self._previous_line: Optional[str] = None
  
      def __enter__(self) -> 'updatable_message':
          return self
  
      def __exit__(self, *args: Any) -> None:
          sys.stdout.write('\n')
          sys.stdout.flush()
  
      def update(self, value: str, color: Optional[str] = None) -> None:
          if not self.enabled:
              return
  
          if self._previous_line is not None:
              message = value.ljust(len(self._previous_line))
  
          else:
              message = value
  
          self._previous_line = value
  
          message = indent(
              self.key,
              value=style(
                  message,
                  fg=color or self.default_value_color),
              color=self.key_color,
              level=self.indent_level)
  
          sys.stdout.write(f"\r{message}")
          sys.stdout.flush()
  
  
  def find_fmf_root(path: str) -> List[str]:
      """
      Search trough path and return all fmf roots that exist there
  
      Returned list is ordered by path length, shortest one first.
  
      Raise `MetadataError` if no fmf root is found.
      """
      fmf_roots = []
      for root, _, files in os.walk(path):
          if not os.path.basename(root) == '.fmf':
              continue
          if 'version' in files:
              fmf_roots.append(os.path.dirname(root))
      if not fmf_roots:
          raise MetadataError(f"No fmf root present inside '{path}'.")
      fmf_roots.sort(key=lambda path: len(path))
      return fmf_roots
  
  
  #
  # JSON schema-based validation helpers
  #
  # Aims at FMF data consumed by tmt, but can be used for any structure.
  #
  
  # `Schema` represents a loaded JSON schema structure. It may be fairly complex,
  # but it's not needed to provide the exhaustive and fully detailed type since
  # tmt code is not actually "reading" it. Loaded schema is passed down to
  # jsonschema library, and while `Any` would be perfectly valid, let's use an
  # alias to make schema easier to track in our code.
  Schema = Dict[str, Any]
  SchemaStore = Dict[str, Schema]
  
  
  def _patch_plan_schema(schema: Schema, store: SchemaStore) -> None:
      """
      Resolve references to per-plugin schema known to steps. All schemas have
      been loaded into store, all that's left is to update each step in plan
      schema with the list of schemas allowed for that particular step.
  
      For each step, we create the following schema (see also plan.yaml for the
      rest of plan schema):
  
      .. code-block:: yaml
  
         <step name>:
           oneOf:
             - $ref: "/schemas/<step name>/plugin1"
             - $ref: "/schemas/<step name>/plugin2"
             ...
             - $ref: "/schemas/<step name>/pluginN"
             - type: array
               items:
                 anyOf:
                   - $ref: "/schemas/<step name>/plugin1"
                   - $ref: "/schemas/<step name>/plugin2"
                   ...
                   - $ref: "/schemas/<step name>/pluginN"
      """
  
      for step in ('discover', 'execute', 'finish', 'prepare', 'provision', 'report'):
          step_schema_prefix = f'/schemas/{step}/'
  
          step_plugin_schema_ids = [schema_id for schema_id in store.keys() if schema_id.startswith(
              step_schema_prefix) and schema_id != '/schemas/provision/hardware']
  
          refs: List[Schema] = [
              {'$ref': schema_id} for schema_id in step_plugin_schema_ids
              ]
  
          schema['properties'][step] = {
              'oneOf': refs + [
                  {
                      'type': 'array',
                      'items': {
                          'anyOf': refs
                          }
                      }
                  ]
              }
  
  
  def _load_schema(schema_filepath: str) -> Schema:
      """
      Load a JSON schema from a given filepath.
  
      A helper returning the raw loaded schema.
      """
  
      if not os.path.isabs(schema_filepath):
          schema_filepath = os.path.join(
              pkg_resources.resource_filename(
                  'tmt', 'schemas'), schema_filepath)
  
      try:
          with open(schema_filepath, 'r', encoding='utf-8') as f:
              return cast(Schema, yaml_to_dict(f.read()))
  
      except Exception as exc:
          raise FileError(f"Failed to load schema file {schema_filepath}\n{exc}")
  
  
  @functools.lru_cache(maxsize=None)
  def load_schema(schema_filepath: str) -> Schema:
      """
      Load a JSON schema from a given filepath.
  
      Recommended for general use, the method may apply some post-loading touches
      to the given schema, and unless caller is interested in the raw content of
      the file, this functions should be used instead of the real workhorse of
      schema loading, :py:func:`_load_schema`.
      """
  
      schema = _load_schema(schema_filepath)
  
      if schema.get('$id') == '/schemas/plan':
          _patch_plan_schema(schema, load_schema_store())
  
      return schema
  
  
  @functools.lru_cache(maxsize=None)
  def load_schema_store() -> SchemaStore:
      """
      Load all available JSON schemas, and put them into a "store".
  
      Schema store is a simple mapping between schema IDs and schemas.
      """
  
      store: SchemaStore = {}
  
      schema_dirpath = pkg_resources.resource_filename('tmt', 'schemas')
  
      try:
          for dirpath, dirnames, filenames in os.walk(
                  schema_dirpath, followlinks=True):
              for filename in filenames:
                  # Ignore all files but YAML files.
                  if os.path.splitext(filename)[1].lower() not in ('.yaml', '.yml'):
                      continue
  
                  schema = _load_schema(os.path.join(dirpath, filename))
  
                  store[schema['$id']] = schema
  
      except Exception as exc:
          raise FileError(f"Failed to discover schema files\n{exc}")
  
      if '/schemas/plan' not in store:
          raise FileError('Failed to discover schema for plans')
  
      _patch_plan_schema(store['/schemas/plan'], store)
  
      return store
  
  
  def _prenormalize_fmf_node(node: fmf.Tree, schema_name: str) -> fmf.Tree:
      """
      Apply the minimal possible normalization steps to nodes before validating them with schemas.
  
      tmt allows some fields to have default values, and at least ``how`` field is necessary for
      schema-based validation to work reliably. Based on ``how`` field, plan schema identifies
      the correct *plugin* schema for step validation. Without ``how``, it's hard to pick the
      correct schema.
  
      This function tries to do minimal number of changes to a given fmf node to honor the promise
      of ``how`` being optional, with known defaults for each step. It might be possible to resolve
      this purely with schemas, but since we don't know how (yet?), a Python implementation has been
      chosen to unblock schema-based validation while keeping things easier for users. This may
      change in the future, dropping the need for this pre-validation step.
  
      .. note::
  
         This function is not part of the normalization process that happens after validation. The
         purpose of this function is to make the world nice and shiny for tmt users while avoiding
         the possibility of schema becoming way too complicated, especially when we would need
         non-trivial amount fo time for experiments.
  
         The real normalization process takes place after validation, and is responsible for
         converting raw fmf data to data types and structures more suited for tmt internal
         implementation.
      """
  
      # As of now, only `how` field in plan steps seems to be required for schema-based validation
      # to work correctly, therefore ignore any other node.
      if schema_name != 'plan.yaml':
          return node
  
      # Perform the very crude and careful semi-validation. We need to set the `how` key to a default
      # value - but it's not our job to validate the general structure of node data. Walk the "happy"
      # path, touch the node only when it matches the specification of being a mapping of steps and
      # these being either mappings or lists of mappings. Whenever we notice some value does not
      # match this basic structure, ignore the step completely - its issues will be cought by schema
      # later, don't waste time on steps that do not follow specification.
  
      # Fmf data describing a plan shall be a mapping (with keys like `discover` or `adjust`).
      if not isinstance(node.data, dict):
          return node
  
      # Avoid possible circular imports
      import tmt.steps
  
      def _process_step(step_name: str, step: Dict[Any, Any]) -> None:
          """
          Process a single step configuration.
          """
  
          # If `how` is set, don't touch it, and there's nothing to do.
          if 'how' in step:
              return
  
          # Magic!
          # No, seriously: step is implemented in `tmt.steps.$step_name` package,
          # by a class `tmt.steps.$step_name.$step_name_with_capitalized_first_letter`.
          # Instead of having a set of if-elif tests, we can reach the default `how`
          # dynamically.
  
          from tmt.plugins import import_
  
          step_module_name = f'tmt.steps.{step_name}'
          step_class_name = step_name.capitalize()
  
          # Make sure the step module is imported. It probably is, but really,
          # make sure of it.
          import_(step_module_name)
  
          # Now the module should be available in `sys.modules` like any
          # other, and we can go and grab the class we need from it.
          step_module = sys.modules[step_module_name]
          step_class = getattr(step_module, step_class_name)
  
          if step_class is None:
              raise GeneralError(
                  f'Step {step_name} implementation cannot be found '
                  f'in {step_module_name}.{step_class_name}')
  
          step['how'] = cast(tmt.steps.Step, step_class).DEFAULT_HOW
  
      def _process_step_collection(step_name: str, step_collection: Any) -> None:
          """
          Process a collection of step configurations.
          """
  
          # Ignore anything that is not a step.
          if step_name not in tmt.steps.STEPS:
              return
  
          # A single step configuration, represented as a mapping.
          if isinstance(step_collection, dict):
              _process_step(step_name, step_collection)
  
              return
  
          # Multiple step configurations, as mappings in a list
          if isinstance(step_collection, list):
              for step_config in step_collection:
                  # Unexpected, maybe instead of a mapping describing a step someone put
                  # in an integer... Ignore, schema will report it.
                  if not isinstance(step_config, dict):
                      continue
  
                  _process_step(step_name, step_config)
  
      for step_name, step_config in node.data.items():
          _process_step_collection(step_name, step_config)
  
      return node
  
  
  def validate_fmf_node(
          node: fmf.Tree, schema_name: str) -> List[Tuple[jsonschema.ValidationError, str]]:
      """ Validate a given fmf node """
  
      node = _prenormalize_fmf_node(node, schema_name)
  
      result = node.validate(load_schema(schema_name), schema_store=load_schema_store())
  
      if result.result is True:
          return []
  
      # A bit of error formatting. It is possible to use str(error), but the result
      # is a bit too JSON-ish. Let's present an error message in a way that helps
      # users to point finger on each and every issue. But don't throw the original
      # errors away!
  
      errors: List[Tuple[jsonschema.ValidationError, str]] = []
  
      for error in result.errors:
          path = f'{node.name}:{".".join(error.path)}'
  
          errors.append((error, f'{path} - {error.message}'))
  
      return errors
  
  
  # A type for callbacks given to wait()
  WaitCheckType = Callable[[], T]
  
  
  def wait(
      parent: Common,
      check: WaitCheckType[T],
      timeout: datetime.timedelta,
      tick: float = DEFAULT_WAIT_TICK,
      tick_increase: float = DEFAULT_WAIT_TICK_INCREASE
  ) -> T:
      """
      Wait for a condition to become true.
  
      To test the condition state, a ``check`` callback is called every ``tick``
      seconds until ``check`` reports a success. The callback may:
  
      * decide the condition has been fulfilled. This is a successfull outcome,
        ``check`` shall then simply return, and waiting ends. Or,
      * decide more time is needed. This is not a successfull outcome, ``check``
        shall then raise :py:clas:`WaitingIncomplete` exception, and ``wait()``
        will try again later.
  
      :param parent: "owner" of the wait process. Used for its logging capability.
      :param check: a callable responsible for testing the condition. Accepts no
          arguments. To indicate more time and attempts are needed, the callable
          shall raise :py:class:`WaitingIncomplete`, otherwise it shall return
          without exception. Its return value will be propagated by ``wait()`` up
          to ``wait()``'s. All other exceptions raised by ``check`` will propagate
          to ``wait()``'s caller as well, terminating the wait.
      :param timeout: amount of time ``wait()`` is alowed to spend waiting for
          successfull outcome of ``check`` call.
      :param tick: how many seconds to wait between two consecutive calls of
          ``check``.
      :param tick_increase: a multiplier applied to ``tick`` after every attempt.
      :returns: value returned by ``check`` reporting success.
      :raises GeneralError: when ``tick`` is not a positive integer.
      :raises WaitingTimedOutError: when time quota has been consumed.
      """
  
      if tick <= 0:
          raise GeneralError('Tick must be a positive integer')
  
      NOW = time.monotonic
  
      deadline = NOW() + timeout.total_seconds()
  
      parent.debug(
          'wait',
          f"waiting for condition '{check.__name__}' with timeout {timeout},"
          f"deadline in {timeout.total_seconds()}, checking every {tick} seconds")
  
      while True:
          now = NOW()
  
          if now > deadline:
              raise WaitingTimedOutError()
  
          try:
              ret = check()
  
              # Perform one extra check: if `check()` succeeded, but took more time than
              # allowed, it should be recognized as a failed waiting too.
              now = NOW()
  
              if now > deadline:
                  parent.debug(
                      'wait',
                      f"'{check.__name__}' finished successfully but took too much time,"
                      f"{now - deadline} over quota")
  
                  raise WaitingTimedOutError()
  
              parent.debug(
                  'wait',
                  f"'{check.__name__}' finished successfully, {deadline - now} seconds left")
  
              return ret
  
          except WaitingIncomplete:
              parent.debug(
                  'wait',
                  f"'{check.__name__}' still pending, {deadline - now} seconds left")
  
              time.sleep(tick)
  
              tick *= tick_increase
  
              continue
  
  
  class ValidateFmfMixin:
      """
      Mixin adding validation of an fmf node.
  
      Loads a schema whose name is derived from class name, and uses fmf's validate()
      method to perform the validation.
      """
  
      def _validate_fmf_node(self, node: fmf.Tree, logger: Common,
                             raise_on_validation_error: bool) -> None:
          """ Validate a given fmf node """
  
          errors = validate_fmf_node(
              node, f'{self.__class__.__name__.lower()}.yaml')
  
          if errors:
              if raise_on_validation_error:
                  raise SpecificationError(
                      f'fmf node {node.name} failed validation',
                      validation_errors=errors)
  
              for _, error_message in errors:
                  logger.warn(error_message, shift=1)
  
      def __init__(
              self,
              *,
              node: fmf.Tree,
              logger: Common,
              skip_validation: bool = False,
              raise_on_validation_error: bool = False,
              **kwargs: Any) -> None:
          # Validate *before* letting next class in line touch the data.
          if not skip_validation:
              self._validate_fmf_node(node, logger, raise_on_validation_error)
  
          kwargs.setdefault('logger', self)
          super().__init__(node=node, **kwargs)  # type: ignore[call-arg]
  
  
  # A type representing compatible sources of keys and values.
  KeySource = Union[Dict[str, Any], fmf.Tree]
  
  
  class NormalizeKeysMixin:
      """
      Mixin adding support for loading fmf keys into object attributes.
  
      When invoked, annotated class-level variables are searched for in a given source
      container - a mapping, an fmf node, etc. - and if the key of the same name as the
      variable exists, its value is "promoted" to instance variable.
  
      If a method named ``_normalize_<variable name>`` exists, it is called with the fmf
      key value as its single argument, and its return value is assigned to instance
      variable. This gives class chance to modify or transform the original value when
      needed, e.g. to convert the original value to a type more suitable for internal
      processing.
      """
  
      # If specified, keys would be iterated over in the order as listed here.
      KEYS_SHOW_ORDER: List[str] = []
  
      # NOTE: these could be static methods, self is probably useless, but that would
      # cause complications when classes assign these to their members. That makes them
      # no longer static as far as class is concerned, which means they get called with
      # `self` as the first argument. A workaround would be to assign staticmethod()-ized
      # version of them, but that's too much repetition.
      #
      # TODO: wouldn't it be nice if these could be mention in dataclass.field()?
      # It would require a clone of dataclass.field() though.
      def _normalize_string_list(self, value: Union[None, str, List[str]]) -> List[str]:
          if value is None:
              return []
  
          return [value] if isinstance(value, str) else value
  
      def _normalize_environment(self, value: Optional[Dict[str, Any]]) -> EnvironmentType:
          if value is None:
              return {}
  
          return {
              name: str(value) for name, value in value.items()
              }
  
      @classmethod
      def _iter_key_annotations(cls) -> Generator[Tuple[str, Any], None, None]:
          """
          Iterate over keys' type annotations.
  
          Keys are yielded in the order: keys declared by parent classes first, then
          keys declared by the class itself, all following the order in which keys
          were defined in their respective classes.
  
          Yields:
              pairs of key name and its annotations.
          """
  
          def _iter_class_annotations(klass: type) -> Generator[Tuple[str, Any], None, None]:
              # Skip, needs fixes to become compatible
              if klass is Common:
                  return
  
              for name, value in klass.__dict__.get('__annotations__', {}).items():
                  # Skip special fields that are not keys.
                  if name == 'KEYS_SHOW_ORDER':
                      continue
  
                  yield (name, value)
  
          # Reverse MRO to start with the most base classes first, to iterate over keys
          # in the order they are defined.
          for klass in reversed(cls.__mro__):
              yield from _iter_class_annotations(klass)
  
      @classmethod
      def keys(cls) -> Generator[str, None, None]:
          """
          Iterate over key names.
  
          Keys are yielded in the order: keys declared by parent classes first, then
          keys declared by the class itself, all following the order in which keys
          were defined in their respective classes.
  
          Yields:
              key names.
          """
  
          for keyname, _ in cls._iter_key_annotations():
              yield keyname
  
      def items(self) -> Generator[Tuple[str, Any], None, None]:
          """
          Iterate over keys and their values.
  
          Keys are yielded in the order: keys declared by parent classes first, then
          keys declared by the class itself, all following the order in which keys
          were defined in their respective classes.
  
          Yields:
              pairs of key name and its value.
          """
  
          for keyname in self.keys():
              yield (keyname, getattr(self, keyname))
  
      # TODO: exists for backward compatibility for the transition period. Once full
      # type annotations land, there should be no need for extra _keys attribute.
      @classmethod
      def _keys(cls) -> List[str]:
          """ Return a list of names of object's keys. """
  
          return list(cls.keys())
  
      def _load_keys(
              self,
              key_source: Dict[str, Any],
              key_source_name: str,
              logger: Common) -> None:
          """ Extract values for class-level attributes, and verify they match declared types. """
  
          LOG_SHIFT, LOG_LEVEL = 2, 4
  
          debug_intro = functools.partial(logger.debug, shift=LOG_SHIFT - 1, level=LOG_LEVEL)
          debug = functools.partial(logger.debug, shift=LOG_SHIFT, level=LOG_LEVEL)
  
          debug_intro('key source')
          for k, v in key_source.items():
              debug(f'{k}: {v} ({type(v)})')
  
          debug('')
  
          for keyname, keytype in self._iter_key_annotations():
              key_address = f'{key_source_name}:{keyname}'
              source_keyname = key_to_option(keyname)
  
              # Do not indent this particular entry like the rest, so it could serve
              # as a "header" for a single key processing.
              debug_intro('key', key_address)
              debug('field', source_keyname)
  
              debug('desired type', str(keytype))
  
              value: Any = None
  
              # Verbose, let's hide it a bit deeper.
              debug('dict', self.__dict__, level=LOG_LEVEL + 1)
  
              if hasattr(self, keyname):
                  # If the key exists as instance's attribute already, it is because it's been
                  # declared with a default value, and the attribute now holds said default value.
                  default_value = getattr(self, keyname)
  
                  # If the default value is a mutable container, we cannot use it directly.
                  # Should we do so, the very same default value would be assigned to multiple
                  # instances/attributes instead of each instance having its own distinct container.
                  if isinstance(default_value, (list, dict)):
                      debug('detected mutable default')
                      default_value = copy.copy(default_value)
  
                  debug('default value', str(default_value))
                  debug('default value type', str(type(default_value)))
  
                  # try+except seems to work better than get(), especially when
                  # semantic of fmf.Tree.get() is slightly different than that
                  # of dict().get().
                  try:
                      value = key_source[source_keyname]
  
                  except KeyError:
                      value = default_value
  
                  debug('raw value', str(value))
                  debug('raw value type', str(type(value)))
  
              else:
                  value = key_source.get(source_keyname)
  
                  debug('raw value', str(value))
                  debug('raw value type', str(type(value)))
  
              normalize_callback = getattr(self, f'_normalize_{keyname}', None)
  
              if normalize_callback:
                  value = normalize_callback(value)
  
                  debug('normalized value', str(value))
                  debug('normalized value type', str(type(value)))
  
              debug('final value', str(value))
              debug('final value type', str(type(value)))
  
              # Set attribute by adding it to __dict__ directly. Messing with setattr()
              # might cause re-use of mutable values by other instances.
              self.__dict__[keyname] = value
  
              # Apparently pointless, but makes the debugging output more readable.
              # There may be plenty of tests and plans and keys, a bit of spacing
              # can't hurt.
              debug('')
  
          debug_intro('normalized fields')
          for k, v in self.__dict__.items():
              debug(f'{k}: {v} ({type(v)})')
  
          debug('')
  
      def __init__(self, **kwargs: Any) -> None:
          super().__init__(**kwargs)
  
  
  class LoadFmfKeysMixin(NormalizeKeysMixin):
      def __init__(
              self,
              *,
              node: fmf.Tree,
              logger: Common,
              **kwargs: Any) -> None:
          self._load_keys(node.get(), node.name, logger)
  
          kwargs.setdefault('logger', logger)
          super().__init__(node=node, **kwargs)
  
files: tmt/utils.py
files: 
  """ Test Metadata Utilities """
  
  import contextlib
  import copy
  import dataclasses
  import datetime
  import functools
  import glob
  import io
  import os
  import pprint
  import re
  import shlex
  import shutil
  import subprocess
  import sys
  import time
  import unicodedata
  from collections import OrderedDict
  from functools import lru_cache
  from pathlib import Path
  from threading import Thread
  from typing import (IO, TYPE_CHECKING, Any, Callable, Dict, Generator,
                      Iterable, List, NamedTuple, Optional, Pattern, Tuple, Type,
                      TypeVar, Union, cast, overload)
  
  import click
  import fmf
  import jsonschema
  import pkg_resources
  import requests
  import requests.adapters
  import requests.packages.urllib3.util.retry
  import urllib3.exceptions
  from click import echo, style, wrap_text
  from ruamel.yaml import YAML, scalarstring
  from ruamel.yaml.comments import CommentedMap
  
  if sys.version_info >= (3, 8):
      from typing import Literal, Protocol
  else:
      from typing_extensions import Literal, Protocol
  
  
  if TYPE_CHECKING:
      import tmt.base
      import tmt.steps
  
  log = fmf.utils.Logging('tmt').logger
  
  # Default workdir root and max
  WORKDIR_ROOT = '/var/tmp/tmt'
  WORKDIR_MAX = 1000
  
  # Log in workdir
  LOG_FILENAME = 'log.txt'
  
  # Maximum number of lines of stdout/stderr to show upon errors
  OUTPUT_LINES = 100
  # Default output width
  OUTPUT_WIDTH = 79
  
  # Hierarchy indent
  INDENT = 4
  
  # Default name and order for step plugins
  DEFAULT_NAME = 'default'
  DEFAULT_PLUGIN_ORDER = 50
  DEFAULT_PLUGIN_ORDER_MULTIHOST = 10
  DEFAULT_PLUGIN_ORDER_REQUIRES = 70
  DEFAULT_PLUGIN_ORDER_RECOMMENDS = 75
  
  # Config directory
  CONFIG_PATH = '~/.config/tmt'
  
  # Special process return code
  PROCESS_TIMEOUT = 124
  
  # Default select.select(timeout) in seconds
  DEFAULT_SELECT_TIMEOUT = 5
  
  # Default shell and options to be set for all shell scripts
  DEFAULT_SHELL = "/bin/bash"
  SHELL_OPTIONS = 'set -eo pipefail'
  
  # Defaults for HTTP/HTTPS retries and timeouts (see `retry_session()`).
  DEFAULT_RETRY_SESSION_RETRIES: int = 3
  DEFAULT_RETRY_SESSION_BACKOFF_FACTOR: float = 0.1
  
  # Defaults for HTTP/HTTPS retries for getting environment file
  # Retry with exponential backoff, maximum duration ~511 seconds
  ENVFILE_RETRY_SESSION_RETRIES: int = 10
  ENVFILE_RETRY_SESSION_BACKOFF_FACTOR: float = 1
  
  # Default for wait()-related options
  DEFAULT_WAIT_TICK: float = 30.0
  DEFAULT_WAIT_TICK_INCREASE: float = 1.0
  
  # A stand-in variable for generic use.
  T = TypeVar('T')
  
  # A FMF context type, representing name/values context.
  FmfContextType = Dict[str, List[str]]
  
  # A "environment" type, representing name/value environment variables.
  EnvironmentType = Dict[str, str]
  
  # Workdir argument type, can be True, a string, a path or None
  WorkdirArgumentType = Union[Literal[True], str, None]
  
  # Workdir type, can be None or a string
  WorkdirType = Optional[str]
  
  # Option to skip to initialize work tree in plan
  PLAN_SKIP_WORKTREE_INIT = 'plan_skip_worktree_init'
  
  
  class BaseLoggerFnType(Protocol):
      def __call__(
              self,
              key: str,
              value: Optional[str] = None,
              color: Optional[str] = None,
              shift: int = 0,
              level: int = 1,
              err: bool = False) -> None:
          pass
  
  
  class LevelessLoggerFnType(Protocol):
      def __call__(
              self,
              key: str,
              value: Optional[str] = None,
              color: Optional[str] = None,
              shift: int = 0,
              err: bool = False) -> None:
          pass
  
  
  class SemanticLoggerFnType(Protocol):
      def __call__(self, message: str, shift: int = 0) -> None:
          pass
  
  
  LoggerFnType = Union[
      BaseLoggerFnType,
      LevelessLoggerFnType,
      SemanticLoggerFnType]
  
  
  def indent(
          key: str,
          value: Optional[str] = None,
          color: Optional[str] = None,
          level: int = 0) -> str:
      """
      Indent a key/value message.
  
      If both ``key`` and ``value`` are specified, ``{key}: {value}``
      message is rendered. Otherwise, just ``key`` is used alone. If
      ``value`` contains multiple lines, each but the very first line is
      indented by one extra level.
  
      :param value: optional value to print at right side of ``key``.
      :param color: optional color to apply on ``key``.
      :param level: number of indentation levels. Each level is indented
                    by :py:data:`INDENT` spaces.
      """
  
      indent = ' ' * INDENT * level
      deeper = ' ' * INDENT * (level + 1)
  
      # Colorize
      if color is not None:
          key = style(key, fg=color)
  
      # Handle key only
      if value is None:
          message = key
  
      # Handle key + value
      else:
          # Multiline content indented deeper
          if isinstance(value, str):
              lines = value.splitlines()
              if len(lines) > 1:
                  value = ''.join([f"\n{deeper}{line}" for line in lines])
  
          message = f'{key}: {value}'
  
      return indent + message
  
  
  class Config:
      """ User configuration """
  
      def __init__(self) -> None:
          """ Initialize config directory path """
          self.path = os.path.expanduser(CONFIG_PATH)
          if not os.path.exists(self.path):
              try:
                  os.makedirs(self.path)
              except OSError as error:
                  raise GeneralError(
                      f"Failed to create config '{self.path}'.\n{error}")
  
      def last_run(self, run_id: Optional[str] = None) -> Optional[str]:
          """ Get and set last run id """
          symlink = os.path.join(self.path, 'last-run')
          if run_id:
              try:
                  os.remove(symlink)
              except OSError:
                  pass
              try:
                  os.symlink(run_id, symlink)
              except FileExistsError:
                  # Race when tmt runs in parallel
                  log.warning(f"Race condition, unable to save last run '{run_id}'.")
              except OSError as error:
                  raise GeneralError(
                      f"Unable to save last run '{self.path}'.\n{error}")
              return run_id
          if os.path.islink(symlink):
              return os.path.realpath(symlink)
          return None
  
  
  class StreamLogger(Thread):
      """
      Reading pipes of running process in threads.
  
      Code based on:
      https://github.com/packit/packit/blob/main/packit/utils/logging.py#L10
      """
  
      def __init__(self,
                   stream: Optional[IO[bytes]],
                   log_header: str,
                   logger: BaseLoggerFnType) -> None:
          super().__init__(daemon=True)
          self.stream = stream
          self.output: List[str] = []
          self.log_header = log_header
          self.logger = logger
  
      def run(self) -> None:
          if self.stream is None:
              return
  
          for _line in self.stream:
              line = _line.decode('utf-8', errors='replace')
              if line != '':
                  self.logger(
                      self.log_header,
                      line.rstrip('\n'),
                      'yellow',
                      level=3)
              self.output.append(line)
  
      def get_output(self) -> str:
          return "".join(self.output)
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Common
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  CommonDerivedType = TypeVar('CommonDerivedType', bound='Common')
  
  
  class CommandOutput(NamedTuple):
      stdout: Optional[str]
      stderr: Optional[str]
  
  
  class Common:
      """
      Common shared stuff
  
      Takes care of command line context, options and workdir handling.
      Provides logging functions info(), verbose() and debug().
      Implements read() and write() for comfortable file access.
      Provides the run() method for easy command execution.
      """
  
      # Command line context, options and workdir
      _context: Optional[click.Context] = None
      # When set to true, _opt will be ignored (default will be returned)
      ignore_class_options: bool = False
      _options: Dict[str, Any] = dict()
      _workdir: WorkdirType = None
  
      # TODO: must be declared outside of __init__(), because it must exist before
      # __init__() gets called to allow logging helpers work correctly when used
      # from mixins. But that's not very clean, is it? :( Maybe decoupling logging
      # from Common class would help, such a class would be able to initialize
      # itself without involving the rest of Common code. On the other hand,
      # Common owns workdir, for example, whose value affects logging too, so no
      # clear solution so far.
      #
      # Note: cannot use CommonDerivedType - it's a TypeVar filled in by the type
      # given to __init__() and therefore the type it's representing *now* is
      # unknown. but we know `parent` will be derived from `Common` class, so it's
      # mostly fine.
      parent: Optional['Common'] = None
  
      def __init__(
              self,
              parent: Optional[CommonDerivedType] = None,
              name: Optional[str] = None,
              workdir: WorkdirArgumentType = None,
              context: Optional[click.Context] = None,
              relative_indent: int = 1,
              **kwargs: Any) -> None:
          """
          Initialize name and relation with the parent object
  
          Prepare the workdir for provided id / directory path
          or generate a new workdir name if workdir=True given.
          Store command line context and options for future use
          if context is provided.
          """
          # Use lowercase class name as the default name
          self.name = name or self.__class__.__name__.lower()
          self.parent = parent
  
          # Relative log indent level shift against the parent
          self._relative_indent = relative_indent
  
          # Store command line context
          if context:
              self._save_context_to_instance(context)
  
          # Initialize the workdir if requested
          self._workdir_load(workdir)
  
      def __str__(self) -> str:
          """ Name is the default string representation """
          return self.name
  
      @classmethod
      def _save_context(cls, context: click.Context) -> None:
          """ Save provided command line context and options for future use """
          cls._context = context
          cls._options = context.params
  
      def _save_context_to_instance(self, context: click.Context) -> None:
          """ Save provided command line context and options to the instance """
          self._context = context
          self._options = context.params
  
      @overload
      @classmethod
      def _opt(cls, option: str) -> Any:
          pass
  
      @overload
      @classmethod
      def _opt(cls, option: str, default: T) -> T:
          pass
  
      @classmethod
      def _opt(cls, option: str, default: Any = None) -> Any:
          """ Get an option from the command line context (class version) """
          if cls.ignore_class_options:
              return default
          return cls._options.get(option, default)
  
      def _fmf_context(self) -> FmfContextType:
          """ Return the current fmf context """
          if self._context is None:
              return dict()
  
          try:
              return cast(FmfContextType, self._context.obj.fmf_context)
          except AttributeError:
              return dict()
  
      def opt(self, option: str, default: Optional[Any] = None) -> Any:
          """
          Get an option from the command line options
  
          Checks also parent options. For flags (boolean values) parent's
          True wins over child's False (e.g. run --quiet enables quiet
          mode for all included plans and steps).
  
          For options that can be used multiple times, the child overrides
          the parent if it was defined (e.g. run -av provision -vvv runs
          all steps except for provision in mildly verbose mode, provision
          is run with the most verbosity).
  
          Environment variables override command line options.
          """
          # Translate dashes to underscores to match click's conversion
          option = option.replace('-', '_')
          # Check the environment first
          if option == 'debug':
              try:
                  debug = os.environ['TMT_DEBUG']
                  return int(debug)
              except ValueError:
                  raise GeneralError(
                      f"Invalid debug level '{debug}', use an integer.")
              except KeyError:
                  pass
  
          # Get local option
          local = self._options.get(option, default)
          # Check parent option
          parent = None
          if self.parent:
              parent = self.parent.opt(option)
          # Special handling for special flags (parent's yes always wins)
          if option in ['quiet', 'force', 'dry']:
              return parent if parent else local
          # Special handling for counting options (child overrides the
          # parent if it was defined)
          elif option in ['debug', 'verbose']:
              winner = local if local else parent
              if winner is None:
                  winner = 0
              return winner
          else:
              return parent if parent is not None else local
  
      def _level(self) -> int:
          """ Hierarchy level """
          if self.parent is None:
              return -1
          else:
              return self.parent._level() + self._relative_indent
  
      def _indent(
              self,
              key: str,
              value: Optional[str] = None,
              color: Optional[str] = None,
              shift: int = 0) -> str:
          """ Indent message according to the object hierarchy """
  
          return indent(
              key,
              value=value,
              color=color,
              level=self._level() + shift)
  
      def _log(self, message: str) -> None:
          """ Append provided message to the current log """
          # Nothing to do if there is no workdir
          if self.workdir is None:
              return
  
          # Store log only in the top parent
          if self.parent:
              self.parent._log(message)
          else:
              with open(os.path.join(self.workdir, LOG_FILENAME), 'a') as log:
                  log.write(datetime.datetime.utcnow().strftime('%H:%M:%S') + ' '
                            + remove_color(message) + '\n')
  
      def print(
              self,
              key: str,
              value: Optional[str] = None,
              color: Optional[str] = None,
              shift: int = 0,
              err: bool = False) -> None:
          """ Print a message regardless the quiet mode """
          self._log(self._indent(key, value, color=None, shift=shift))
          echo(self._indent(key, value, color, shift), err=err)
  
      def info(
              self,
              key: str,
              value: Optional[str] = None,
              color: Optional[str] = None,
              shift: int = 0,
              err: bool = False) -> None:
          """ Show a message unless in quiet mode """
          self._log(self._indent(key, value, color=None, shift=shift))
          if not self.opt('quiet'):
              echo(self._indent(key, value, color, shift), err=err)
  
      def verbose(
              self,
              key: str,
              value: Optional[str] = None,
              color: Optional[str] = None,
              shift: int = 0,
              level: int = 1,
              err: bool = False) -> None:
          """
          Show message if in requested verbose mode level
  
          In quiet mode verbose messages are not displayed.
          """
          self._log(self._indent(key, value, color=None, shift=shift))
          if not self.opt('quiet') and self.opt('verbose') >= level:
              echo(self._indent(key, value, color, shift), err=err)
  
      def debug(
              self,
              key: str,
              value: Optional[str] = None,
              color: Optional[str] = None,
              shift: int = 0,
              level: int = 1,
              err: bool = False) -> None:
          """
          Show message if in requested debug mode level
  
          In quiet mode debug messages are not displayed.
          """
          self._log(self._indent(key, value, color=None, shift=shift))
          if not self.opt('quiet') and self.opt('debug') >= level:
              echo(self._indent(key, value, color, shift), err=err)
  
      def warn(self, message: str, shift: int = 0) -> None:
          """ Show a yellow warning message on info level, send to stderr """
          self.info('warn', message, color='yellow', shift=shift, err=True)
  
      def fail(self, message: str, shift: int = 0) -> None:
          """ Show a red failure message on info level, send to stderr """
          self.info('fail', message, color='red', shift=shift, err=True)
  
      def _run(self,
               command: Union[str, List[str]],
               cwd: Optional[str],
               shell: bool,
               env: Optional[EnvironmentType],
               log: Optional[BaseLoggerFnType],
               join: bool = False,
               interactive: bool = False,
               timeout: Optional[int] = None) -> CommandOutput:
          """
          Run command, capture the output
  
          By default stdout and stderr are captured separately.
          Use join=True to merge stderr into stdout.
          Use timeout=<seconds> to finish process after given time
          """
          # By default command ouput is logged using debug
          if not log:
              log = self.debug
          # Prepare the environment
          if env:
              if not isinstance(env, dict):
                  raise GeneralError(f"Invalid environment '{env}'.")
              # Do not modify current process environment
              environment = os.environ.copy()
              environment.update(env)
          else:
              environment = None
          self.debug('environment', pprint.pformat(environment), level=4)
  
          # Set only for shell=True as it would affect command
          executable = DEFAULT_SHELL if shell else None
  
          # Run the command in interactive mode if requested
          if interactive:
              try:
                  subprocess.run(
                      command, cwd=cwd, shell=shell, env=environment, check=True,
                      executable=executable)
              except subprocess.CalledProcessError:
                  # Interactive mode can return non-zero if the last command
                  # failed, ignore errors here
                  pass
              finally:
                  return CommandOutput(None, None)
  
          # Create the process
          try:
              process = subprocess.Popen(
                  command, cwd=cwd, shell=shell, env=environment,
                  stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
                  stderr=subprocess.STDOUT if join else subprocess.PIPE,
                  executable=executable)
          except FileNotFoundError as error:
              raise RunError(
                  f"File '{error.filename}' not found.", command, 127)
  
          stdout_thread = StreamLogger(
              process.stdout, log_header='out', logger=log)
          stderr_thread = stdout_thread
          if not join:
              stderr_thread = StreamLogger(
                  process.stderr, log_header='err', logger=log)
          stdout_thread.start()
          if not join:
              stderr_thread.start()
          try:
              process.wait(timeout=timeout)
          except subprocess.TimeoutExpired:
              process.kill()
              process.returncode = PROCESS_TIMEOUT
          stdout_thread.join()
          if not join:
              stderr_thread.join()
  
          # Handle the exit code, return output
          if process.returncode != 0:
              raise RunError(
                  message=f"Command returned '{process.returncode}'.",
                  command=command,
                  returncode=process.returncode,
                  stdout=stdout_thread.get_output(),
                  stderr=stderr_thread.get_output())
          if join:
              return CommandOutput(
                  stdout_thread.get_output(), None)
          else:
              return CommandOutput(
                  stdout_thread.get_output(), stderr_thread.get_output())
  
      def run(self,
              command: Union[str, List[str]],
              message: Optional[str] = None,
              cwd: Optional[str] = None,
              dry: bool = False,
              shell: bool = False,
              env: Optional[EnvironmentType] = None,
              interactive: bool = False,
              join: bool = False,
              log: Optional[BaseLoggerFnType] = None,
              timeout: Optional[int] = None) -> CommandOutput:
          """
          Run command, give message, handle errors
  
          Command is run in the workdir be default.
          In dry mode commands are not executed unless dry=True.
          Environment is updated with variables from the 'env' dictionary.
          Output is logged using self.debug() or custom 'log' function.
          Returns stdout if join=True, (stdout, stderr) tuple otherwise.
          """
  
          # A bit of logging - command, default message, error message for later...
          if isinstance(command, (list, tuple)):
              printable_command = ' '.join(shlex.quote(s) for s in command)
          else:
              printable_command = command
  
          if message:
              self.debug(message, level=2)
  
          self.debug(f'Run command: {printable_command}', level=2)
  
          # Nothing more to do in dry mode (unless requested)
          if self.opt('dry') and not dry:
              return CommandOutput(None, None)
  
          # Run the command, handle the exit code
          cwd = cwd or self.workdir
  
          # Fail nicely if the working directory does not exist
          if cwd and not os.path.exists(cwd):
              raise GeneralError(
                  f"The working directory '{cwd}' does not exist.")
  
          try:
              return self._run(
                  command, cwd, shell, env, log, join, interactive, timeout)
          except RunError as error:
              self.debug(error.message, level=3)
              message = f"Failed to run command: {printable_command} Reason: {error.message}"
              raise RunError(
                  message, error.command, error.returncode,
                  error.stdout, error.stderr)
  
      def read(self, path: str, level: int = 2) -> str:
          """ Read a file from the workdir """
          if self.workdir:
              path = os.path.join(self.workdir, path)
          self.debug(f"Read file '{path}'.", level=level)
          try:
              with open(path, encoding='utf-8', errors='replace') as data:
                  return data.read()
          except OSError as error:
              raise FileError(f"Failed to read '{path}'.\n{error}")
  
      def write(
              self,
              path: str,
              data: Any,
              mode: str = 'w',
              level: int = 2) -> None:
          """ Write a file to the workdir """
          if self.workdir:
              path = os.path.join(self.workdir, path)
          action = 'Append to' if mode == 'a' else 'Write'
          self.debug(f"{action} file '{path}'.", level=level)
          # Dry mode
          if self.opt('dry'):
              return
          try:
              with open(path, mode, encoding='utf-8', errors='replace') as file:
                  file.write(data)
          except OSError as error:
              raise FileError(f"Failed to write '{path}'.\n{error}")
  
      def _workdir_init(self, id_: WorkdirArgumentType = None) -> None:
          """
          Initialize the work directory
  
          Workdir under WORKDIR_ROOT is used/created if 'id' is provided.
          If 'id' is a path, that directory is used instead. Otherwise a
          new workdir is created under the WORKDIR_ROOT directory.
          """
          # Prepare the workdir name from given id or path
          if isinstance(id_, str):
              # Use provided directory if full path given
              if '/' in id_:
                  workdir = id_
              # Construct directory name under workdir root
              else:
                  workdir = os.path.join(WORKDIR_ROOT, id_)
          # Weird workdir id
          elif id_ is not None:
              raise GeneralError(
                  f"Invalid workdir '{id_}', expected a string or None.")
  
          def _check_or_create_workdir_root_with_perms() -> None:
              """ If created WORKDIR_ROOT has to be 1777 for multi-user"""
              if not os.path.isdir(WORKDIR_ROOT):
                  try:
                      os.makedirs(WORKDIR_ROOT, exist_ok=True)
                      os.chmod(WORKDIR_ROOT, 0o1777)
                  except OSError as error:
                      raise FileError(f"Failed to prepare workdir '{WORKDIR_ROOT}': {error}")
  
          if id_ is None:
              # Prepare WORKDIR_ROOT first
              _check_or_create_workdir_root_with_perms()
  
              # Generated unique id or fail, has to be atomic call
              for id_bit in range(1, WORKDIR_MAX + 1):
                  directory = 'run-{}'.format(str(id_bit).rjust(3, '0'))
                  workdir = os.path.join(WORKDIR_ROOT, directory)
                  try:
                      # Call is atomic, no race possible
                      os.makedirs(workdir)
                      break
                  except FileExistsError:
                      pass
              else:
                  raise GeneralError(
                      f"Workdir full. Cleanup the '{WORKDIR_ROOT}' directory.")
          else:
              # Cleanup possible old workdir if called with --scratch
              if self.opt('scratch'):
                  self._workdir_cleanup(workdir)
  
              if workdir.startswith(WORKDIR_ROOT):
                  _check_or_create_workdir_root_with_perms()
  
              # Create the workdir
              create_directory(workdir, 'workdir', quiet=True)
          self._workdir = workdir
  
      def _workdir_name(self) -> Optional[str]:
          """ Construct work directory name from parent workdir """
          # Need the parent workdir
          if self.parent is None or self.parent.workdir is None:
              return None
          # Join parent name with self
          return os.path.join(self.parent.workdir, self.name.lstrip('/'))
  
      def _workdir_load(self, workdir: WorkdirArgumentType) -> None:
          """
          Create the given workdir if it is not None
  
          If workdir=True, the directory name is automatically generated.
          """
          if workdir is True:
              self._workdir_init()
          elif workdir is not None:
              self._workdir_init(workdir)
  
      def _workdir_cleanup(self, path: Optional[str] = None) -> None:
          """ Clean up the work directory """
          directory = path or self._workdir_name()
          if directory is not None:
              if os.path.isdir(directory):
                  self.debug(f"Clean up workdir '{directory}'.", level=2)
                  shutil.rmtree(directory)
          self._workdir = None
  
      @property
      def workdir(self) -> Optional[str]:
          """ Get the workdir, create if does not exist """
          if self._workdir is None:
              self._workdir = self._workdir_name()
              # Workdir not enabled, even parent does not have one
              if self._workdir is None:
                  return None
              # Create a child workdir under the parent workdir
              create_directory(self._workdir, 'workdir', quiet=True)
          return self._workdir
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Exceptions
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  
  class GeneralError(Exception):
      """ General error """
  
      def __init__(self, *args: Any, **kwargs: Any) -> None:
          # Store the original exception for future use
          self.original = kwargs.get('original')
  
  
  class GitUrlError(GeneralError):
      """ Remote git url is not reachable """
  
  
  class FileError(GeneralError):
      """ File operation error """
  
  
  class RunError(GeneralError):
      """ Command execution error """
  
      def __init__(
              self,
              message: str,
              command: Union[str, List[str]],
              returncode: int,
              stdout: Optional[str] = None,
              stderr: Optional[str] = None,
              *args: Any,
              **kwargs: Any) -> None:
          super().__init__(*args, **kwargs)
          self.message = message
          self.command = command
          self.returncode = returncode
          self.stdout = stdout
          self.stderr = stderr
  
  
  class MetadataError(GeneralError):
      """ General metadata error """
  
  
  class SpecificationError(MetadataError):
      """ Metadata specification error """
  
      def __init__(
              self,
              message: str,
              validation_errors: Optional[List[Tuple[jsonschema.ValidationError, str]]] = None,
              *args: Any,
              **kwargs: Any) -> None:
          super().__init__(*args, **kwargs)
  
          self.message = message
          self.validation_errors = validation_errors
  
  
  class ConvertError(MetadataError):
      """ Metadata conversion error """
  
  
  class StructuredFieldError(GeneralError):
      """ StructuredField parsing error """
  
  
  class WaitingIncomplete(GeneralError):
      """ Waiting incomplete """
  
  
  class WaitingTimedOutError(GeneralError):
      """ Waiting ran out of time """
  
  
  # Step exceptions
  
  
  class DiscoverError(GeneralError):
      """ Discover step error """
  
  
  class ProvisionError(GeneralError):
      """ Provision step error """
  
  
  class PrepareError(GeneralError):
      """ Prepare step error """
  
  
  class ExecuteError(GeneralError):
      """ Execute step error """
  
  
  class RebootTimeoutError(ExecuteError):
      """ Reboot failed due to a timeout """
  
  
  class ReportError(GeneralError):
      """ Report step error """
  
  
  class FinishError(GeneralError):
      """ Finish step error """
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  Utilities
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  def quote(string: str) -> str:
      """ Surround a string with double quotes """
      return f'"{string}"'
  
  
  def ascii(text: Any) -> bytes:
      """ Transliterate special unicode characters into pure ascii """
      if not isinstance(text, str):
          text = str(text)
      return unicodedata.normalize('NFKD', text).encode('ascii', 'ignore')
  
  
  def listify(
          data: Union[Tuple[Any, ...], List[Any], str, Dict[Any, Any]],
          split: bool = False,
          keys: Optional[List[str]] = None) -> Union[List[Any], Dict[Any, Any]]:
      """
      Ensure that variable is a list, convert if necessary
      For dictionaries check all items or only those with provided keys.
      Also split strings on white-space/comma if split=True.
      """
      separator = re.compile(r'[\s,]+')
      if isinstance(data, tuple):
          data = list(data)
      if isinstance(data, list):
          return fmf.utils.split(data, separator) if split else data
      if isinstance(data, str):
          return fmf.utils.split(data, separator) if split else [data]
      if isinstance(data, dict):
          for key in keys or data:
              if key in data:
                  data[key] = listify(data[key], split=split)
          return data
      return [data]
  
  
  def copytree(
          src: str,
          dst: str,
          symlinks: bool = False,
          dirs_exist_ok: bool = False,
          ) -> Any:
      """ Similar to shutil.copytree but with dirs_exist_ok for Python < 3.8 """
      # No need to reimplement for newer python or if argument is not requested
      if not dirs_exist_ok or sys.version_info >= (3, 8):
          return shutil.copytree(
              src=src, dst=dst, symlinks=symlinks, dirs_exist_ok=dirs_exist_ok)
      # Choice was to either copy python implementation and change ONE line
      # or use rsync (or cp with shell)
      # We need to copy CONTENT of src into dst
      # so src has to end with / and dst cannot
      if src[-1] != '/':
          src += '/'
      if dst[-1] == '/':
          dst = dst[:-1]
  
      command = ["rsync", "-r"]
      if symlinks:
          command.append('-l')
      command.extend([src, dst])
  
      log.debug(f"Calling command '{command}'.")
      outcome = subprocess.run(
          command,
          stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
          stderr=subprocess.STDOUT, universal_newlines=True)
  
      if outcome.returncode != 0:
          raise shutil.Error(
              [f"Unable to copy '{src}' into '{dst}' using rsync.",
               outcome.returncode, outcome.stdout])
      return dst
  
  
  # These two are helpers for shell_to_dict and environment_to_dict -
  # there is some overlap of their functionality.
  def _add_simple_var(result: EnvironmentType, var: str) -> None:
      """
      Add a single NAME=VALUE pair into result dictionary
  
      Parse given string VAR to its constituents, NAME and VALUE, and add
      them to the provided dict.
      """
  
      matched = re.match("([^=]+)=(.*)", var)
      if not matched:
          raise GeneralError(f"Invalid variable specification '{var}'.")
      name, value = matched.groups()
      result[name] = value
  
  
  def _add_file_vars(result: EnvironmentType, filepath: str) -> None:
      """
      Add variables loaded from file into the result dictionary
  
      Load mapping from a YAML file 'filepath', and add its content -
      "name: value" entries - to the provided dict.
      """
  
      if not filepath[1:]:
          raise GeneralError(
              f"Invalid variable file specification '{filepath}'.")
  
      try:
          with open(filepath[1:], 'r') as file:
              # Handle empty file as an empty environment
              content = file.read()
              if not content:
                  log.warn(f"Empty environment file '{filepath}'.")
                  return
              file_vars = yaml_to_dict(content)
      except Exception as exception:
          raise GeneralError(
              f"Failed to load variables from '{filepath}': {exception}")
  
      for name, value in file_vars.items():
          result[name] = str(value)
  
  
  def shell_to_dict(variables: Union[str, List[str]]) -> EnvironmentType:
      """
      Convert shell-like variables into a dictionary
  
      Accepts single string or list of strings. Allowed forms are:
      'X=1'
      'X=1 Y=2 Z=3'
      ['X=1', 'Y=2', 'Z=3']
      ['X=1 Y=2 Z=3', 'A=1 B=2 C=3']
      'TXT="Some text with spaces in it"'
      """
      if not isinstance(variables, (list, tuple)):
          variables = [variables]
      result: EnvironmentType = dict()
      for variable in variables:
          if variable is None:
              continue
          for var in shlex.split(variable):
              _add_simple_var(result, var)
  
      return result
  
  
  def environment_to_dict(variables: Union[str, List[str]]) -> EnvironmentType:
      """
      Convert environment variables into a dictionary
  
      Variables may be specified in the following two ways:
  
      * NAME=VALUE pairs
      * @foo.yaml
  
      If "variable" starts with "@" character, it is treated as a path to
      a YAML file that contains "key: value" pairs which are then
      transparently loaded and added to the final dictionary.
  
      In general, allowed inputs are the same as in "shell_to_dict"
      function, with the addition of "@foo.yaml" form:
      'X=1'
      'X=1 Y=2 Z=3'
      ['X=1', 'Y=2', 'Z=3']
      ['X=1 Y=2 Z=3', 'A=1 B=2 C=3']
      'TXT="Some text with spaces in it"'
      @foo.yaml
      @../../bar.yaml
      """
  
      if not isinstance(variables, (list, tuple)):
          variables = [variables]
      result: EnvironmentType = dict()
  
      for variable in variables:
          if variable is None:
              continue
          for var in shlex.split(variable):
              if var.startswith('@'):
                  _add_file_vars(result, var)
              else:
                  _add_simple_var(result, var)
  
      return result
  
  
  @lru_cache(maxsize=None)
  def environment_file_to_dict(env_file: str, root: str = ".") -> EnvironmentType:
      """
      Read environment variables from the given file.
  
      File should be in YAML format (``.yaml`` or ``.yml`` suffixes), or in dotenv format.
  
      .. code-block:: bash
         :caption: dotenv file example
  
         A=B
         C=D
  
      .. code-block:: yaml
         :caption: YAML file example
  
         A: B
         C: D
  
      Path to each file should be relative to the metadata tree root.
  
      .. note::
  
         For loading environment variables from multiple files, see
         :py:func:`environment_files_to_dict`.
      """
  
      env_file = env_file.strip()
  
      # Fetch a remote file
      if env_file.startswith("http"):
          # Create retry session for longer retries, see #1229
          session = retry_session.create(
              retries=ENVFILE_RETRY_SESSION_RETRIES,
              backoff_factor=ENVFILE_RETRY_SESSION_BACKOFF_FACTOR,
              allowed_methods=('GET',),
              status_forcelist=(
                  429,  # Too Many Requests
                  500,  # Internal Server Error
                  502,  # Bad Gateway
                  503,  # Service Unavailable
                  504   # Gateway Timeout
                  ),
              )
          try:
              response = session.get(env_file)
              response.raise_for_status()
              content = response.text
          except requests.RequestException as error:
              raise GeneralError(
                  f"Failed to fetch the environment file from '{env_file}'. "
                  f"The problem was: '{error}'")
  
      # Read a local file
      else:
          # Ensure we don't escape from the metadata tree root
          try:
              root_path = Path(root).resolve()
              full_path = (Path(root_path) / Path(env_file)).resolve()
              full_path.relative_to(root_path)
          except ValueError:
              raise GeneralError(
                  f"The 'environment-file' path '{full_path}' is outside "
                  f"of the metadata tree root '{root}'.")
          if not Path(full_path).is_file():
              raise GeneralError(f"File '{full_path}' doesn't exist.")
  
          content = Path(full_path).read_text()
  
      # Parse yaml file
      if os.path.splitext(env_file)[1].lower() in ('.yaml', '.yml'):
          environment = parse_yaml(content)
  
      else:
          try:
              environment = parse_dotenv(content)
  
          except ValueError:
              raise GeneralError(
                  f"Failed to extract variables from environment file "
                  f"'{full_path}'. Ensure it has the proper format "
                  f"(i.e. A=B).")
  
      if not environment:
          log.warn(f"Empty environment file '{env_file}'.")
  
          return {}
  
      return environment
  
  
  def environment_files_to_dict(env_files: Iterable[str], root: str = ".") -> EnvironmentType:
      """
      Read environment variables from the given list of files.
  
      Files should be in YAML format (``.yaml`` or ``.yml`` suffixes), or in dotenv format.
  
      .. code-block:: bash
         :caption: dotenv file example
  
         A=B
         C=D
  
      .. code-block:: yaml
         :caption: YAML file example
  
         A: B
         C: D
  
      Path to each file should be relative to the metadata tree root.
  
      .. note::
  
         For loading environment variables from a single file, see
         :py:func:`environment_file_to_dict`, which is a function
         ``environment_files_to_dict()`` calls for each file,
         accumulating data from all input files.
      """
  
      result: EnvironmentType = {}
  
      for env_file in env_files:
          result.update(environment_file_to_dict(env_file, root=root))
  
      return result
  
  
  @contextlib.contextmanager
  def modify_environ(
          new_elements: EnvironmentType) -> Generator[None, None, None]:
      """ A context manager for os.environ that restores the initial state """
      environ_backup = os.environ.copy()
      os.environ.clear()
      os.environ.update(new_elements)
      try:
          yield
      finally:
          os.environ.clear()
          os.environ.update(environ_backup)
  
  
  def context_to_dict(context: List[str]) -> FmfContextType:
      """
      Convert command line context definition into a dictionary
  
      Does the same as environment_to_dict() plus separates possible
      comma-separated values into lists. Here's a couple of examples:
  
      distro=fedora-33 ---> {'distro': ['fedora']}
      arch=x86_64,ppc64 ---> {'arch': ['x86_64', 'ppc64']}
      """
      return {
          key: value.split(',')
          for key, value in environment_to_dict(context).items()}
  
  
  def dict_to_yaml(
          data: Union[Dict[str, Any], List[Any]],
          width: Optional[int] = None,
          sort: bool = False,
          start: bool = False) -> str:
      """ Convert dictionary into yaml """
      output = io.StringIO()
      yaml = YAML()
      yaml.indent(mapping=4, sequence=4, offset=2)
      yaml.default_flow_style = False
      yaml.allow_unicode = True
      yaml.encoding = 'utf-8'
      yaml.width = width
      yaml.explicit_start = start
      # Convert multiline strings
      scalarstring.walk_tree(data)
      if sort:
          # Sort the data https://stackoverflow.com/a/40227545
          sorted_data = CommentedMap()
          for key in sorted(data):
              sorted_data[key] = data[key]
          data = sorted_data
      yaml.dump(data, output)
      return output.getvalue()
  
  
  YamlTypType = Literal['rt', 'safe', 'unsafe', 'base']
  
  
  def yaml_to_dict(data: Any,
                   yaml_type: Optional[YamlTypType] = None) -> Dict[Any, Any]:
      """ Convert yaml into dictionary """
      yaml = YAML(typ=yaml_type)
      loaded_data = yaml.load(data)
      if loaded_data is None:
          return dict()
      if not isinstance(loaded_data, dict):
          raise GeneralError(
              f"Expected dictionary in yaml data, "
              f"got '{type(loaded_data).__name__}'.")
      return loaded_data
  
  
  def key_to_option(key: str) -> str:
      """ Convert a key name to corresponding option name """
  
      return key.replace('_', '-')
  
  
  def option_to_key(option: str) -> str:
      """ Convert an option name to corresponding key name """
  
      return option.replace('-', '_')
  
  
  SerializableContainerDerivedType = TypeVar(
      'SerializableContainerDerivedType',
      bound='SerializableContainer')
  
  
  @dataclasses.dataclass
  class SerializableContainer:
      """
      A mixin class for objects that may be saved in files and restored later
      """
  
      def to_dict(self) -> Dict[str, Any]:
          """ Return keys and values in the form of a dictionary """
  
          return dataclasses.asdict(self)
  
      # This method should remain a class-method: 1. list of keys is known
      # already, therefore it's not necessary to create an instance, and
      # 2. some functionality makes use of this knowledge.
      @classmethod
      def keys(cls) -> Generator[str, None, None]:
          """ Iterate over key names """
  
          for field in dataclasses.fields(cls):
              yield field.name
  
      def values(self) -> Generator[Any, None, None]:
          """ Iterate over key values """
  
          yield from self.to_dict().values()
  
      def items(self) -> Generator[Tuple[str, Any], None, None]:
          """ Iterate over key/value pairs """
  
          yield from self.to_dict().items()
  
      @classmethod
      def default(cls, key: str, default: Any = None) -> Any:
          """
          Return a default value for a given key.
  
          Keys may have a default value, or a default *factory* has been specified.
  
          :param key: key to look for.
          :param default: when key has no default value, ``default`` is returned.
          :returns: a default value defined for the key, or its ``default_factory``'s
              return value of ``default_factory``, or ``default`` when key has no
              default value.
          """
  
          for field in dataclasses.fields(cls):
              if key != field.name:
                  continue
  
              if not isinstance(field.default_factory, dataclasses._MISSING_TYPE):
                  return field.default_factory()
  
              if not isinstance(field.default, dataclasses._MISSING_TYPE):
                  return field.default
  
          else:
              return default
  
      @property
      def is_bare(self) -> bool:
          """
          Check whether all keys are either unset or have their default value.
  
          :returns: ``True`` if all keys either hold their default value
              or are not set at all, ``False`` otherwise.
          """
  
          for field in dataclasses.fields(self):
              value = getattr(self, field.name)
  
              if not isinstance(field.default_factory, dataclasses._MISSING_TYPE):
                  if value != field.default_factory():
                      return False
  
              elif not isinstance(field.default, dataclasses._MISSING_TYPE):
                  if value != field.default:
                      return False
  
              else:
                  pass
  
          return True
  
      #
      # Moving data between containers and objects owning them
      #
  
      def inject_to(self, obj: Any) -> None:
          """
          Inject keys from this container into attributes of a given object
          """
  
          for name, value in self.items():
              setattr(obj, name, value)
  
      @classmethod
      def extract_from(cls: Type[SerializableContainerDerivedType],
                       obj: Any) -> SerializableContainerDerivedType:
          """ Extract keys from given object, and save them in a container """
  
          data = cls()
  
          for key in cls.keys():
              value = getattr(obj, key)
              if value is not None:
                  setattr(data, key, value)
  
          return data
  
      #
      # Serialization - writing containers into YAML files, and restoring
      # them later.
      #
  
      def to_serialized(self) -> Dict[str, Any]:
          """
          Return keys and values in the form allowing later reconstruction.
  
          Used to transform container into a structure one can save in a
          YAML file, and restore it later.
  
          See :py:meth:`from_serialized` for its counterpart.
          """
  
          fields = self.to_dict()
  
          # Add a special field tracking what class we just shattered to pieces.
          fields.update({
              '__class__': {
                  'module': self.__class__.__module__,
                  'name': self.__class__.__name__
                  }
              })
  
          return fields
  
      @classmethod
      def from_serialized(
              cls: Type[SerializableContainerDerivedType],
              serialized: Dict[str, Any]) -> SerializableContainerDerivedType:
          """
          Recreate container from its serialized form.
  
          Used to transform data read from a YAML file into the original
          container.
  
          See :py:meth:`to_serialized` for its counterpart.
          """
  
          # Our special key may or may not be present, depending on who
          # calls this method.  In any case, it is not needed, because we
          # already know what class to restore: this one.
          serialized.pop('__class__', None)
  
          return cls(**serialized)
  
      @staticmethod
      def unserialize(serialized: Dict[str, Any]
                      ) -> SerializableContainerDerivedType:
          """
          Recreate container from its serialized form.
  
          Similar to :py:meth:`from_serialized`, but this method knows
          nothing about container's class, and will locate the correct
          module and class by inspecting serialized data. Discovered
          class' :py:meth:`from_serialized` is then used to create the
          container.
  
          Used to transform data read from a YAML file into original
          containers when their classes are not know to the code.
          Restoring such containers requires inspection of serialized data
          and dynamic imports of modules as needed.
          """
  
          from tmt.plugins import import_
  
          # Unpack class info, to get nicer variable names
          if "__class__" not in serialized:
              raise GeneralError(
                  "Failed to load saved state, probably because of old data format.\n"
                  "Use 'tmt clean runs' to clean up old runs.")
  
          klass_info = serialized.pop('__class__')
          klass_module_name = klass_info['module']
          klass_name = klass_info['name']
  
          # Make sure the module is imported. It probably is, but really,
          # make sure of it.
          import_(klass_module_name)
  
          # Now the module should be available in `sys.modules` like any
          # other, and we can go and grab the class we need from it.
          klass_module = sys.modules[klass_module_name]
          klass = getattr(klass_module, klass_name)
  
          if klass is None:
              raise SystemExit(
                  f"Failed to import '{klass_name}' "
                  f"from '{klass_module_name}' module.")
  
          # Stay away from classes that are not derived from this one, to
          # honor promise given by return value annotation.
          assert issubclass(klass, SerializableContainer)
  
          # Apparently, the issubclass() check above is not good enough for mypy.
          return cast(
              SerializableContainerDerivedType,
              klass.from_serialized(serialized))
  
  
  def markdown_to_html(filename: str) -> str:
      """
      Convert markdown to html
  
      Expects: Markdown document as a file.
      Returns: An HTML document as a string.
      """
      try:
          import markdown
      except ImportError:
          raise ConvertError("Install tmt-test-convert to export tests.")
  
      try:
          with open(filename, 'r') as file:
              try:
                  text = file.read()
              except UnicodeError:
                  raise MetadataError(f"Unable to read '{filename}'.")
              return markdown.markdown(text)
      except IOError:
          raise ConvertError(f"Unable to open '{filename}'.")
  
  
  def shell_variables(
          data: Union[List[str], Tuple[str, ...], Dict[str, Any]]) -> List[str]:
      """
      Prepare variables to be consumed by shell
  
      Convert dictionary or list/tuple of key=value pairs to list of
      key=value pairs where value is quoted with shlex.quote().
      """
  
      # Convert from list/tuple
      if isinstance(data, list) or isinstance(data, tuple):
          converted_data = []
          for item in data:
              splitted_item = item.split('=')
              key = splitted_item[0]
              value = shlex.quote('='.join(splitted_item[1:]))
              converted_data.append(f'{key}={value}')
          return converted_data
  
      # Convert from dictionary
      return [f"{key}={shlex.quote(str(value))}" for key, value in data.items()]
  
  
  def duration_to_seconds(duration: str) -> int:
      """ Convert sleep time format into seconds """
      units = {
          's': 1,
          'm': 60,
          'h': 60 * 60,
          'd': 60 * 60 * 24,
          }
      try:
          match = re.match(r'^(\d+)([smhd]?)$', str(duration))
          if match is None:
              raise SpecificationError(f"Invalid duration '{duration}'.")
          number, suffix = match.groups()
          return int(number) * units.get(suffix, 1)
      except (ValueError, AttributeError):
          raise SpecificationError(f"Invalid duration '{duration}'.")
  
  
  def verdict(
          decision: Optional[bool],
          comment: Optional[str] = None,
          good: str = 'pass',
          bad: str = 'fail',
          problem: str = 'warn',
          **kwargs: Any) -> Optional[bool]:
      """
      Print verdict in green, red or yellow based on the decision
  
      The supported decision values are:
  
          True .... good (green)
          False ... bad (red)
          None .... problem (yellow)
  
      Anything else raises an exception. Additional arguments
      are passed to the `echo` function. Returns back the decision.
      """
  
      if decision is False:
          text = style(bad, fg='red')
      elif decision is True:
          text = style(good, fg='green')
      elif decision is None:
          text = style(problem, fg='yellow')
      else:
          raise GeneralError(
              "Invalid decision value, must be 'True', 'False' or 'None'.")
      if comment:
          text = text + ' ' + comment
      echo(text, **kwargs)
      return decision
  
  
  def format(
          key: str,
          value: Union[None, str, List[Any], Dict[Any, Any]] = None,
          indent: int = 12,
          width: int = 72,
          wrap: Literal[True, False, 'auto'] = 'auto',
          key_color: str = 'green',
          value_color: str = 'black') -> str:
      """
      Nicely format and indent a key-value pair
  
      The following values for 'wrap' are supported:
  
          True .... always reformat text and wrap long lines
          False ... preserve text, no new line changes
          auto .... wrap only if text contains a long line
      """
      indent_string = (indent + 1) * ' '
      # Key
      output = '{} '.format(str(key).rjust(indent, ' '))
      if key_color is not None:
          output = style(output, fg=key_color)
      # Bool
      if isinstance(value, bool):
          output += ('true' if value else 'false')
      # List
      elif isinstance(value, list):
          # Make sure everything is string, prepare list, check for spaces
          value = [str(item) for item in value]
          listed_text = fmf.utils.listed(value)
          has_spaces = any([item.find(' ') > -1 for item in value])
          # Use listed output only for short lists without spaces
          if len(listed_text) < width - indent and not has_spaces:
              output += listed_text
          # Otherwise just place each item on a new line
          else:
              output += ('\n' + indent_string).join(value)
      # Dictionary
      elif isinstance(value, dict):
          # Place each key value pair on a separate line
          output += ('\n' + indent_string).join(
              f'{item[0]}: {item[1]}' for item in value.items())
      # Text
      elif isinstance(value, str):
          # In 'auto' mode enable wrapping when long lines present
          if wrap == 'auto':
              wrap = any(
                  [len(line) + indent - 7 > width
                   for line in value.split('\n')])
          if wrap:
              output += (wrap_text(
                  value, width=width,
                  preserve_paragraphs=True,
                  initial_indent=indent_string,
                  subsequent_indent=indent_string).lstrip())
          else:
              output += (('\n' + indent_string).join(
                  value.rstrip().split('\n')))
      else:
          output += str(value)
      return output
  
  
  def create_directory(
          path: str,
          name: str,
          dry: bool = False,
          quiet: bool = False) -> None:
      """ Create a new directory, handle errors """
      say = log.debug if quiet else echo
      if os.path.isdir(path):
          say("Directory '{}' already exists.".format(path))
          return
      if dry:
          say("Directory '{}' would be created.".format(path))
          return
      try:
          os.makedirs(path, exist_ok=True)
          say("Directory '{}' created.".format(path))
      except OSError as error:
          raise FileError("Failed to create {} '{}' ({})".format(
              name, path, error))
  
  
  def create_file(
          path: str,
          content: str,
          name: str,
          dry: bool = False,
          force: bool = False,
          mode: int = 0o664,
          quiet: bool = False) -> None:
      """ Create a new file, handle errors """
      say = log.debug if quiet else echo
      action = 'would be created' if dry else 'created'
      if os.path.exists(path):
          if force:
              action = 'would be overwritten' if dry else 'overwritten'
          else:
              raise FileError("File '{}' already exists.".format(path))
  
      if dry:
          say("{} '{}' {}.".format(name.capitalize(), path, action))
          return
  
      try:
          with open(path, 'w') as file_:
              file_.write(content)
          say("{} '{}' {}.".format(name.capitalize(), path, action))
          os.chmod(path, mode)
      except OSError as error:
          raise FileError("Failed to create {} '{}' ({})".format(
              name, path, error))
  
  
  # Avoid multiple subprocess calls for the same url
  @lru_cache(maxsize=None)
  def check_git_url(url: str) -> str:
      """ Check that a remote git url is accessible """
      try:
          log.debug(f"Check git url '{url}'.")
          subprocess.check_call(
              ["git", "ls-remote", "--heads", url],
              stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
              env={"GIT_ASKPASS": "echo", "GIT_TERMINAL_PROMPT": "0"})
          return url
      except subprocess.CalledProcessError:
          raise GitUrlError(f"Unable to contact remote git via '{url}'.")
  
  
  def public_git_url(url: str) -> str:
      """
      Convert a git url into a public format
  
      Return url in the format which can be accessed without
      authentication. For now just cover the most common services.
      """
  
      # GitHub, GitLab
      # old: git@github.com:teemtee/tmt.git
      # new: https://github.com/teemtee/tmt.git
      matched = re.match('git@(.*):(.*)', url)
      if matched:
          host, project = matched.groups()
          return f'https://{host}/{project}'
  
      # RHEL packages
      # old: git+ssh://psplicha@pkgs.devel.redhat.com/tests/bash
      # old: ssh://psplicha@pkgs.devel.redhat.com/tests/bash
      # old: ssh://pkgs.devel.redhat.com/tests/bash
      # new: git://pkgs.devel.redhat.com/tests/bash
      matched = re.match(
          r'(git\+)?ssh://(\w+@)?(pkgs\.devel\.redhat\.com)/(.*)', url)
      if matched:
          _, _, host, project = matched.groups()
          return f'git://{host}/{project}'
  
      # Fedora packages, Pagure
      # old: git+ssh://psss@pkgs.fedoraproject.org/tests/shell
      # old: ssh://psss@pkgs.fedoraproject.org/tests/shell
      # new: https://pkgs.fedoraproject.org/tests/shell
      matched = re.match(r'(git\+)?ssh://(\w+@)?([^/]*)/(.*)', url)
      if matched:
          _, _, host, project = matched.groups()
          return f'https://{host}/{project}'
  
      # Otherwise return unmodified
      return url
  
  
  class TimeoutHTTPAdapter(requests.adapters.HTTPAdapter):
      """
      Spice up request's session with custom timeout.
      """
  
      def __init__(self, *args: Any, **kwargs: Any) -> None:
          self.timeout = kwargs.pop('timeout', None)
  
          super().__init__(*args, **kwargs)
  
      def send(  # type: ignore # does not match superclass type on purpose
              self,
              request: requests.PreparedRequest,
              **kwargs: Any) -> requests.Response:
          kwargs.setdefault('timeout', self.timeout)
  
          return super().send(request, **kwargs)
  
  
  class RetryStrategy(requests.packages.urllib3.util.retry.Retry):  # type: ignore[misc]
      def increment(
              self,
              *args: Any,
              **kwargs: Any
              ) -> requests.packages.urllib3.util.retry.Retry:
          error = cast(Optional[Exception], kwargs.get('error', None))
  
          # Detect a subset of exception we do not want to follow with a retry.
          if error is not None:
              # Failed certificate verification - this issue will probably not get any better
              # should we try again.
              if isinstance(error, urllib3.exceptions.SSLError) \
                      and 'certificate verify failed' in str(error):
  
                  # [mpr] I'm not sure how stable this *iternal* API is, but pool seems to be the
                  # only place aware of the remote hostname. Try our best to get the hostname for
                  # a better error message, but don't crash because of a missing attribute or
                  # something as dumb.
  
                  connection_pool = kwargs.get('_pool', None)
  
                  if connection_pool is not None and hasattr(connection_pool, 'host'):
                      message = f"Certificate verify failed for '{connection_pool.host}'."
                  else:
                      message = 'Certificate verify failed.'
  
                  raise GeneralError(message, original=error) from error
  
          return super().increment(*args, **kwargs)
  
  
  class retry_session(contextlib.AbstractContextManager):  # type: ignore
      """
      Context manager for requests.Session() with retries and timeout
      """
      @staticmethod
      def create(
              retries: int = DEFAULT_RETRY_SESSION_RETRIES,
              backoff_factor: float = DEFAULT_RETRY_SESSION_BACKOFF_FACTOR,
              allowed_methods: Optional[Tuple[str, ...]] = None,
              status_forcelist: Optional[Tuple[int, ...]] = None,
              timeout: Optional[int] = None
              ) -> requests.Session:
          retry_strategy = RetryStrategy(
              total=retries,
              status_forcelist=status_forcelist,
              # `method_whitelist`` has been renamed to `allowed_methods` since
              # urllib3 1.26, and it will be removed in urllib3 2.0.
              # `allowed_methods` is therefore the future-proof name, but for the
              # sake of backward compatibility, internally we need to use the
              # deprecated parameter for now. Or request newer urllib3, but that
              # might a problem because of RPM availability.
              method_whitelist=allowed_methods,
              backoff_factor=backoff_factor)
  
          if timeout is not None:
              http_adapter: requests.adapters.HTTPAdapter = TimeoutHTTPAdapter(
                  timeout=timeout, max_retries=retry_strategy)
          else:
              http_adapter = requests.adapters.HTTPAdapter(
                  max_retries=retry_strategy)
  
          session = requests.Session()
          session.mount('http://', http_adapter)
          session.mount('https://', http_adapter)
  
          return session
  
      def __init__(
              self,
              retries: int = DEFAULT_RETRY_SESSION_RETRIES,
              backoff_factor: float = DEFAULT_RETRY_SESSION_BACKOFF_FACTOR,
              allowed_methods: Optional[Tuple[str, ...]] = None,
              status_forcelist: Optional[Tuple[int, ...]] = None,
              timeout: Optional[int] = None
              ) -> None:
          self.retries = retries
          self.backoff_factor = backoff_factor
          self.allowed_methods = allowed_methods
          self.status_forcelist = status_forcelist
          self.timeout = timeout
  
      def __enter__(self) -> requests.Session:
          return self.create(
              retries=self.retries,
              backoff_factor=self.backoff_factor,
              allowed_methods=self.allowed_methods,
              status_forcelist=self.status_forcelist,
              timeout=self.timeout)
  
      def __exit__(self, *args: Any) -> None:
          pass
  
  
  def remove_color(text: str) -> str:
      """ Remove ansi color sequences from the string """
      return re.sub(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])', '', text)
  
  
  def default_branch(repository: str, remote: str = 'origin') -> str:
      """ Detect default branch from given local git repository """
      head = os.path.join(repository, f'.git/refs/remotes/{remote}/HEAD')
      # Make sure the HEAD reference is available
      if not os.path.exists(head):
          subprocess.run(
              f'git remote set-head {remote} --auto'.split(), cwd=repository)
      # The ref format is 'ref: refs/remotes/origin/main'
      with open(head) as ref:
          return ref.read().strip().split('/')[-1]
  
  
  def parse_dotenv(content: str) -> EnvironmentType:
      """ Parse dotenv (shell) format of variables """
      return dict([line.split("=", maxsplit=1)
                  for line in shlex.split(content, comments=True)])
  
  
  def parse_yaml(content: str) -> EnvironmentType:
      """ Parse variables from yaml, ensure flat dictionary format """
      yaml_as_dict = YAML(typ="safe").load(content)
      # Handle empty file as an empty environment
      if yaml_as_dict is None:
          return dict()
      if any(isinstance(val, dict) for val in yaml_as_dict.values()):
          raise GeneralError(
              "Can't set the environment from the nested yaml config. The "
              "config should be just key, value pairs.")
      return {key: str(value) for key, value in yaml_as_dict.items()}
  
  
  def validate_git_status(test: 'tmt.base.Test') -> Tuple[bool, str]:
      """
      Validate that test has current metadata on fmf_id
  
      Return a tuple (boolean, message) as the result of validation.
  
      Checks that sources:
      - all local changes are committed
      - up to date on remote repository
      - .fmf/version marking fmf root is committed as well
  
      When all checks pass returns (True, '').
      """
      sources = test.node.sources + \
          [os.path.join(test.node.root, '.fmf', 'version')]
  
      # Use tmt's run instead of subprocess.run
      run = Common().run
  
      # Check for not committed metadata changes
      cmd = ['git', 'status', '--porcelain', '--'] + sources
      try:
          result = run(cmd, cwd=test.node.root, join=True)
      except RunError as error:
          return (
              False,
              f"Failed to run git status: {error.stdout}"
              )
  
      not_committed = []
      assert result.stdout is not None
      for line in result.stdout.split('\n'):
          if line:
              # XY PATH or XY ORIG -> PATH. XY and PATH are separated by space
              not_committed.append(line[3:])
  
      if not_committed:
          return (False, "Uncommitted changes in " + " ".join(not_committed))
  
      # Check for not pushed changes
      cmd = ["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"]
      try:
          result = run(cmd, cwd=test.node.root)
      except RunError as error:
          return (
              False,
              f'Failed to get remote branch, error raised: "{error.stderr}"'
              )
  
      assert result.stdout is not None
      remote_ref = result.stdout.strip()
  
      cmd = [
          'git',
          'diff',
          f'HEAD..{remote_ref}',
          '--name-status',
          '--'] + sources
      try:
          result = run(cmd, cwd=test.node.root)
      except RunError as error:
          return (
              False,
              f'Failed to diff against remote branch, error raised: "{error.stderr}"')
  
      not_pushed = []
      assert result.stdout is not None
      for line in result.stdout.split('\n'):
          if line:
              _, path = line.strip().split('\t', maxsplit=2)
              not_pushed.append(path)
      if not_pushed:
          return (False, "Not pushed changes in " + " ".join(not_pushed))
  
      return (True, '')
  
  
  def generate_runs(
          path: str, id_: Optional[str] = None) -> Generator[str, None, None]:
      """ Generate absolute paths to runs from path """
      # Prepare absolute workdir path if --id was used
      if id_:
          if '/' not in id_:
              id_ = os.path.join(path, id_)
          if os.path.isabs(id_):
              if os.path.exists(id_):
                  yield id_
              return
      if not os.path.exists(path):
          return
      for filename in os.listdir(path):
          abs_path = os.path.join(path, filename)
          # If id_ is None, the abs_path is considered valid (no filtering
          # is being applied). If it is defined, it has been transformed
          # to absolute path and must be equal to abs_path for the run
          # in abs_path to be generated.
          invalid_id = id_ and abs_path != id_
          invalid_run = not os.path.exists(
              os.path.join(abs_path, 'run.yaml'))
          if not os.path.isdir(abs_path) or invalid_id or invalid_run:
              continue
          yield abs_path
  
  
  def load_run(run: 'tmt.base.Run') -> Tuple[bool, Optional[Exception]]:
      """ Load a run and its steps from the workdir """
      try:
          run.load_from_workdir()
      except GeneralError as error:
          return False, error
      for plan in run.plans:
          for step in plan.steps(disabled=True):
              step.load()
      return True, None
  
  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  #  StructuredField
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
  SFSectionValueType = Union[str, List[str]]
  
  
  class StructuredField:
      """
      Handling multiple text data in a single text field
  
      The StructuredField allows you to easily store and extract several
      sections of text data to/from a single text field. The sections are
      separated by section names in square brackets and can be hosted in
      other text as well.
  
      The section names have to be provided on a separate line and there
      must be no leading/trailing white space before/after the brackets.
      The StructuredField supports two versions of the format:
  
      Version 0: Simple, concise, useful when neither the surrounding text
      or the section data can contain lines which could resemble section
      names. Here's an example of a simple StructuredField::
  
          Note written by human.
  
          [section-one]
          Section one content.
  
          [section-two]
          Section two content.
  
          [section-three]
          Section three content.
  
          [end]
  
          Another note written by human.
  
      Version 1: Includes unique header to prevent collisions with the
      surrounding text and escapes any section-like lines in the content::
  
          Note written by human.
  
          [structured-field-start]
          This is StructuredField version 1. Please, edit with care.
  
          [section-one]
          Section one content.
  
          [section-two]
          Section two content.
          [structured-field-escape][something-resembling-section-name]
  
          [section-three]
          Section three content.
  
          [structured-field-end]
  
          Another note written by human.
  
      Note that an additional empty line is added at the end of each
      section to improve the readability. This line is not considered
      to be part of the section content.
  
      Besides handling the whole section content it's also possible to
      store several key-value pairs in a single section, similarly as in
      the ini config format::
  
          [section]
          key1 = value1
          key2 = value2
          key3 = value3
  
      Provide the key name as the optional argument 'item' when accessing
      these single-line items. Note that the section cannot contain both
      plain text data and key-value pairs.
  
      Example::
  
          field = qe.StructuredField()
          field.set("project", "Project Name")
          field.set("details", "somebody", "owner")
          field.set("details", "2013-05-27", "started")
          field.set("description", "This is a description.\\n"
                  "It spans across multiple lines.\\n")
          print field.save()
  
              [structured-field-start]
              This is StructuredField version 1. Please, edit with care.
  
              [project]
              Project Name
  
              [details]
              owner = somebody
              started = 2013-05-27
  
              [description]
              This is a description.
              It spans across multiple lines.
  
              [structured-field-end]
  
          field.version(0)
          print field.save()
  
              [project]
              Project Name
  
              [details]
              owner = somebody
              started = 2013-05-27
  
              [description]
              This is a description.
              It spans across multiple lines.
  
              [end]
  
      Multiple values for the same key are supported as well. Enable this
      feature with 'multi=True' when initializing the structured field.
      If multiple values are present their list will be returned instead
      of a single string. Similarly use list for setting multiple values::
  
          field = qe.StructuredField(multi=True)
          requirements = ['hypervisor=', 'labcontroller=lab.example.com']
          field.set("hardware", requirements, "hostrequire")
          print field.save()
  
              [structured-field-start]
              This is StructuredField version 1. Please, edit with care.
  
              [hardware]
              hostrequire = hypervisor=
              hostrequire = labcontroller=lab.example.com
  
              [structured-field-end]
  
          print field.get("hardware", "hostrequire")
  
              ['hypervisor=', 'labcontroller=lab.example.com']
      """
  
      # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
      #  StructuredField Special
      # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
      def __init__(
              self,
              text: Optional[str] = None,
              version: int = 1,
              multi: bool = False) -> None:
          """ Initialize the structured field """
          self.version(version)
          self._header: str = ""
          self._footer: str = ""
          # Sections are internally stored in their serialized form, i.e. as
          # strings.
          self._sections: Dict[str, str] = {}
          self._order: List[str] = []
          self._multi = multi
          if text is not None:
              self.load(text)
  
      def __iter__(self) -> Generator[str, None, None]:
          """ By default iterate through all available section names """
          for section in self._order:
              yield section
  
      def __nonzero__(self) -> bool:
          """ True when any section is defined """
          return len(self._order) > 0
  
      __bool__ = __nonzero__
  
      # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
      #  StructuredField Private
      # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
      def _load_version_zero(self, text: str) -> None:
          """ Load version 0 format """
          # Attempt to split the text according to the section tag
          section = re.compile(r"\n?^\[([^\]]+)\]\n", re.MULTILINE)
          parts = section.split(text)
          # If just one part ---> no sections present, just plain text
          if len(parts) == 1:
              self._header = parts[0]
              return
          # Pick header & footer, make sure [end] tag is present
          self._header = parts[0]
          self._footer = re.sub("^\n", "", parts[-1])
          if parts[-2] != "end":
              raise StructuredFieldError("No [end] section tag found")
          # Convert to dictionary and save the order
          keys = parts[1:-2:2]
          values = parts[2:-2:2]
          for key, value in zip(keys, values):
              self.set(key, value)
  
      def _load(self, text: str) -> None:
          """ Load version 1+ format """
          # The text must exactly match the format
          format = re.compile(
              r"(.*)^\[structured-field-start\][ \t]*\n"
              r"(.*)\n\[structured-field-end\][ \t]*\n(.*)",
              re.DOTALL + re.MULTILINE)
          # No match ---> plain text or broken structured field
          matched = format.search(text)
          if not matched:
              if "[structured-field" in text:
                  raise StructuredFieldError("StructuredField parse error")
              self._header = text
              log.debug("StructuredField not found, treating as a plain text")
              return
          # Save header & footer (remove trailing new lines)
          self._header = re.sub("\n\n$", "\n", matched.groups()[0])
          if self._header:
              log.debug(u"Parsed header:\n{0}".format(self._header))
          self._footer = re.sub("^\n", "", matched.groups()[2])
          if self._footer:
              log.debug(u"Parsed footer:\n{0}".format(self._footer))
          # Split the content on the section names
          section = re.compile(r"\n\[([^\]]+)\][ \t]*\n", re.MULTILINE)
          parts = section.split(matched.groups()[1])
          # Detect the version
          version_match = re.search(r"version (\d+)", parts[0])
          if not version_match:
              log.error(parts[0])
              raise StructuredFieldError(
                  "Unable to detect StructuredField version")
          self.version(int(version_match.groups()[0]))
          log.debug(
              "Detected StructuredField version {0}".format(
                  self.version()))
          # Convert to dictionary, remove escapes and save the order
          keys = parts[1::2]
          escape = re.compile(r"^\[structured-field-escape\]", re.MULTILINE)
          values = [escape.sub("", value) for value in parts[2::2]]
          for key, value in zip(keys, values):
              self.set(key, value)
          log.debug(u"Parsed sections:\n{0}".format(
              pprint.pformat(self._sections)))
  
      def _save_version_zero(self) -> str:
          """ Save version 0 format """
          result = []
          if self._header:
              result.append(self._header)
          for section, content in self.iterate():
              result.append(u"[{0}]\n{1}".format(section, content))
          if self:
              result.append(u"[end]\n")
          if self._footer:
              result.append(self._footer)
          return "\n".join(result)
  
      def _save(self) -> str:
          """ Save version 1+ format """
          result = []
          # Regular expression for escaping section-like lines
          escape = re.compile(r"^(\[.+\])$", re.MULTILINE)
          # Header
          if self._header:
              result.append(self._header)
          # Sections
          if self:
              result.append(
                  u"[structured-field-start]\n"
                  u"This is StructuredField version {0}. "
                  u"Please, edit with care.\n".format(self._version))
              for section, content in self.iterate():
                  result.append(u"[{0}]\n{1}".format(section, escape.sub(
                      "[structured-field-escape]\\1", content)))
              result.append(u"[structured-field-end]\n")
          # Footer
          if self._footer:
              result.append(self._footer)
          return "\n".join(result)
  
      def _read_section(self, content: str) -> Dict[str, SFSectionValueType]:
          """ Parse config section and return ordered dictionary """
          dictionary: Dict[str, SFSectionValueType] = OrderedDict()
          for line in content.split("\n"):
              # Remove comments and skip empty lines
              line = re.sub("#.*", "", line)
              if re.match(r"^\s*$", line):
                  continue
              # Parse key and value
              matched = re.search("([^=]+)=(.*)", line)
              if not matched:
                  raise StructuredFieldError(
                      "Invalid key/value line: {0}".format(line))
              key = matched.groups()[0].strip()
              value = matched.groups()[1].strip()
              # Handle multiple values if enabled
              if key in dictionary and self._multi:
                  stored_value = dictionary[key]
                  if isinstance(stored_value, list):
                      stored_value.append(value)
                  else:
                      dictionary[key] = [stored_value, value]
              else:
                  dictionary[key] = value
          return dictionary
  
      def _write_section(self, dictionary: Dict[str, SFSectionValueType]) -> str:
          """ Convert dictionary into a config section format """
          section = ""
          for key in dictionary:
              if isinstance(dictionary[key], list):
                  for value in dictionary[key]:
                      section += "{0} = {1}\n".format(key, value)
              else:
                  section += "{0} = {1}\n".format(key, dictionary[key])
          return section
  
      # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
      #  StructuredField Methods
      # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
      def iterate(self) -> Generator[Tuple[str, str], None, None]:
          """ Return (section, content) tuples for all sections """
          for section in self:
              yield section, self._sections[section]
  
      def version(self, version: Optional[int] = None) -> int:
          """ Get or set the StructuredField version """
          if version is not None:
              if version in [0, 1]:
                  self._version = version
              else:
                  raise StructuredFieldError(
                      "Bad StructuredField version: {0}".format(version))
          return self._version
  
      def load(self, text: str, version: Optional[int] = None) -> None:
          """ Load the StructuredField from a string """
          if version is not None:
              self.version(version)
          # Make sure we got a text, convert from bytes if necessary
          if isinstance(text, bytes):
              text = text.decode("utf8")
          if not isinstance(text, str):
              raise StructuredFieldError(
                  "Invalid StructuredField, expecting string")
          # Remove possible carriage returns
          text = re.sub("\r\n", "\n", text)
          # Make sure the text has a new line at the end
          if text and text[-1] != "\n":
              text += "\n"
          log.debug(u"Parsing StructuredField\n{0}".format(text))
          # Parse respective format version
          if self._version == 0:
              self._load_version_zero(text)
          else:
              self._load(text)
  
      def save(self) -> str:
          """ Convert the StructuredField into a string """
          if self.version() == 0:
              return self._save_version_zero()
          else:
              return self._save()
  
      def header(self, content: Optional[str] = None) -> str:
          """ Get or set the header content """
          if content is not None:
              self._header = content
          return self._header
  
      def footer(self, content: Optional[str] = None) -> str:
          """ Get or set the footer content """
          if content is not None:
              self._footer = content
          return self._footer
  
      def sections(self) -> List[str]:
          """ Get the list of available sections """
          return self._order
  
      def get(
              self,
              section: str,
              item: Optional[str] = None) -> SFSectionValueType:
          """ Return content of given section or section item """
          try:
              content = self._sections[section]
          except KeyError:
              raise StructuredFieldError(
                  "Section [{0!r}] not found".format(ascii(section)))
          # Return the whole section content
          if item is None:
              return content
          # Return only selected item from the section
          try:
              return self._read_section(content)[item]
          except KeyError:
              raise StructuredFieldError(
                  "Unable to read '{0!r}' from section '{1!r}'".format(
                      ascii(item), ascii(section)))
  
      def set(self, section: str, content: Any,
              item: Optional[str] = None) -> None:
          """ Update content of given section or section item """
          # Convert to string if necessary, keep lists untouched
          if isinstance(content, list):
              pass
          elif isinstance(content, bytes):
              content = content.decode("utf8")
          elif not isinstance(content, str):
              content = str(content)
          # Set the whole section content
          if item is None:
              # Add new line if missing
              if content and content[-1] != "\n":
                  content += "\n"
              self._sections[section] = content
          # Set only selected item from the section
          else:
              try:
                  current = self._sections[section]
              except KeyError:
                  current = ""
              dictionary = self._read_section(current)
              dictionary[item] = content
              self._sections[section] = self._write_section(dictionary)
          # Remember the order when adding a new section
          if section not in self._order:
              self._order.append(section)
  
      def remove(self, section: str, item: Optional[str] = None) -> None:
          """ Remove given section or section item """
          # Remove the whole section
          if item is None:
              try:
                  del self._sections[section]
                  del self._order[self._order.index(section)]
              except KeyError:
                  raise StructuredFieldError(
                      "Section [{0!r}] not found".format(ascii(section)))
          # Remove only selected item from the section
          else:
              try:
                  dictionary = self._read_section(self._sections[section])
                  del(dictionary[item])
              except KeyError:
                  raise StructuredFieldError(
                      "Unable to remove '{0!r}' from section '{1!r}'".format(
                          ascii(item), ascii(section)))
              self._sections[section] = self._write_section(dictionary)
  
  
  class DistGitHandler:
      """ Common functionality for DistGit handlers """
      sources_file_name = 'sources'
      uri = "/rpms/{name}/{filename}/{hashtype}/{hash}/{filename}"
  
      usage_name: str  # Name to use for dist-git-type
      re_source: Pattern[str]
      re_ignore_extensions: Pattern[str] = re.compile(r'\.(sign|asc|key)$')
      lookaside_server: str
      remote_substring: Pattern[str]
  
      def url_and_name(self, cwd: str = '.') -> List[Tuple[str, str]]:
          """
          Return list of urls and basenames of the used source
  
          The 'cwd' parameter has to be a DistGit directory.
          """
          # Assumes <package>.spec
          globbed = glob.glob(os.path.join(cwd, '*.spec'))
          if len(globbed) != 1:
              raise GeneralError(f"No .spec file is present in '{cwd}'.")
          package = os.path.basename(globbed[0])[:-len('.spec')]
          ret_values = []
          try:
              with open(os.path.join(cwd, self.sources_file_name)) as f:
                  for line in f.readlines():
                      match = self.re_source.match(line)
                      if match is None:
                          raise GeneralError(
                              f"Couldn't match '{self.sources_file_name}' "
                              f"content with '{self.re_source.pattern}'.")
                      used_hash, source_name, hash_value = match.groups()
                      ret_values.append((self.lookaside_server + self.uri.format(
                          name=package,
                          filename=source_name,
                          hash=hash_value,
                          hashtype=used_hash.lower()
                          ), source_name))
          except Exception as error:
              raise GeneralError(
                  f"Couldn't read '{self.sources_file_name}' file.",
                  original=error)
          if not ret_values:
              raise GeneralError(
                  "No sources found in '{self.sources_file_name}' file.")
          return ret_values
  
      def its_me(self, remotes: List[str]) -> bool:
          """ True if self can work with remotes """
          return any([self.remote_substring.search(item) for item in remotes])
  
  
  class FedoraDistGit(DistGitHandler):
      """ Fedora Handler """
      usage_name = "fedora"
      re_source = re.compile(r"^(\w+) \(([^)]+)\) = ([0-9a-fA-F]+)$")
      lookaside_server = "https://src.fedoraproject.org/repo/pkgs"
      remote_substring = re.compile(r'fedoraproject\.org')
  
  
  class CentOSDistGit(DistGitHandler):
      """ CentOS Handler """
      usage_name = "centos"
      re_source = re.compile(r"^(\w+) \(([^)]+)\) = ([0-9a-fA-F]+)$")
      lookaside_server = "https://sources.stream.centos.org/sources"
      remote_substring = re.compile(r'redhat/centos')
  
  
  def get_distgit_handler(
          remotes: Optional[List[str]] = None,
          usage_name: Optional[str] = None) -> DistGitHandler:
      """
      Return the right DistGitHandler
  
      Pick the DistGitHandler class which understands specified
      remotes or by usage_name.
      """
      for candidate_class in DistGitHandler.__subclasses__():
          if usage_name is not None and usage_name == candidate_class.usage_name:
              return candidate_class()
          if remotes is not None:
              ret_val = candidate_class()
              if ret_val.its_me(remotes):
                  return ret_val
      raise GeneralError(f"No known remote in '{remotes}'.")
  
  
  def get_distgit_handler_names() -> List[str]:
      """ All known distgit handlers """
      return [i.usage_name for i in DistGitHandler.__subclasses__()]
  
  
  def git_clone(
          url: str,
          destination: str,
          common: Common,
          env: Optional[EnvironmentType] = None,
          shallow: bool = False
          ) -> CommandOutput:
      """
      Git clone url to destination, retry without shallow if necessary
  
      For shallow=True attempt to clone repository using --depth=1 option first.
      If not successful attempt to clone whole repo.
  
      Common instance is used to run the command for appropriate logging.
      Environment is updated by 'env' dictionary.
      """
      depth = ['--depth=1'] if shallow else []
      command = ['git', 'clone'] + depth + [url, destination]
      try:
          return common.run(command, env=env)
      except RunError:
          if not shallow:
              # Do not retry if shallow was not used
              raise
          # Git server might not support shallow cloning, try again
          return git_clone(url, destination, common, env, shallow=False)
  
  
  class updatable_message(contextlib.AbstractContextManager):  # type: ignore
      """ Updatable message suitable for progress-bar-like reporting """
  
      def __init__(
              self,
              key: str,
              enabled: bool = True,
              indent_level: int = 0,
              key_color: Optional[str] = None,
              default_value_color: Optional[str] = None
              ) -> None:
          """
          Updatable message suitable for progress-bar-like reporting.
  
          .. code:block:: python3
  
             with updatable_message('foo') as message:
                 while ...:
                     ...
  
                     # check state of remote request, and update message
                     state = remote_api.check()
                     message.update(state)
  
          :param key: a string to use as the left-hand part of logged message.
          :param enabled: if unset, no output would be performed.
          :param indent_level: desired indentation level.
          :param key_color: optional color to apply to ``key``.
          :param default_color: optional color to apply to value when
              :py:meth:`update` is called with ``color`` left out.
          """
  
          self.key = key
          self.enabled = enabled
          self.indent_level = indent_level
          self.key_color = key_color
          self.default_value_color = default_value_color
  
          # No progress if terminal not attached
          if not sys.stdout.isatty():
              self.enabled = False
  
          self._previous_line: Optional[str] = None
  
      def __enter__(self) -> 'updatable_message':
          return self
  
      def __exit__(self, *args: Any) -> None:
          sys.stdout.write('\n')
          sys.stdout.flush()
  
      def update(self, value: str, color: Optional[str] = None) -> None:
          if not self.enabled:
              return
  
          if self._previous_line is not None:
              message = value.ljust(len(self._previous_line))
  
          else:
              message = value
  
          self._previous_line = value
  
          message = indent(
              self.key,
              value=style(
                  message,
                  fg=color or self.default_value_color),
              color=self.key_color,
              level=self.indent_level)
  
          sys.stdout.write(f"\r{message}")
          sys.stdout.flush()
  
  
  def find_fmf_root(path: str) -> List[str]:
      """
      Search trough path and return all fmf roots that exist there
  
      Returned list is ordered by path length, shortest one first.
  
      Raise `MetadataError` if no fmf root is found.
      """
      fmf_roots = []
      for root, _, files in os.walk(path):
          if not os.path.basename(root) == '.fmf':
              continue
          if 'version' in files:
              fmf_roots.append(os.path.dirname(root))
      if not fmf_roots:
          raise MetadataError(f"No fmf root present inside '{path}'.")
      fmf_roots.sort(key=lambda path: len(path))
      return fmf_roots
  
  
  #
  # JSON schema-based validation helpers
  #
  # Aims at FMF data consumed by tmt, but can be used for any structure.
  #
  
  # `Schema` represents a loaded JSON schema structure. It may be fairly complex,
  # but it's not needed to provide the exhaustive and fully detailed type since
  # tmt code is not actually "reading" it. Loaded schema is passed down to
  # jsonschema library, and while `Any` would be perfectly valid, let's use an
  # alias to make schema easier to track in our code.
  Schema = Dict[str, Any]
  SchemaStore = Dict[str, Schema]
  
  
  def _patch_plan_schema(schema: Schema, store: SchemaStore) -> None:
      """
      Resolve references to per-plugin schema known to steps. All schemas have
      been loaded into store, all that's left is to update each step in plan
      schema with the list of schemas allowed for that particular step.
  
      For each step, we create the following schema (see also plan.yaml for the
      rest of plan schema):
  
      .. code-block:: yaml
  
         <step name>:
           oneOf:
             - $ref: "/schemas/<step name>/plugin1"
             - $ref: "/schemas/<step name>/plugin2"
             ...
             - $ref: "/schemas/<step name>/pluginN"
             - type: array
               items:
                 anyOf:
                   - $ref: "/schemas/<step name>/plugin1"
                   - $ref: "/schemas/<step name>/plugin2"
                   ...
                   - $ref: "/schemas/<step name>/pluginN"
      """
  
      for step in ('discover', 'execute', 'finish', 'prepare', 'provision', 'report'):
          step_schema_prefix = f'/schemas/{step}/'
  
          step_plugin_schema_ids = [schema_id for schema_id in store.keys() if schema_id.startswith(
              step_schema_prefix) and schema_id != '/schemas/provision/hardware']
  
          refs: List[Schema] = [
              {'$ref': schema_id} for schema_id in step_plugin_schema_ids
              ]
  
          schema['properties'][step] = {
              'oneOf': refs + [
                  {
                      'type': 'array',
                      'items': {
                          'anyOf': refs
                          }
                      }
                  ]
              }
  
  
  def _load_schema(schema_filepath: str) -> Schema:
      """
      Load a JSON schema from a given filepath.
  
      A helper returning the raw loaded schema.
      """
  
      if not os.path.isabs(schema_filepath):
          schema_filepath = os.path.join(
              pkg_resources.resource_filename(
                  'tmt', 'schemas'), schema_filepath)
  
      try:
          with open(schema_filepath, 'r', encoding='utf-8') as f:
              return cast(Schema, yaml_to_dict(f.read()))
  
      except Exception as exc:
          raise FileError(f"Failed to load schema file {schema_filepath}\n{exc}")
  
  
  @functools.lru_cache(maxsize=None)
  def load_schema(schema_filepath: str) -> Schema:
      """
      Load a JSON schema from a given filepath.
  
      Recommended for general use, the method may apply some post-loading touches
      to the given schema, and unless caller is interested in the raw content of
      the file, this functions should be used instead of the real workhorse of
      schema loading, :py:func:`_load_schema`.
      """
  
      schema = _load_schema(schema_filepath)
  
      if schema.get('$id') == '/schemas/plan':
          _patch_plan_schema(schema, load_schema_store())
  
      return schema
  
  
  @functools.lru_cache(maxsize=None)
  def load_schema_store() -> SchemaStore:
      """
      Load all available JSON schemas, and put them into a "store".
  
      Schema store is a simple mapping between schema IDs and schemas.
      """
  
      store: SchemaStore = {}
  
      schema_dirpath = pkg_resources.resource_filename('tmt', 'schemas')
  
      try:
          for dirpath, dirnames, filenames in os.walk(
                  schema_dirpath, followlinks=True):
              for filename in filenames:
                  # Ignore all files but YAML files.
                  if os.path.splitext(filename)[1].lower() not in ('.yaml', '.yml'):
                      continue
  
                  schema = _load_schema(os.path.join(dirpath, filename))
  
                  store[schema['$id']] = schema
  
      except Exception as exc:
          raise FileError(f"Failed to discover schema files\n{exc}")
  
      if '/schemas/plan' not in store:
          raise FileError('Failed to discover schema for plans')
  
      _patch_plan_schema(store['/schemas/plan'], store)
  
      return store
  
  
  def _prenormalize_fmf_node(node: fmf.Tree, schema_name: str) -> fmf.Tree:
      """
      Apply the minimal possible normalization steps to nodes before validating them with schemas.
  
      tmt allows some fields to have default values, and at least ``how`` field is necessary for
      schema-based validation to work reliably. Based on ``how`` field, plan schema identifies
      the correct *plugin* schema for step validation. Without ``how``, it's hard to pick the
      correct schema.
  
      This function tries to do minimal number of changes to a given fmf node to honor the promise
      of ``how`` being optional, with known defaults for each step. It might be possible to resolve
      this purely with schemas, but since we don't know how (yet?), a Python implementation has been
      chosen to unblock schema-based validation while keeping things easier for users. This may
      change in the future, dropping the need for this pre-validation step.
  
      .. note::
  
         This function is not part of the normalization process that happens after validation. The
         purpose of this function is to make the world nice and shiny for tmt users while avoiding
         the possibility of schema becoming way too complicated, especially when we would need
         non-trivial amount fo time for experiments.
  
         The real normalization process takes place after validation, and is responsible for
         converting raw fmf data to data types and structures more suited for tmt internal
         implementation.
      """
  
      # As of now, only `how` field in plan steps seems to be required for schema-based validation
      # to work correctly, therefore ignore any other node.
      if schema_name != 'plan.yaml':
          return node
  
      # Perform the very crude and careful semi-validation. We need to set the `how` key to a default
      # value - but it's not our job to validate the general structure of node data. Walk the "happy"
      # path, touch the node only when it matches the specification of being a mapping of steps and
      # these being either mappings or lists of mappings. Whenever we notice some value does not
      # match this basic structure, ignore the step completely - its issues will be cought by schema
      # later, don't waste time on steps that do not follow specification.
  
      # Fmf data describing a plan shall be a mapping (with keys like `discover` or `adjust`).
      if not isinstance(node.data, dict):
          return node
  
      # Avoid possible circular imports
      import tmt.steps
  
      def _process_step(step_name: str, step: Dict[Any, Any]) -> None:
          """
          Process a single step configuration.
          """
  
          # If `how` is set, don't touch it, and there's nothing to do.
          if 'how' in step:
              return
  
          # Magic!
          # No, seriously: step is implemented in `tmt.steps.$step_name` package,
          # by a class `tmt.steps.$step_name.$step_name_with_capitalized_first_letter`.
          # Instead of having a set of if-elif tests, we can reach the default `how`
          # dynamically.
  
          from tmt.plugins import import_
  
          step_module_name = f'tmt.steps.{step_name}'
          step_class_name = step_name.capitalize()
  
          # Make sure the step module is imported. It probably is, but really,
          # make sure of it.
          import_(step_module_name)
  
          # Now the module should be available in `sys.modules` like any
          # other, and we can go and grab the class we need from it.
          step_module = sys.modules[step_module_name]
          step_class = getattr(step_module, step_class_name)
  
          if step_class is None:
              raise GeneralError(
                  f'Step {step_name} implementation cannot be found '
                  f'in {step_module_name}.{step_class_name}')
  
          step['how'] = cast(tmt.steps.Step, step_class).DEFAULT_HOW
  
      def _process_step_collection(step_name: str, step_collection: Any) -> None:
          """
          Process a collection of step configurations.
          """
  
          # Ignore anything that is not a step.
          if step_name not in tmt.steps.STEPS:
              return
  
          # A single step configuration, represented as a mapping.
          if isinstance(step_collection, dict):
              _process_step(step_name, step_collection)
  
              return
  
          # Multiple step configurations, as mappings in a list
          if isinstance(step_collection, list):
              for step_config in step_collection:
                  # Unexpected, maybe instead of a mapping describing a step someone put
                  # in an integer... Ignore, schema will report it.
                  if not isinstance(step_config, dict):
                      continue
  
                  _process_step(step_name, step_config)
  
      for step_name, step_config in node.data.items():
          _process_step_collection(step_name, step_config)
  
      return node
  
  
  def validate_fmf_node(
          node: fmf.Tree, schema_name: str) -> List[Tuple[jsonschema.ValidationError, str]]:
      """ Validate a given fmf node """
  
      node = _prenormalize_fmf_node(node, schema_name)
  
      result = node.validate(load_schema(schema_name), schema_store=load_schema_store())
  
      if result.result is True:
          return []
  
      # A bit of error formatting. It is possible to use str(error), but the result
      # is a bit too JSON-ish. Let's present an error message in a way that helps
      # users to point finger on each and every issue. But don't throw the original
      # errors away!
  
      errors: List[Tuple[jsonschema.ValidationError, str]] = []
  
      for error in result.errors:
          path = f'{node.name}:{".".join(error.path)}'
  
          errors.append((error, f'{path} - {error.message}'))
  
      return errors
  
  
  # A type for callbacks given to wait()
  WaitCheckType = Callable[[], T]
  
  
  def wait(
      parent: Common,
      check: WaitCheckType[T],
      timeout: datetime.timedelta,
      tick: float = DEFAULT_WAIT_TICK,
      tick_increase: float = DEFAULT_WAIT_TICK_INCREASE
  ) -> T:
      """
      Wait for a condition to become true.
  
      To test the condition state, a ``check`` callback is called every ``tick``
      seconds until ``check`` reports a success. The callback may:
  
      * decide the condition has been fulfilled. This is a successfull outcome,
        ``check`` shall then simply return, and waiting ends. Or,
      * decide more time is needed. This is not a successfull outcome, ``check``
        shall then raise :py:clas:`WaitingIncomplete` exception, and ``wait()``
        will try again later.
  
      :param parent: "owner" of the wait process. Used for its logging capability.
      :param check: a callable responsible for testing the condition. Accepts no
          arguments. To indicate more time and attempts are needed, the callable
          shall raise :py:class:`WaitingIncomplete`, otherwise it shall return
          without exception. Its return value will be propagated by ``wait()`` up
          to ``wait()``'s. All other exceptions raised by ``check`` will propagate
          to ``wait()``'s caller as well, terminating the wait.
      :param timeout: amount of time ``wait()`` is alowed to spend waiting for
          successfull outcome of ``check`` call.
      :param tick: how many seconds to wait between two consecutive calls of
          ``check``.
      :param tick_increase: a multiplier applied to ``tick`` after every attempt.
      :returns: value returned by ``check`` reporting success.
      :raises GeneralError: when ``tick`` is not a positive integer.
      :raises WaitingTimedOutError: when time quota has been consumed.
      """
  
      if tick <= 0:
          raise GeneralError('Tick must be a positive integer')
  
      NOW = time.monotonic
  
      deadline = NOW() + timeout.total_seconds()
  
      parent.debug(
          'wait',
          f"waiting for condition '{check.__name__}' with timeout {timeout},"
          f"deadline in {timeout.total_seconds()}, checking every {tick} seconds")
  
      while True:
          now = NOW()
  
          if now > deadline:
              raise WaitingTimedOutError()
  
          try:
              ret = check()
  
              # Perform one extra check: if `check()` succeeded, but took more time than
              # allowed, it should be recognized as a failed waiting too.
              now = NOW()
  
              if now > deadline:
                  parent.debug(
                      'wait',
                      f"'{check.__name__}' finished successfully but took too much time,"
                      f"{now - deadline} over quota")
  
                  raise WaitingTimedOutError()
  
              parent.debug(
                  'wait',
                  f"'{check.__name__}' finished successfully, {deadline - now} seconds left")
  
              return ret
  
          except WaitingIncomplete:
              parent.debug(
                  'wait',
                  f"'{check.__name__}' still pending, {deadline - now} seconds left")
  
              time.sleep(tick)
  
              tick *= tick_increase
  
              continue
  
  
  class ValidateFmfMixin:
      """
      Mixin adding validation of an fmf node.
  
      Loads a schema whose name is derived from class name, and uses fmf's validate()
      method to perform the validation.
      """
  
      def _validate_fmf_node(self, node: fmf.Tree, logger: Common,
                             raise_on_validation_error: bool) -> None:
          """ Validate a given fmf node """
  
          errors = validate_fmf_node(
              node, f'{self.__class__.__name__.lower()}.yaml')
  
          if errors:
              if raise_on_validation_error:
                  raise SpecificationError(
                      f'fmf node {node.name} failed validation',
                      validation_errors=errors)
  
              for _, error_message in errors:
                  logger.warn(error_message, shift=1)
  
      def __init__(
              self,
              *,
              node: fmf.Tree,
              logger: Common,
              skip_validation: bool = False,
              raise_on_validation_error: bool = False,
              **kwargs: Any) -> None:
          # Validate *before* letting next class in line touch the data.
          if not skip_validation:
              self._validate_fmf_node(node, logger, raise_on_validation_error)
  
          kwargs.setdefault('logger', self)
          super().__init__(node=node, **kwargs)  # type: ignore[call-arg]
  
  
  # A type representing compatible sources of keys and values.
  KeySource = Union[Dict[str, Any], fmf.Tree]
  
  
  class NormalizeKeysMixin:
      """
      Mixin adding support for loading fmf keys into object attributes.
  
      When invoked, annotated class-level variables are searched for in a given source
      container - a mapping, an fmf node, etc. - and if the key of the same name as the
      variable exists, its value is "promoted" to instance variable.
  
      If a method named ``_normalize_<variable name>`` exists, it is called with the fmf
      key value as its single argument, and its return value is assigned to instance
      variable. This gives class chance to modify or transform the original value when
      needed, e.g. to convert the original value to a type more suitable for internal
      processing.
      """
  
      # If specified, keys would be iterated over in the order as listed here.
      KEYS_SHOW_ORDER: List[str] = []
  
      # NOTE: these could be static methods, self is probably useless, but that would
      # cause complications when classes assign these to their members. That makes them
      # no longer static as far as class is concerned, which means they get called with
      # `self` as the first argument. A workaround would be to assign staticmethod()-ized
      # version of them, but that's too much repetition.
      #
      # TODO: wouldn't it be nice if these could be mention in dataclass.field()?
      # It would require a clone of dataclass.field() though.
      def _normalize_string_list(self, value: Union[None, str, List[str]]) -> List[str]:
          if value is None:
              return []
  
          return [value] if isinstance(value, str) else value
  
      def _normalize_environment(self, value: Optional[Dict[str, Any]]) -> EnvironmentType:
          if value is None:
              return {}
  
          return {
              name: str(value) for name, value in value.items()
              }
  
      @classmethod
      def _iter_key_annotations(cls) -> Generator[Tuple[str, Any], None, None]:
          """
          Iterate over keys' type annotations.
  
          Keys are yielded in the order: keys declared by parent classes first, then
          keys declared by the class itself, all following the order in which keys
          were defined in their respective classes.
  
          Yields:
              pairs of key name and its annotations.
          """
  
          def _iter_class_annotations(klass: type) -> Generator[Tuple[str, Any], None, None]:
              # Skip, needs fixes to become compatible
              if klass is Common:
                  return
  
              for name, value in klass.__dict__.get('__annotations__', {}).items():
                  # Skip special fields that are not keys.
                  if name == 'KEYS_SHOW_ORDER':
                      continue
  
                  yield (name, value)
  
          # Reverse MRO to start with the most base classes first, to iterate over keys
          # in the order they are defined.
          for klass in reversed(cls.__mro__):
              yield from _iter_class_annotations(klass)
  
      @classmethod
      def keys(cls) -> Generator[str, None, None]:
          """
          Iterate over key names.
  
          Keys are yielded in the order: keys declared by parent classes first, then
          keys declared by the class itself, all following the order in which keys
          were defined in their respective classes.
  
          Yields:
              key names.
          """
  
          for keyname, _ in cls._iter_key_annotations():
              yield keyname
  
      def items(self) -> Generator[Tuple[str, Any], None, None]:
          """
          Iterate over keys and their values.
  
          Keys are yielded in the order: keys declared by parent classes first, then
          keys declared by the class itself, all following the order in which keys
          were defined in their respective classes.
  
          Yields:
              pairs of key name and its value.
          """
  
          for keyname in self.keys():
              yield (keyname, getattr(self, keyname))
  
      # TODO: exists for backward compatibility for the transition period. Once full
      # type annotations land, there should be no need for extra _keys attribute.
      @classmethod
      def _keys(cls) -> List[str]:
          """ Return a list of names of object's keys. """
  
          return list(cls.keys())
  
      def _load_keys(
              self,
              key_source: Dict[str, Any],
              key_source_name: str,
              logger: Common) -> None:
          """ Extract values for class-level attributes, and verify they match declared types. """
  
          LOG_SHIFT, LOG_LEVEL = 2, 4
  
          debug_intro = functools.partial(logger.debug, shift=LOG_SHIFT - 1, level=LOG_LEVEL)
          debug = functools.partial(logger.debug, shift=LOG_SHIFT, level=LOG_LEVEL)
  
          debug_intro('key source')
          for k, v in key_source.items():
              debug(f'{k}: {v} ({type(v)})')
  
          debug('')
  
          for keyname, keytype in self._iter_key_annotations():
              key_address = f'{key_source_name}:{keyname}'
              source_keyname = key_to_option(keyname)
  
              # Do not indent this particular entry like the rest, so it could serve
              # as a "header" for a single key processing.
              debug_intro('key', key_address)
              debug('field', source_keyname)
  
              debug('desired type', str(keytype))
  
              value: Any = None
  
              # Verbose, let's hide it a bit deeper.
              debug('dict', self.__dict__, level=LOG_LEVEL + 1)
  
              if hasattr(self, keyname):
                  # If the key exists as instance's attribute already, it is because it's been
                  # declared with a default value, and the attribute now holds said default value.
                  default_value = getattr(self, keyname)
  
                  # If the default value is a mutable container, we cannot use it directly.
                  # Should we do so, the very same default value would be assigned to multiple
                  # instances/attributes instead of each instance having its own distinct container.
                  if isinstance(default_value, (list, dict)):
                      debug('detected mutable default')
                      default_value = copy.copy(default_value)
  
                  debug('default value', str(default_value))
                  debug('default value type', str(type(default_value)))
  
                  # try+except seems to work better than get(), especially when
                  # semantic of fmf.Tree.get() is slightly different than that
                  # of dict().get().
                  try:
                      value = key_source[source_keyname]
  
                  except KeyError:
                      value = default_value
  
                  debug('raw value', str(value))
                  debug('raw value type', str(type(value)))
  
              else:
                  value = key_source.get(source_keyname)
  
                  debug('raw value', str(value))
                  debug('raw value type', str(type(value)))
  
              normalize_callback = getattr(self, f'_normalize_{keyname}', None)
  
              if normalize_callback:
                  value = normalize_callback(value)
  
                  debug('normalized value', str(value))
                  debug('normalized value type', str(type(value)))
  
              debug('final value', str(value))
              debug('final value type', str(type(value)))
  
              # Set attribute by adding it to __dict__ directly. Messing with setattr()
              # might cause re-use of mutable values by other instances.
              self.__dict__[keyname] = value
  
              # Apparently pointless, but makes the debugging output more readable.
              # There may be plenty of tests and plans and keys, a bit of spacing
              # can't hurt.
              debug('')
  
          debug_intro('normalized fields')
          for k, v in self.__dict__.items():
              debug(f'{k}: {v} ({type(v)})')
  
          debug('')
  
      def __init__(self, **kwargs: Any) -> None:
          super().__init__(**kwargs)
  
  
  class LoadFmfKeysMixin(NormalizeKeysMixin):
      def __init__(
              self,
              *,
              node: fmf.Tree,
              logger: Common,
              **kwargs: Any) -> None:
          self._load_keys(node.get(), node.name, logger)
  
          kwargs.setdefault('logger', logger)
          super().__init__(node=node, **kwargs)
  
files: tmt/utils.py