files: 
  """ Step Classes """
  
  import re
  import sys
  import textwrap
  import warnings
  from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type,
                      TypeVar, Union, cast, overload)
  
  if sys.version_info >= (3, 8):
      from typing import TypedDict
  else:
      from typing_extensions import TypedDict
  
  import click
  from click import echo
  
  import tmt.options
  import tmt.utils
  from tmt.options import show_step_method_hints
  
  if TYPE_CHECKING:
      import tmt.base
      from tmt.base import Plan
      from tmt.steps.provision import Guest
  
  
  DEFAULT_PLUGIN_METHOD_ORDER: int = 50
  
  
  # Supported steps and actions
  STEPS = ['discover', 'provision', 'prepare', 'execute', 'report', 'finish']
  ACTIONS = ['login', 'reboot']
  
  # Step phase order
  PHASE_START = 10
  PHASE_BASE = 50
  PHASE_END = 90
  
  
  class Phase(tmt.utils.Common):
      """ A phase of a step """
  
      def __init__(
              self,
              order: int = tmt.utils.DEFAULT_PLUGIN_ORDER,
              *args: Any,
              **kwargs: Any):
          super().__init__(*args, **kwargs)
          self.order: int = order
  
      def enabled_on_guest(self, guest: 'Guest') -> bool:
          """ Phases are enabled across all guests by default """
          return True
  
      @property
      def is_in_standalone_mode(self) -> bool:
          """
          True if the phase is in stand-alone mode.
  
          Stand-alone mode means that only this phase should be run as a part
          of the run (and not any other even if requested using --all).
          This is useful as some plugin options may completely change its
          behaviour from the regular behaviour based on options
          (e.g. listing images inside a provision plugin).
          """
          return False
  
  
  # A variable used to describe a generic type for all classes derived from Phase
  PhaseT = TypeVar('PhaseT', bound=Phase)
  
  
  class StepData(TypedDict, total=False):
      """
      Step data structure
      """
      name: str
      how: str
      order: Optional[int]
      tests: Optional[List['tmt.base.Test']]
  
  
  class Step(tmt.utils.Common):
      """ Common parent of all test steps """
  
      # Default implementation for all steps is "shell", but some
      # steps like provision may have better defaults for their
      # area of expertise.
      DEFAULT_HOW: str = 'shell'
  
      def __init__(
              self,
              plan: 'Plan',
              data: Optional[Union[StepData, List[StepData]]] = None,
              name: Optional[str] = None,
              workdir: tmt.utils.WorkdirArgumentType = None) -> None:
          """ Initialize and check the step data """
          super().__init__(name=name, parent=plan, workdir=workdir)
          # Initialize data
          self.plan: 'Plan' = plan
          self._status: Optional[str] = None
          self._phases: List[Phase] = []
  
          # Normalize `data` attribute to carry a list of step configuration data, one item per
          # distinct step configuration. Make sure all items have `name`` and `how` keys.
          self.data: List[StepData] = []
  
          # Create an empty step by default (can be updated from cli)
          if data is None:
              self.data = [{}]
  
          # Convert to list if only a single config provided
          elif isinstance(data, dict):
              self.data = [data]
  
          # List is as good as it gets
          elif isinstance(data, list):
              self.data = data
  
          # Shout about invalid configuration
          else:
              raise tmt.utils.GeneralError(
                  f"Invalid '{self}' config in '{self.plan}'.")
  
          # Final sanity checks
          for i, single_config_data in enumerate(self.data):
              # Add default unique names even to multiple configs so that the users
              # don't need to specify it if they don't care about the name
              if 'name' not in single_config_data:
                  single_config_data['name'] = f'{tmt.utils.DEFAULT_NAME}-{i}'
  
              # Set 'how' to the default if not specified
              if single_config_data.get('how') is None:
                  single_config_data['how'] = self.DEFAULT_HOW
  
      @property
      def enabled(self) -> Optional[bool]:
          """ True if the step is enabled """
          try:
              return self.name in self.plan.my_run._context.obj.steps
          except AttributeError:
              return None
  
      @property
      def plugins_in_standalone_mode(self) -> int:
          """
          The number of plugins in standalone mode.
  
          Stand-alone mode means that only this step should be run as a part
          of the run (and not any other even if requested using --all).
          This is useful as some step options may completely change its
          behaviour from the regular behaviour based on options
          (e.g. listing images inside provision).
          """
          return sum(phase.is_in_standalone_mode for phase in self.phases())
  
      @classmethod
      def usage(cls, method_overview: str) -> str:
          """ Prepare general usage message for the step """
          # Main description comes from the class docstring
          if cls.__name__ is None:
              raise tmt.utils.GeneralError("Missing name of the step.")
  
          if cls.__doc__ is None:
              raise tmt.utils.GeneralError(
                  f"Missing docstring of the step {cls.__name__.lower()}.")
  
          usage = textwrap.dedent(cls.__doc__)
          # Append the list of supported methods
          usage += '\n\n' + method_overview
          # Give a hint about detailed help
          name = cls.__name__.lower()
          usage += (
              f"\n\nUse 'tmt run {name} --help --how <method>' to learn more "
              f"about given {name} method and all its supported options.")
          return usage
  
      def status(self, status: Optional[str] = None) -> Optional[str]:
          """
          Get and set current step status
  
          The meaning of the status is as follows:
          todo ... config, data and command line processed (we know what to do)
          done ... the final result of the step stored to workdir (we are done)
          """
          # Update status
          if status is not None:
              # Check for valid values
              if status not in ['todo', 'done']:
                  raise tmt.utils.GeneralError(f"Invalid status '{status}'.")
              # Show status only if changed
              elif self._status != status:
                  self._status = status
                  self.debug('status', status, color='yellow', level=2)
          # Return status
          return self._status
  
      def load(self) -> None:
          """ Load status and step data from the workdir """
          try:
              content: Dict[Any, Any] = tmt.utils.yaml_to_dict(
                  self.read('step.yaml'))
              self.debug('Successfully loaded step data.', level=2)
              self.data = content['data']
              self.status(content['status'])
          except tmt.utils.GeneralError:
              self.debug('Step data not found.', level=2)
  
      def save(self) -> None:
          """ Save status and step data to the workdir """
          content: Dict[str, Any] = {
              'status': self.status(),
              'data': self.data
              }
          self.write('step.yaml', tmt.utils.dict_to_yaml(content))
  
      def wake(self) -> None:
          """ Wake up the step (process workdir and command line) """
          # Cleanup possible old workdir if called with --force
          if self.opt('force'):
              self._workdir_cleanup()
  
          # Load stored data
          self.load()
  
          # Status 'todo' means the step has not finished successfully.
          # Probably interrupted in the middle. Clean up the work
          # directory to give it another chance with a fresh start.
          if self.status() == 'todo':
              self.debug("Step has not finished. Let's try once more!", level=2)
              self._workdir_cleanup()
  
          # Importing here to avoid circular imports
          import tmt.steps.report
  
          # Special handling for the report step to always enable force mode in
          # order to cover a very frequent use case 'tmt run --last report'
          # FIXME find a better way how to enable always-force per plugin
          if (isinstance(self, tmt.steps.report.Report) and
                  self.data[0].get('how') in ['display', 'html']):
              self.debug("Report step always force mode enabled.")
              self._workdir_cleanup()
              self.status('todo')
  
          # Nothing more to do when the step is already done
          if self.status() == 'done':
              return
  
          # Override step data with command line options
          how: str = self.opt('how')
          if how is not None:
              for data in self.data:
                  assert isinstance(data, dict)
                  data['how'] = how
  
      def setup_actions(self) -> None:
          """ Insert login and reboot plugins if requested """
          for login_plugin in Login.plugins(step=self):
              self.debug(
                  f"Insert a login plugin into the '{self}' step "
                  f"with order '{login_plugin.order}'.", level=2)
              self._phases.append(login_plugin)
  
          for reboot_plugin in Reboot.plugins(step=self):
              self.debug(
                  f"Insert a reboot plugin into the '{self}' step "
                  f"with order '{reboot_plugin.order}'.", level=2)
              self._phases.append(reboot_plugin)
  
      @overload
      def phases(self, classes: None = None) -> List[Phase]:
          pass
  
      @overload
      def phases(self, classes: Type[PhaseT]) -> List[PhaseT]:
          pass
  
      @overload
      def phases(self, classes: Tuple[Type[PhaseT], ...]) -> List[PhaseT]:
          pass
  
      def phases(self, classes: Optional[Union[Type[PhaseT],
                 Tuple[Type[PhaseT], ...]]] = None) -> List[PhaseT]:
          """
          Iterate over phases by their order
  
          By default iterates over all available phases. Optional filter
          'classes' can be used to iterate only over instances of given
          class (single class or tuple of classes).
          """
  
          if classes is None:
              _classes: Tuple[Union[Type[Phase], Type[PhaseT]], ...] = (Phase,)
  
          elif not isinstance(classes, tuple):
              _classes = (classes,)
  
          else:
              _classes = classes
  
          return sorted(
              [cast(PhaseT, phase) for phase in self._phases if isinstance(phase, _classes)],
              key=lambda phase: phase.order)
  
      def actions(self) -> None:
          """ Run all loaded Login or Reboot action instances of the step """
          for phase in self.phases(classes=Action):
              phase.go()
  
      def go(self) -> None:
          """ Execute the test step """
          # Show step header and how
          self.info(self.name, color='blue')
          # Show workdir in verbose mode
          self.debug('workdir', self.workdir, 'magenta')
  
  
  class Method:
      """ Step implementation method """
  
      def __init__(
              self,
              name: str,
              class_: Optional[Type['BasePlugin']] = None,
              doc: Optional[str] = None,
              order: int = DEFAULT_PLUGIN_METHOD_ORDER
              ) -> None:
          """ Store method data """
  
          doc = (doc or getattr(class_, '__doc__') or '').strip()
  
          if not doc:
              if class_:
                  raise tmt.utils.GeneralError(f"Plugin class '{class_}' provides no docstring.")
  
              raise tmt.utils.GeneralError(f"Plugin method '{name}' provides no docstring.")
  
          self.name = name
          self.class_ = class_
          self.doc = doc
          self.order = order
  
          # Parse summary and description from provided doc string
          lines: List[str] = [re.sub('^    ', '', line)
                              for line in self.doc.split('\n')]
          self.summary: str = lines[0].strip()
          self.description: str = '\n'.join(lines[1:]).lstrip()
  
      def describe(self) -> str:
          """ Format name and summary for a nice method overview """
          return f'{self.name} '.ljust(22, '.') + f' {self.summary}'
  
      def usage(self) -> str:
          """ Prepare a detailed usage from summary and description """
          if self.description:
              usage: str = self.summary + '\n\n' + self.description
          else:
              usage = self.summary
          # Disable wrapping for all paragraphs
          return re.sub('\n\n', '\n\n\b\n', usage)
  
  
  class PluginData(TypedDict, total=False):
      """ Step data structure """
      name: Optional[str]
      how: Optional[str]
      order: Optional[int]
  
  
  def provides_method(
          name: str,
          doc: Optional[str] = None,
          order: int = DEFAULT_PLUGIN_METHOD_ORDER) -> Any:
      """
      A plugin class decorator to register plugin's method with tmt steps.
  
      In the following example, developer marks ``SomePlugin`` as providing two discover methods,
      ``foo`` and ``bar``, with ``bar`` being sorted to later position among methods:
  
      .. code-block:: python
  
         @tmt.steps.provides_method('foo')
         @tmt.steps.provides_method('bar', order=80)
         class SomePlugin(tmt.steps.discover.DicoverPlugin):
           ...
  
      :param name: name of the method.
      :param doc: method documentation. If not specified, docstring of the decorated class is used.
      :param order: order of the method among other step methods.
      """
  
      def _method(cls: Type['BasePlugin']) -> Any:
          plugin_method = Method(name, class_=cls, doc=doc, order=order)
  
          cast('BasePlugin', cls.__bases__[0])._supported_methods.append(plugin_method)
  
          return cls
  
      return _method
  
  
  class PluginIndex(type):
      """
      Plugin metaclass used to register all available plugins.
  
      .. deprecated:: 1.17
         Instead of declaring `_methods` list, use :py:func:`provides_method` decorator
         to announce provided methods.
      """
  
      def __init__(
              cls,
              name: str,
              bases: List['BasePlugin'],
              attributes: Any) -> None:
          """ Store all defined methods in the parent class """
  
          plugin_methods = cast(Optional[List[Method]], getattr(cls, '_methods'))
  
          if not plugin_methods:
              return
  
          warnings.warn(
              'Use of _methods to declare plugin methods is deprecated.'
              'Use tmt.steps.provides_method decorator instead.',
              DeprecationWarning,
              stacklevel=2)
  
          for plugin_method in plugin_methods:
              plugin_method.class_ = cast(Type['BasePlugin'], cls)
              # Add to the list of supported methods in parent class
              bases[0]._supported_methods.append(plugin_method)
  
  
  class BasePlugin(Phase, metaclass=PluginIndex):
      """ Common parent of all step plugins """
  
      # Deprecated, use @provides_method(...) instead. left for backward
      # compatibility with out-of-tree plugins.
      _methods: List[Method] = []
  
      # Default implementation for all steps is shell
      # except for provision (virtual) and report (display)
      how: str = 'shell'
  
      # Common keys for all plugins of given step
      _common_keys: List[str] = []
  
      # Keys specific for given plugin
      _keys: List[str] = []
  
      # List of all supported methods aggregated from all plugins of the same step.
      _supported_methods: List[Method] = []
  
      def __init__(
              self,
              step: Step,
              data: StepData,
              workdir: tmt.utils.WorkdirArgumentType = None) -> None:
          """ Store plugin name, data and parent step """
  
          # Ensure that plugin data contains name
          if 'name' not in data:
              raise tmt.utils.GeneralError(
                  f"Missing 'name' in the {step} step config "
                  f"of the '{step.plan}' plan.")
  
          # Initialize plugin order
          if 'order' not in data or data['order'] is None:
              order = tmt.utils.DEFAULT_PLUGIN_ORDER
          else:
              try:
                  order = int(data['order'])
              except ValueError:
                  raise tmt.utils.SpecificationError(
                      f"Invalid order '{data['order']}' in {step} config "
                      f"'{data['name']}'. Should be an integer.")
  
          # Store name, data and parent step
          super().__init__(
              parent=step,
              name=data['name'],
              workdir=workdir,
              order=order)
          # It is not possible to use TypedDict here because
          # all keys are not known at the time of the class definition
          self.data = data
          self.step = step
  
      @classmethod
      def base_command(
              cls,
              usage: str,
              method_class: Optional[Type[click.Command]] = None) -> click.Command:
          """ Create base click command (common for all step plugins) """
          raise NotImplementedError
  
      @classmethod
      def options(cls, how: Optional[str] = None) -> List[tmt.options.ClickOptionDecoratorType]:
          """ Prepare command line options for given method """
          # Include common options supported across all plugins
          return tmt.options.verbose_debug_quiet + tmt.options.force_dry
  
      @classmethod
      def command(cls) -> click.Command:
          """ Prepare click command for all supported methods """
          # Create one command for each supported method
          commands: Dict[str, click.Command] = {}
          method_overview: str = f'Supported methods ({cls.how} by default):\n\n\b'
          for method in cls.methods():
              assert method.class_ is not None
              method_overview += f'\n{method.describe()}'
              command: click.Command = cls.base_command(usage=method.usage())
              # Apply plugin specific options
              for option in method.class_.options(method.name):
                  command = option(command)
              commands[method.name] = command
  
          # Create base command with common options using method class
          method_class = tmt.options.create_method_class(commands)
          command = cls.base_command(usage=method_overview, method_class=method_class)
          # Apply common options
          for option in cls.options():
              command = option(command)
          return command
  
      @classmethod
      def methods(cls) -> List[Method]:
          """ Return all supported methods ordered by priority """
          return sorted(cls._supported_methods, key=lambda method: method.order)
  
      @classmethod
      def delegate(
              cls,
              step: Step,
              data: StepData) -> 'BasePlugin':
          """
          Return plugin instance implementing the data['how'] method
  
          Supports searching by method prefix as well (e.g. 'virtual').
          The first matching method with the lowest 'order' wins.
          """
          # Filter matching methods, pick the one with the lowest order
          for method in cls.methods():
              assert method.class_ is not None
              if method.name.startswith(data['how']):
                  step.debug(
                      f"Using the '{method.class_.__name__}' plugin "
                      f"for the '{data['how']}' method.", level=2)
                  plugin = method.class_(step, data)
                  assert isinstance(plugin, BasePlugin)
                  return plugin
  
          show_step_method_hints(step, step.name, data['how'])
          # Report invalid method
          if step.plan is None:
              raise tmt.utils.GeneralError(f"Plan for {step.name} is not set.")
          raise tmt.utils.SpecificationError(
              f"Unsupported {step.name} method '{data['how']}' "
              f"in the '{step.plan.name}' plan.")
  
      def default(self, option: str, default: Optional[Any] = None) -> Any:
          """ Return default data for given option """
          return default
  
      def get(self, option: str, default: Optional[Any] = None) -> Any:
          """ Get option from plugin data, user/system config or defaults """
          # Check plugin data first
          try:
              # FIXME Enable type check once StepData defined more precisely
              return self.data[option]  # type: ignore
          except KeyError:
              pass
  
          # Check user config and system config
          # TODO
  
          # Finally check plugin defaults
          return self.default(option, default)
  
      def show(self, keys: Optional[List[str]] = None) -> None:
          """ Show plugin details for given or all available keys """
          # Show empty config with default method only in verbose mode
          if (set(self.data.keys()) == set(['how', 'name'])
                  and not self.opt('verbose')
                  and self.data['how'] == self.how):
              return
          # Step name (and optional summary)
          echo(tmt.utils.format(
              self.step.name, self.get('summary', ''),
              key_color='blue', value_color='blue'))
          # Show all or requested step attributes
          base_keys = ['name', 'how']
          if keys is None:
              keys = self._common_keys + self._keys
          for key in base_keys + keys:
              # Skip showing the default name
              if key == 'name' and self.name.startswith(tmt.utils.DEFAULT_NAME):
                  continue
              # Skip showing summary again
              if key == 'summary':
                  continue
              value = self.get(key)
              if value is not None:
                  echo(tmt.utils.format(key, value))
  
      def enabled_on_guest(self, guest: 'Guest') -> bool:
          """ Check if the plugin is enabled on the specific guest """
          where: str = self.get('where')
          if not where:
              return True
          return where in (guest.name, guest.role)
  
      def wake(self) -> None:
          """
          Wake up the plugin, process data, apply options
  
          Check command line options corresponding to plugin keys
          and store their value into the 'self.data' dictionary if
          their value is True or non-empty.
  
          By default, all supported options corresponding to common
          and plugin-specific keys are processed. List of key names
          in the 'keys' parameter can be used to override only
          selected ones.
          """
          keys = self._common_keys + self._keys
  
          for key in keys:
              value = self.opt(key)
              if value:
                  # FIXME Enable type check once StepData defined more precisely
                  self.data[key] = value  # type: ignore
  
      # NOTE: it's tempting to rename this method to `go()` and use more natural
      # `super().go()` in child classes' `go()` methods. But, `go()` does not have
      # the same signature across all plugin types, therefore we cannot have shared
      # `go()` method in superclass - overriding it in (some) child classes would
      # raise a typing linter error reporting superclass signature differs from the
      # one in a subclass.
      #
      # Therefore we need a different name, and a way how not to forget to call this
      # method from child classes.
      def go_prolog(self) -> None:
          """ Perform actions shared among plugins when beginning their tasks """
          # Show the method
          self.info('how', self.get('how'), 'magenta')
          # Give summary if provided
          if self.get('summary'):
              self.info('summary', self.get('summary'), 'magenta')
          # Show name only if it's not the default one
          if not self.name.startswith(tmt.utils.DEFAULT_NAME):
              self.info('name', self.name, 'magenta')
          # Include order in verbose mode
          self.verbose('order', str(self.order), 'magenta', level=3)
  
      def requires(self) -> List[str]:
          """ List of packages required by the plugin on the guest """
          return []
  
  
  class GuestlessPlugin(BasePlugin):
      """ Common parent of all step plugins that do not work against a particular guest """
  
      def go(self) -> None:
          """ Perform actions shared among plugins when beginning their tasks """
  
          self.go_prolog()
  
  
  class Plugin(BasePlugin):
      """ Common parent of all step plugins that do work against a particular guest """
  
      def go(self, guest: 'Guest') -> None:
          """ Perform actions shared among plugins when beginning their tasks """
  
          self.go_prolog()
  
  
  class Action(Phase):
      """ A special action performed during a normal step. """
  
      # Dictionary containing list of requested phases for each enabled step
      _phases: Optional[Dict[str, List[int]]] = None
  
      @classmethod
      def phases(cls, step: Step) -> List[int]:
          """ Return list of phases enabled for given step """
          # Build the phase list unless done before
          if cls._phases is None:
              cls._phases = cls._parse_phases(step)
          # Return enabled phases, empty list if step not found
          try:
              return cls._phases[step.name]
          except KeyError:
              return []
  
      @classmethod
      def _parse_phases(cls, step: Step) -> Dict[str, List[int]]:
          """ Parse options and store phase order """
          phases = dict()
          options: List[str] = cls._opt('step', default=[])
  
          # Use the end of the last enabled step if no --step given
          if not options:
              login_during: Optional[Step] = None
              # The last run may have failed before all enabled steps were
              # completed, select the last step done
              if step.plan is None:
                  raise tmt.utils.GeneralError(
                      f"Plan for {step.name} is not set.")
              if step.plan.my_run.opt('last'):
                  steps: List[Step] = [
                      s for s in step.plan.steps() if s.status() == 'done']
                  login_during = steps[-1] if steps else None
              # Default to the last enabled step if no completed step found
              if login_during is None:
                  login_during = list(step.plan.steps())[-1]
              # Only login if the error occurred after provision
              if login_during != step.plan.discover:
                  phases[login_during.name] = [PHASE_END]
  
          # Process provided options
          for option in options:
              # Parse the step:phase format
              matched = re.match(r'(\w+)(:(\w+))?', option)
              if matched:
                  step_name, _, phase = matched.groups()
              if not matched or step_name not in STEPS:
                  raise tmt.utils.GeneralError(f"Invalid step '{option}'.")
              # Check phase format, convert into int, use end by default
              try:
                  phase = int(phase)
              except TypeError:
                  phase = PHASE_END
              except ValueError:
                  # Convert 'start' and 'end' aliases
                  try:
                      phase = cast(Dict[str, int],
                                   dict(start=PHASE_START, end=PHASE_END))[phase]
                  except KeyError:
                      raise tmt.utils.GeneralError(f"Invalid phase '{phase}'.")
              # Store the phase for given step
              try:
                  phases[step_name].append(phase)
              except KeyError:
                  phases[step_name] = [phase]
          return phases
  
      def go(self) -> None:
          raise NotImplementedError()
  
  
  class Reboot(Action):
      """ Reboot guest """
  
      # True if reboot enabled
      _enabled: bool = False
  
      def __init__(self, step: Step, order: int) -> None:
          """ Initialize relations, store the reboot order """
          super().__init__(parent=step, name='reboot', order=order)
  
      @classmethod
      def command(
              cls,
              method_class: Optional[Method] = None,
              usage: Optional[str] = None) -> click.Command:
          """ Create the reboot command """
          @click.command()
          @click.pass_context
          @click.option(
              '-s', '--step', metavar='STEP[:PHASE]', multiple=True,
              help='Reboot machine during given phase of selected step(s).')
          @click.option(
              '--hard', is_flag=True,
              help='Hard reboot of the machine. Unsaved data may be lost.')
          def reboot(context: Any, **kwargs: Any) -> None:
              """ Reboot the guest. """
              Reboot._save_context(context)
              Reboot._enabled = True
  
          return reboot
  
      @classmethod
      def plugins(cls, step: Step) -> List['Reboot']:
          """ Return list of reboot instances for given step """
          if not Reboot._enabled:
              return []
          return [Reboot(step, phase) for phase in cls.phases(step)]
  
      def go(self) -> None:
          """ Reboot the guest(s) """
          self.info('reboot', 'Rebooting guest', color='yellow')
          assert isinstance(self.parent, Step)
          assert hasattr(self.parent, 'plan') and self.parent.plan is not None
          for guest in self.parent.plan.provision.guests():
              guest.reboot(hard=self.opt('hard'))
          self.info('reboot', 'Reboot finished', color='yellow')
  
  
  class Login(Action):
      """ Log into the guest """
  
      # True if interactive login enabled
      _enabled: bool = False
  
      def __init__(self, step: Step, order: int):
          """ Initialize relations, store the login order """
          super().__init__(parent=step, name='login', order=order)
  
      @classmethod
      def command(
              cls,
              method_class: Optional[Method] = None,
              usage: Optional[str] = None) -> click.Command:
          """ Create the login command """
          @click.command()
          @click.pass_context
          @click.option(
              '-s', '--step', metavar='STEP[:PHASE]', multiple=True,
              help='Log in during given phase of selected step(s).')
          @click.option(
              '-w', '--when', metavar='RESULT', multiple=True,
              type=click.Choice(tmt.base.Result._results),
              help='Log in if a test finished with given result(s).')
          @click.option(
              '-c', '--command', metavar='COMMAND',
              multiple=True, default=['bash'],
              help="Run given command(s). Default is 'bash'.")
          def login(context: Any, **kwargs: Any) -> None:
              """
              Provide user with an interactive shell on the guest.
  
              By default the shell is provided at the end of the last
              enabled step. When used together with the --last option the
              last completed step is selected. Use one or more --step
              options to select a different step instead.
  
              Optional phase can be provided to specify the exact phase of
              the step when the shell should be provided. The following
              values are supported:
  
              \b
                  start ... beginning of the step (same as '10')
                  end ..... end of the step (default, same as '90')
                  00-99 ... integer order defining the exact phase
  
              Usually the main step execution happens with order 50.
              Consult individual step documentation for more details.
  
              For the execute step and following steps it is also possible
              to conditionally enable the login feature only if some of
              the tests finished with given result (pass, info, fail,
              warn, error).
              """
              Login._save_context(context)
              Login._enabled = True
  
          return login
  
      @classmethod
      def plugins(cls, step: Step) -> List['Login']:
          """ Return list of login instances for given step """
          if not Login._enabled:
              return []
          return [Login(step, phase) for phase in cls.phases(step)]
  
      def go(self) -> None:
          """ Login to the guest(s) """
          # Verify possible test result condition
          count: Dict[str, int] = dict()
          expected_results: Optional[List[str]] = self.opt('when')
  
          assert isinstance(self.parent, Step)
          assert hasattr(self.parent, 'plan') and self.parent.plan is not None
  
          if expected_results:
              for expected_result in expected_results:
                  count[expected_result] = len([
                      result for result in self.parent.plan.execute.results()
                      if result.result == expected_result])
              if not any(count.values()):
                  self.info('Skipping interactive shell', color='yellow')
                  return
  
          # Run the interactive command
          commands: List[str] = self.opt('command')
          self.info('login', 'Starting interactive shell', color='yellow')
          for guest in self.parent.plan.provision.guests():
              # Attempt to push the workdir to the guest
              try:
                  guest.push()
                  cwd: Optional[str] = self.parent.plan.worktree
              except tmt.utils.GeneralError:
                  self.warn("Failed to push workdir to the guest.")
                  cwd = None
              # Execute all requested commands
              for command in commands:
                  self.debug(f"Run '{command}' in interactive mode.")
                  guest.execute(command, interactive=True, cwd=cwd)
          self.info('login', 'Interactive shell finished', color='yellow')
  
files: tmt/steps/__init__.py
files: 
  """ Step Classes """
  
  import re
  import sys
  import textwrap
  import warnings
  from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type,
                      TypeVar, Union, cast, overload)
  
  if sys.version_info >= (3, 8):
      from typing import TypedDict
  else:
      from typing_extensions import TypedDict
  
  import click
  from click import echo
  
  import tmt.options
  import tmt.utils
  from tmt.options import show_step_method_hints
  
  if TYPE_CHECKING:
      import tmt.base
      from tmt.base import Plan
      from tmt.steps.provision import Guest
  
  
  DEFAULT_PLUGIN_METHOD_ORDER: int = 50
  
  
  # Supported steps and actions
  STEPS = ['discover', 'provision', 'prepare', 'execute', 'report', 'finish']
  ACTIONS = ['login', 'reboot']
  
  # Step phase order
  PHASE_START = 10
  PHASE_BASE = 50
  PHASE_END = 90
  
  
  class Phase(tmt.utils.Common):
      """ A phase of a step """
  
      def __init__(
              self,
              order: int = tmt.utils.DEFAULT_PLUGIN_ORDER,
              *args: Any,
              **kwargs: Any):
          super().__init__(*args, **kwargs)
          self.order: int = order
  
      def enabled_on_guest(self, guest: 'Guest') -> bool:
          """ Phases are enabled across all guests by default """
          return True
  
      @property
      def is_in_standalone_mode(self) -> bool:
          """
          True if the phase is in stand-alone mode.
  
          Stand-alone mode means that only this phase should be run as a part
          of the run (and not any other even if requested using --all).
          This is useful as some plugin options may completely change its
          behaviour from the regular behaviour based on options
          (e.g. listing images inside a provision plugin).
          """
          return False
  
  
  # A variable used to describe a generic type for all classes derived from Phase
  PhaseT = TypeVar('PhaseT', bound=Phase)
  
  
  class StepData(TypedDict, total=False):
      """
      Step data structure
      """
      name: str
      how: str
      order: Optional[int]
      tests: Optional[List['tmt.base.Test']]
  
  
  class Step(tmt.utils.Common):
      """ Common parent of all test steps """
  
      # Default implementation for all steps is "shell", but some
      # steps like provision may have better defaults for their
      # area of expertise.
      DEFAULT_HOW: str = 'shell'
  
      def __init__(
              self,
              plan: 'Plan',
              data: Optional[Union[StepData, List[StepData]]] = None,
              name: Optional[str] = None,
              workdir: tmt.utils.WorkdirArgumentType = None) -> None:
          """ Initialize and check the step data """
          super().__init__(name=name, parent=plan, workdir=workdir)
          # Initialize data
          self.plan: 'Plan' = plan
          self._status: Optional[str] = None
          self._phases: List[Phase] = []
  
          # Normalize `data` attribute to carry a list of step configuration data, one item per
          # distinct step configuration. Make sure all items have `name`` and `how` keys.
          self.data: List[StepData] = []
  
          # Create an empty step by default (can be updated from cli)
          if data is None:
              self.data = [{}]
  
          # Convert to list if only a single config provided
          elif isinstance(data, dict):
              self.data = [data]
  
          # List is as good as it gets
          elif isinstance(data, list):
              self.data = data
  
          # Shout about invalid configuration
          else:
              raise tmt.utils.GeneralError(
                  f"Invalid '{self}' config in '{self.plan}'.")
  
          # Final sanity checks
          for i, single_config_data in enumerate(self.data):
              # Add default unique names even to multiple configs so that the users
              # don't need to specify it if they don't care about the name
              if 'name' not in single_config_data:
                  single_config_data['name'] = f'{tmt.utils.DEFAULT_NAME}-{i}'
  
              # Set 'how' to the default if not specified
              if single_config_data.get('how') is None:
                  single_config_data['how'] = self.DEFAULT_HOW
  
      @property
      def enabled(self) -> Optional[bool]:
          """ True if the step is enabled """
          try:
              return self.name in self.plan.my_run._context.obj.steps
          except AttributeError:
              return None
  
      @property
      def plugins_in_standalone_mode(self) -> int:
          """
          The number of plugins in standalone mode.
  
          Stand-alone mode means that only this step should be run as a part
          of the run (and not any other even if requested using --all).
          This is useful as some step options may completely change its
          behaviour from the regular behaviour based on options
          (e.g. listing images inside provision).
          """
          return sum(phase.is_in_standalone_mode for phase in self.phases())
  
      @classmethod
      def usage(cls, method_overview: str) -> str:
          """ Prepare general usage message for the step """
          # Main description comes from the class docstring
          if cls.__name__ is None:
              raise tmt.utils.GeneralError("Missing name of the step.")
  
          if cls.__doc__ is None:
              raise tmt.utils.GeneralError(
                  f"Missing docstring of the step {cls.__name__.lower()}.")
  
          usage = textwrap.dedent(cls.__doc__)
          # Append the list of supported methods
          usage += '\n\n' + method_overview
          # Give a hint about detailed help
          name = cls.__name__.lower()
          usage += (
              f"\n\nUse 'tmt run {name} --help --how <method>' to learn more "
              f"about given {name} method and all its supported options.")
          return usage
  
      def status(self, status: Optional[str] = None) -> Optional[str]:
          """
          Get and set current step status
  
          The meaning of the status is as follows:
          todo ... config, data and command line processed (we know what to do)
          done ... the final result of the step stored to workdir (we are done)
          """
          # Update status
          if status is not None:
              # Check for valid values
              if status not in ['todo', 'done']:
                  raise tmt.utils.GeneralError(f"Invalid status '{status}'.")
              # Show status only if changed
              elif self._status != status:
                  self._status = status
                  self.debug('status', status, color='yellow', level=2)
          # Return status
          return self._status
  
      def load(self) -> None:
          """ Load status and step data from the workdir """
          try:
              content: Dict[Any, Any] = tmt.utils.yaml_to_dict(
                  self.read('step.yaml'))
              self.debug('Successfully loaded step data.', level=2)
              self.data = content['data']
              self.status(content['status'])
          except tmt.utils.GeneralError:
              self.debug('Step data not found.', level=2)
  
      def save(self) -> None:
          """ Save status and step data to the workdir """
          content: Dict[str, Any] = {
              'status': self.status(),
              'data': self.data
              }
          self.write('step.yaml', tmt.utils.dict_to_yaml(content))
  
      def wake(self) -> None:
          """ Wake up the step (process workdir and command line) """
          # Cleanup possible old workdir if called with --force
          if self.opt('force'):
              self._workdir_cleanup()
  
          # Load stored data
          self.load()
  
          # Status 'todo' means the step has not finished successfully.
          # Probably interrupted in the middle. Clean up the work
          # directory to give it another chance with a fresh start.
          if self.status() == 'todo':
              self.debug("Step has not finished. Let's try once more!", level=2)
              self._workdir_cleanup()
  
          # Importing here to avoid circular imports
          import tmt.steps.report
  
          # Special handling for the report step to always enable force mode in
          # order to cover a very frequent use case 'tmt run --last report'
          # FIXME find a better way how to enable always-force per plugin
          if (isinstance(self, tmt.steps.report.Report) and
                  self.data[0].get('how') in ['display', 'html']):
              self.debug("Report step always force mode enabled.")
              self._workdir_cleanup()
              self.status('todo')
  
          # Nothing more to do when the step is already done
          if self.status() == 'done':
              return
  
          # Override step data with command line options
          how: str = self.opt('how')
          if how is not None:
              for data in self.data:
                  assert isinstance(data, dict)
                  data['how'] = how
  
      def setup_actions(self) -> None:
          """ Insert login and reboot plugins if requested """
          for login_plugin in Login.plugins(step=self):
              self.debug(
                  f"Insert a login plugin into the '{self}' step "
                  f"with order '{login_plugin.order}'.", level=2)
              self._phases.append(login_plugin)
  
          for reboot_plugin in Reboot.plugins(step=self):
              self.debug(
                  f"Insert a reboot plugin into the '{self}' step "
                  f"with order '{reboot_plugin.order}'.", level=2)
              self._phases.append(reboot_plugin)
  
      @overload
      def phases(self, classes: None = None) -> List[Phase]:
          pass
  
      @overload
      def phases(self, classes: Type[PhaseT]) -> List[PhaseT]:
          pass
  
      @overload
      def phases(self, classes: Tuple[Type[PhaseT], ...]) -> List[PhaseT]:
          pass
  
      def phases(self, classes: Optional[Union[Type[PhaseT],
                 Tuple[Type[PhaseT], ...]]] = None) -> List[PhaseT]:
          """
          Iterate over phases by their order
  
          By default iterates over all available phases. Optional filter
          'classes' can be used to iterate only over instances of given
          class (single class or tuple of classes).
          """
  
          if classes is None:
              _classes: Tuple[Union[Type[Phase], Type[PhaseT]], ...] = (Phase,)
  
          elif not isinstance(classes, tuple):
              _classes = (classes,)
  
          else:
              _classes = classes
  
          return sorted(
              [cast(PhaseT, phase) for phase in self._phases if isinstance(phase, _classes)],
              key=lambda phase: phase.order)
  
      def actions(self) -> None:
          """ Run all loaded Login or Reboot action instances of the step """
          for phase in self.phases(classes=Action):
              phase.go()
  
      def go(self) -> None:
          """ Execute the test step """
          # Show step header and how
          self.info(self.name, color='blue')
          # Show workdir in verbose mode
          self.debug('workdir', self.workdir, 'magenta')
  
  
  class Method:
      """ Step implementation method """
  
      def __init__(
              self,
              name: str,
              class_: Optional[Type['BasePlugin']] = None,
              doc: Optional[str] = None,
              order: int = DEFAULT_PLUGIN_METHOD_ORDER
              ) -> None:
          """ Store method data """
  
          doc = (doc or getattr(class_, '__doc__') or '').strip()
  
          if not doc:
              if class_:
                  raise tmt.utils.GeneralError(f"Plugin class '{class_}' provides no docstring.")
  
              raise tmt.utils.GeneralError(f"Plugin method '{name}' provides no docstring.")
  
          self.name = name
          self.class_ = class_
          self.doc = doc
          self.order = order
  
          # Parse summary and description from provided doc string
          lines: List[str] = [re.sub('^    ', '', line)
                              for line in self.doc.split('\n')]
          self.summary: str = lines[0].strip()
          self.description: str = '\n'.join(lines[1:]).lstrip()
  
      def describe(self) -> str:
          """ Format name and summary for a nice method overview """
          return f'{self.name} '.ljust(22, '.') + f' {self.summary}'
  
      def usage(self) -> str:
          """ Prepare a detailed usage from summary and description """
          if self.description:
              usage: str = self.summary + '\n\n' + self.description
          else:
              usage = self.summary
          # Disable wrapping for all paragraphs
          return re.sub('\n\n', '\n\n\b\n', usage)
  
  
  class PluginData(TypedDict, total=False):
      """ Step data structure """
      name: Optional[str]
      how: Optional[str]
      order: Optional[int]
  
  
  def provides_method(
          name: str,
          doc: Optional[str] = None,
          order: int = DEFAULT_PLUGIN_METHOD_ORDER) -> Any:
      """
      A plugin class decorator to register plugin's method with tmt steps.
  
      In the following example, developer marks ``SomePlugin`` as providing two discover methods,
      ``foo`` and ``bar``, with ``bar`` being sorted to later position among methods:
  
      .. code-block:: python
  
         @tmt.steps.provides_method('foo')
         @tmt.steps.provides_method('bar', order=80)
         class SomePlugin(tmt.steps.discover.DicoverPlugin):
           ...
  
      :param name: name of the method.
      :param doc: method documentation. If not specified, docstring of the decorated class is used.
      :param order: order of the method among other step methods.
      """
  
      def _method(cls: Type['BasePlugin']) -> Any:
          plugin_method = Method(name, class_=cls, doc=doc, order=order)
  
          cast('BasePlugin', cls.__bases__[0])._supported_methods.append(plugin_method)
  
          return cls
  
      return _method
  
  
  class PluginIndex(type):
      """
      Plugin metaclass used to register all available plugins.
  
      .. deprecated:: 1.17
         Instead of declaring `_methods` list, use :py:func:`provides_method` decorator
         to announce provided methods.
      """
  
      def __init__(
              cls,
              name: str,
              bases: List['BasePlugin'],
              attributes: Any) -> None:
          """ Store all defined methods in the parent class """
  
          plugin_methods = cast(Optional[List[Method]], getattr(cls, '_methods'))
  
          if not plugin_methods:
              return
  
          warnings.warn(
              'Use of _methods to declare plugin methods is deprecated.'
              'Use tmt.steps.provides_method decorator instead.',
              DeprecationWarning,
              stacklevel=2)
  
          for plugin_method in plugin_methods:
              plugin_method.class_ = cast(Type['BasePlugin'], cls)
              # Add to the list of supported methods in parent class
              bases[0]._supported_methods.append(plugin_method)
  
  
  class BasePlugin(Phase, metaclass=PluginIndex):
      """ Common parent of all step plugins """
  
      # Deprecated, use @provides_method(...) instead. left for backward
      # compatibility with out-of-tree plugins.
      _methods: List[Method] = []
  
      # Default implementation for all steps is shell
      # except for provision (virtual) and report (display)
      how: str = 'shell'
  
      # Common keys for all plugins of given step
      _common_keys: List[str] = []
  
      # Keys specific for given plugin
      _keys: List[str] = []
  
      # List of all supported methods aggregated from all plugins of the same step.
      _supported_methods: List[Method] = []
  
      def __init__(
              self,
              step: Step,
              data: StepData,
              workdir: tmt.utils.WorkdirArgumentType = None) -> None:
          """ Store plugin name, data and parent step """
  
          # Ensure that plugin data contains name
          if 'name' not in data:
              raise tmt.utils.GeneralError(
                  f"Missing 'name' in the {step} step config "
                  f"of the '{step.plan}' plan.")
  
          # Initialize plugin order
          if 'order' not in data or data['order'] is None:
              order = tmt.utils.DEFAULT_PLUGIN_ORDER
          else:
              try:
                  order = int(data['order'])
              except ValueError:
                  raise tmt.utils.SpecificationError(
                      f"Invalid order '{data['order']}' in {step} config "
                      f"'{data['name']}'. Should be an integer.")
  
          # Store name, data and parent step
          super().__init__(
              parent=step,
              name=data['name'],
              workdir=workdir,
              order=order)
          # It is not possible to use TypedDict here because
          # all keys are not known at the time of the class definition
          self.data = data
          self.step = step
  
      @classmethod
      def base_command(
              cls,
              usage: str,
              method_class: Optional[Type[click.Command]] = None) -> click.Command:
          """ Create base click command (common for all step plugins) """
          raise NotImplementedError
  
      @classmethod
      def options(cls, how: Optional[str] = None) -> List[tmt.options.ClickOptionDecoratorType]:
          """ Prepare command line options for given method """
          # Include common options supported across all plugins
          return tmt.options.verbose_debug_quiet + tmt.options.force_dry
  
      @classmethod
      def command(cls) -> click.Command:
          """ Prepare click command for all supported methods """
          # Create one command for each supported method
          commands: Dict[str, click.Command] = {}
          method_overview: str = f'Supported methods ({cls.how} by default):\n\n\b'
          for method in cls.methods():
              assert method.class_ is not None
              method_overview += f'\n{method.describe()}'
              command: click.Command = cls.base_command(usage=method.usage())
              # Apply plugin specific options
              for option in method.class_.options(method.name):
                  command = option(command)
              commands[method.name] = command
  
          # Create base command with common options using method class
          method_class = tmt.options.create_method_class(commands)
          command = cls.base_command(usage=method_overview, method_class=method_class)
          # Apply common options
          for option in cls.options():
              command = option(command)
          return command
  
      @classmethod
      def methods(cls) -> List[Method]:
          """ Return all supported methods ordered by priority """
          return sorted(cls._supported_methods, key=lambda method: method.order)
  
      @classmethod
      def delegate(
              cls,
              step: Step,
              data: StepData) -> 'BasePlugin':
          """
          Return plugin instance implementing the data['how'] method
  
          Supports searching by method prefix as well (e.g. 'virtual').
          The first matching method with the lowest 'order' wins.
          """
          # Filter matching methods, pick the one with the lowest order
          for method in cls.methods():
              assert method.class_ is not None
              if method.name.startswith(data['how']):
                  step.debug(
                      f"Using the '{method.class_.__name__}' plugin "
                      f"for the '{data['how']}' method.", level=2)
                  plugin = method.class_(step, data)
                  assert isinstance(plugin, BasePlugin)
                  return plugin
  
          show_step_method_hints(step, step.name, data['how'])
          # Report invalid method
          if step.plan is None:
              raise tmt.utils.GeneralError(f"Plan for {step.name} is not set.")
          raise tmt.utils.SpecificationError(
              f"Unsupported {step.name} method '{data['how']}' "
              f"in the '{step.plan.name}' plan.")
  
      def default(self, option: str, default: Optional[Any] = None) -> Any:
          """ Return default data for given option """
          return default
  
      def get(self, option: str, default: Optional[Any] = None) -> Any:
          """ Get option from plugin data, user/system config or defaults """
          # Check plugin data first
          try:
              # FIXME Enable type check once StepData defined more precisely
              return self.data[option]  # type: ignore
          except KeyError:
              pass
  
          # Check user config and system config
          # TODO
  
          # Finally check plugin defaults
          return self.default(option, default)
  
      def show(self, keys: Optional[List[str]] = None) -> None:
          """ Show plugin details for given or all available keys """
          # Show empty config with default method only in verbose mode
          if (set(self.data.keys()) == set(['how', 'name'])
                  and not self.opt('verbose')
                  and self.data['how'] == self.how):
              return
          # Step name (and optional summary)
          echo(tmt.utils.format(
              self.step.name, self.get('summary', ''),
              key_color='blue', value_color='blue'))
          # Show all or requested step attributes
          base_keys = ['name', 'how']
          if keys is None:
              keys = self._common_keys + self._keys
          for key in base_keys + keys:
              # Skip showing the default name
              if key == 'name' and self.name.startswith(tmt.utils.DEFAULT_NAME):
                  continue
              # Skip showing summary again
              if key == 'summary':
                  continue
              value = self.get(key)
              if value is not None:
                  echo(tmt.utils.format(key, value))
  
      def enabled_on_guest(self, guest: 'Guest') -> bool:
          """ Check if the plugin is enabled on the specific guest """
          where: str = self.get('where')
          if not where:
              return True
          return where in (guest.name, guest.role)
  
      def wake(self, keys: Optional[List[str]] = None) -> None:
          """
          Wake up the plugin, process data, apply options
  
          Check command line options corresponding to plugin keys
          and store their value into the 'self.data' dictionary if
          their value is True or non-empty.
  
          By default, all supported options corresponding to common
          and plugin-specific keys are processed. List of key names
          in the 'keys' parameter can be used to override only
          selected ones.
          """
          if keys is None:
              keys = self._common_keys + self._keys
          for key in keys:
              value = self.opt(key)
              if value:
                  # FIXME Enable type check once StepData defined more precisely
                  self.data[key] = value  # type: ignore
  
      # NOTE: it's tempting to rename this method to `go()` and use more natural
      # `super().go()` in child classes' `go()` methods. But, `go()` does not have
      # the same signature across all plugin types, therefore we cannot have shared
      # `go()` method in superclass - overriding it in (some) child classes would
      # raise a typing linter error reporting superclass signature differs from the
      # one in a subclass.
      #
      # Therefore we need a different name, and a way how not to forget to call this
      # method from child classes.
      def go_prolog(self) -> None:
          """ Perform actions shared among plugins when beginning their tasks """
          # Show the method
          self.info('how', self.get('how'), 'magenta')
          # Give summary if provided
          if self.get('summary'):
              self.info('summary', self.get('summary'), 'magenta')
          # Show name only if it's not the default one
          if not self.name.startswith(tmt.utils.DEFAULT_NAME):
              self.info('name', self.name, 'magenta')
          # Include order in verbose mode
          self.verbose('order', str(self.order), 'magenta', level=3)
  
      def requires(self) -> List[str]:
          """ List of packages required by the plugin on the guest """
          return []
  
  
  class GuestlessPlugin(BasePlugin):
      """ Common parent of all step plugins that do not work against a particular guest """
  
      def go(self) -> None:
          """ Perform actions shared among plugins when beginning their tasks """
  
          self.go_prolog()
  
  
  class Plugin(BasePlugin):
      """ Common parent of all step plugins that do work against a particular guest """
  
      def go(self, guest: 'Guest') -> None:
          """ Perform actions shared among plugins when beginning their tasks """
  
          self.go_prolog()
  
  
  class Action(Phase):
      """ A special action performed during a normal step. """
  
      # Dictionary containing list of requested phases for each enabled step
      _phases: Optional[Dict[str, List[int]]] = None
  
      @classmethod
      def phases(cls, step: Step) -> List[int]:
          """ Return list of phases enabled for given step """
          # Build the phase list unless done before
          if cls._phases is None:
              cls._phases = cls._parse_phases(step)
          # Return enabled phases, empty list if step not found
          try:
              return cls._phases[step.name]
          except KeyError:
              return []
  
      @classmethod
      def _parse_phases(cls, step: Step) -> Dict[str, List[int]]:
          """ Parse options and store phase order """
          phases = dict()
          options: List[str] = cls._opt('step', default=[])
  
          # Use the end of the last enabled step if no --step given
          if not options:
              login_during: Optional[Step] = None
              # The last run may have failed before all enabled steps were
              # completed, select the last step done
              if step.plan is None:
                  raise tmt.utils.GeneralError(
                      f"Plan for {step.name} is not set.")
              if step.plan.my_run.opt('last'):
                  steps: List[Step] = [
                      s for s in step.plan.steps() if s.status() == 'done']
                  login_during = steps[-1] if steps else None
              # Default to the last enabled step if no completed step found
              if login_during is None:
                  login_during = list(step.plan.steps())[-1]
              # Only login if the error occurred after provision
              if login_during != step.plan.discover:
                  phases[login_during.name] = [PHASE_END]
  
          # Process provided options
          for option in options:
              # Parse the step:phase format
              matched = re.match(r'(\w+)(:(\w+))?', option)
              if matched:
                  step_name, _, phase = matched.groups()
              if not matched or step_name not in STEPS:
                  raise tmt.utils.GeneralError(f"Invalid step '{option}'.")
              # Check phase format, convert into int, use end by default
              try:
                  phase = int(phase)
              except TypeError:
                  phase = PHASE_END
              except ValueError:
                  # Convert 'start' and 'end' aliases
                  try:
                      phase = cast(Dict[str, int],
                                   dict(start=PHASE_START, end=PHASE_END))[phase]
                  except KeyError:
                      raise tmt.utils.GeneralError(f"Invalid phase '{phase}'.")
              # Store the phase for given step
              try:
                  phases[step_name].append(phase)
              except KeyError:
                  phases[step_name] = [phase]
          return phases
  
      def go(self) -> None:
          raise NotImplementedError()
  
  
  class Reboot(Action):
      """ Reboot guest """
  
      # True if reboot enabled
      _enabled: bool = False
  
      def __init__(self, step: Step, order: int) -> None:
          """ Initialize relations, store the reboot order """
          super().__init__(parent=step, name='reboot', order=order)
  
      @classmethod
      def command(
              cls,
              method_class: Optional[Method] = None,
              usage: Optional[str] = None) -> click.Command:
          """ Create the reboot command """
          @click.command()
          @click.pass_context
          @click.option(
              '-s', '--step', metavar='STEP[:PHASE]', multiple=True,
              help='Reboot machine during given phase of selected step(s).')
          @click.option(
              '--hard', is_flag=True,
              help='Hard reboot of the machine. Unsaved data may be lost.')
          def reboot(context: Any, **kwargs: Any) -> None:
              """ Reboot the guest. """
              Reboot._save_context(context)
              Reboot._enabled = True
  
          return reboot
  
      @classmethod
      def plugins(cls, step: Step) -> List['Reboot']:
          """ Return list of reboot instances for given step """
          if not Reboot._enabled:
              return []
          return [Reboot(step, phase) for phase in cls.phases(step)]
  
      def go(self) -> None:
          """ Reboot the guest(s) """
          self.info('reboot', 'Rebooting guest', color='yellow')
          assert isinstance(self.parent, Step)
          assert hasattr(self.parent, 'plan') and self.parent.plan is not None
          for guest in self.parent.plan.provision.guests():
              guest.reboot(hard=self.opt('hard'))
          self.info('reboot', 'Reboot finished', color='yellow')
  
  
  class Login(Action):
      """ Log into the guest """
  
      # True if interactive login enabled
      _enabled: bool = False
  
      def __init__(self, step: Step, order: int):
          """ Initialize relations, store the login order """
          super().__init__(parent=step, name='login', order=order)
  
      @classmethod
      def command(
              cls,
              method_class: Optional[Method] = None,
              usage: Optional[str] = None) -> click.Command:
          """ Create the login command """
          @click.command()
          @click.pass_context
          @click.option(
              '-s', '--step', metavar='STEP[:PHASE]', multiple=True,
              help='Log in during given phase of selected step(s).')
          @click.option(
              '-w', '--when', metavar='RESULT', multiple=True,
              type=click.Choice(tmt.base.Result._results),
              help='Log in if a test finished with given result(s).')
          @click.option(
              '-c', '--command', metavar='COMMAND',
              multiple=True, default=['bash'],
              help="Run given command(s). Default is 'bash'.")
          def login(context: Any, **kwargs: Any) -> None:
              """
              Provide user with an interactive shell on the guest.
  
              By default the shell is provided at the end of the last
              enabled step. When used together with the --last option the
              last completed step is selected. Use one or more --step
              options to select a different step instead.
  
              Optional phase can be provided to specify the exact phase of
              the step when the shell should be provided. The following
              values are supported:
  
              \b
                  start ... beginning of the step (same as '10')
                  end ..... end of the step (default, same as '90')
                  00-99 ... integer order defining the exact phase
  
              Usually the main step execution happens with order 50.
              Consult individual step documentation for more details.
  
              For the execute step and following steps it is also possible
              to conditionally enable the login feature only if some of
              the tests finished with given result (pass, info, fail,
              warn, error).
              """
              Login._save_context(context)
              Login._enabled = True
  
          return login
  
      @classmethod
      def plugins(cls, step: Step) -> List['Login']:
          """ Return list of login instances for given step """
          if not Login._enabled:
              return []
          return [Login(step, phase) for phase in cls.phases(step)]
  
      def go(self) -> None:
          """ Login to the guest(s) """
          # Verify possible test result condition
          count: Dict[str, int] = dict()
          expected_results: Optional[List[str]] = self.opt('when')
  
          assert isinstance(self.parent, Step)
          assert hasattr(self.parent, 'plan') and self.parent.plan is not None
  
          if expected_results:
              for expected_result in expected_results:
                  count[expected_result] = len([
                      result for result in self.parent.plan.execute.results()
                      if result.result == expected_result])
              if not any(count.values()):
                  self.info('Skipping interactive shell', color='yellow')
                  return
  
          # Run the interactive command
          commands: List[str] = self.opt('command')
          self.info('login', 'Starting interactive shell', color='yellow')
          for guest in self.parent.plan.provision.guests():
              # Attempt to push the workdir to the guest
              try:
                  guest.push()
                  cwd: Optional[str] = self.parent.plan.worktree
              except tmt.utils.GeneralError:
                  self.warn("Failed to push workdir to the guest.")
                  cwd = None
              # Execute all requested commands
              for command in commands:
                  self.debug(f"Run '{command}' in interactive mode.")
                  guest.execute(command, interactive=True, cwd=cwd)
          self.info('login', 'Interactive shell finished', color='yellow')
  
files: tmt/steps/__init__.py
files: import glob
  import os
  import re
  import shutil
  import subprocess
  
  import click
  import fmf
  
  import tmt
  import tmt.beakerlib
  import tmt.steps
  import tmt.steps.discover
  from tmt.utils import git_clone
  
  
  @tmt.steps.provides_method('fmf')
  class DiscoverFmf(tmt.steps.discover.DiscoverPlugin):
      """
      Discover available tests from fmf metadata
  
      By default all available tests from the current repository are used
      so the minimal configuration looks like this:
  
          discover:
              how: fmf
  
      Full config example:
  
          discover:
              how: fmf
              url: https://github.com/teemtee/tmt
              ref: main
              path: /fmf/root
              test: /tests/basic
              filter: 'tier: 1'
  
      If no 'ref' is provided, the default branch from the origin is used.
  
      For DistGit repo one can extract source tarball first and discover
      tests from it by using 'distgit-source: true'. It can be used
      together with 'ref', 'path' and 'url', However 'ref' is not possible
      without using 'url'.
  
          discover:
              how: fmf
              dist-git-source: true
  
      Related config options (all optional):
      * dist-git-merge - set to True if you want to copy in extracted
        sources to the local repo
      * dist-git-init - set to True and 'fmf init' will be called inside
        extracted sources (at dist-git-extract or top directory)
      * dist-git-extract - directory (glob supported) to copy from
        extracted sources (defaults to inner fmf-root)
      * dist-git-remove-fmf-root - set to True to remove fmf root from
        extracted sources
  
      Selecting tests containing specified link is possible using 'link'
      option accepting RELATION:TARGET format of values. Regular
      expressions are supported for both relation and target part of the
      value. Relation can be omitted to target match any relation.
  
          discover:
              how: fmf
              link: verifies:.*issue/850$
  
      It is also possible to limit tests only to those that have changed
      in git since a given revision. This can be particularly useful when
      testing changes to tests themselves (e.g. in a pull request CI).
  
      Related config options (all optional):
      * modified-only - set to True if you want to filter modified tests
      * modified-url - fetched as "reference" remote in the test dir
      * modified-ref - the ref to compare against
  
      Example to compare local repo against upstream 'main' branch:
  
          discover:
              how: fmf
              modified-only: True
              modified-url: https://github.com/teemtee/tmt
              modified-ref: reference/main
  
      Note that internally the modified tests are appended to the list
      specified via 'test', so those tests will also be selected even if
      not modified.
      """
  
      # Supported keys
      _keys = [
          "url", "ref", "path", "test", "link", "filter",
          "modified-only", "modified-url", "modified-ref",
          "dist-git-init", "dist-git-remove-fmf-root", "dist-git-merge",
          "dist-git-extract",
          "fmf-id", "exclude"]
  
      REF_OPTION = click.option(
          '-r', '--ref', metavar='REVISION',
          help='Branch, tag or commit specifying the git revision.')
      TEST_OPTION = click.option(
          '-t', '--test', metavar='NAMES', multiple=True,
          help='Select tests by name.')
      FILTER_OPTION = click.option(
          '-F', '--filter', metavar='FILTERS', multiple=True,
          help='Include only tests matching the filter.')
      EXCLUDE_OPTION = click.option(
          '-x', '--exclude', metavar='[REGEXP]', multiple=True,
          help="Exclude a regular expression from search result.")
  
      # Options which require .git to be present for their functionality
      _REQUIRES_GIT = (
          "ref",
          "modified-url",
          "modified-only",
          "fmf-id",
          )
  
      @classmethod
      def options(cls, how=None):
          """ Prepare command line options for given method """
          return [
              click.option(
                  '-u', '--url', metavar='REPOSITORY',
                  help='URL of the git repository with fmf metadata.'),
              cls.REF_OPTION,
              click.option(
                  '--modified-url', metavar='REPOSITORY',
                  help='URL of the reference git repository with fmf metadata.'),
              click.option(
                  '--modified-ref', metavar='REVISION',
                  help='Branch, tag or commit specifying the reference git '
                  'revision (if not provided, the default branch is used).'),
              click.option(
                  '-m', '--modified-only', is_flag=True,
                  help='If set, select only tests modified '
                  'since reference revision.'),
              click.option(
                  '-p', '--path', metavar='ROOT',
                  help='Path to the metadata tree root.'),
              cls.TEST_OPTION,
              click.option(
                  '--link', metavar="RELATION:TARGET", multiple=True,
                  help="Filter by linked objects (regular expressions are "
                       "supported for both relation and target)."),
              cls.FILTER_OPTION,
              cls.EXCLUDE_OPTION,
              click.option(
                  '--fmf-id', default=False, is_flag=True,
                  help='Show fmf identifiers for tests discovered in plan.'),
              click.option(
                  '--sync-repo', default=False, is_flag=True,
                  help='Force the sync of the whole git repo. By default, the '
                       'repo is copied only if the used options require it.'),
              click.option(
                  '--dist-git-init', is_flag=True,
                  help='Initialize fmf root inside extracted sources '
                       '(at dist-git-extract or top directory).'),
              click.option(
                  '--dist-git-remove-fmf-root', is_flag=True,
                  help='Remove fmf root from extracted source '
                       '(top one or selected by copy-path, '
                       'happens before dist-git-extract.'),
              click.option(
                  '--dist-git-merge', is_flag=True,
                  help='Merge copied sources and plan fmf root.'),
              click.option(
                  '--dist-git-extract',
                  help='What to copy from extracted sources, globbing is '
                       'supported. Defaults to the top fmf root if it is '
                       'present, otherwise top directory (shortcut "/").'
                  ),
              ] + super().options(how)
  
      def wake(self):
          """ Wake up the plugin, process data, apply options """
          # Handle backward-compatible stuff
          if 'repository' in self.data:
              self.data['url'] = self.data.pop('repository')
          if 'revision' in self.data:
              self.data['ref'] = self.data.pop('revision')
  
          # Make sure that 'exclude', 'filter' and 'test' keys are lists
          tmt.utils.listify(self.data, keys=["exclude", "filter", "test"])
  
          # Process command line options, apply defaults
          super().wake()
  
      @property
      def is_in_standalone_mode(self):
          """ Enable standalone mode when listing fmf ids """
          if self.opt('fmf_id'):
              return True
          return super().is_in_standalone_mode
  
      def go(self):
          """ Discover available tests """
          super(DiscoverFmf, self).go()
  
          # Check url and path, prepare test directory
          url = self.get('url')
          path = self.get('path')
          # Save the test directory so that others can reference it
          ref = self.get('ref')
          self.testdir = os.path.join(self.workdir, 'tests')
          sourcedir = os.path.join(self.workdir, 'source')
          dist_git_source = self.get('dist-git-source', False)
          dist_git_merge = self.get('dist-git-merge', False)
          dist_git_init = self.get('dist-git-init', False)
          dist_git_extract = self.get('dist-git-extract', None)
          dist_git_remove_fmf_root = self.get('dist-git-remove-fmf-root', False)
  
          # Self checks
          if dist_git_source and not dist_git_merge and (ref or url):
              raise tmt.utils.DiscoverError(
                  "Cannot manipulate with dist-git without "
                  "the `--dist-git-merge` option.")
  
          def get_git_root(dir):
              return self.run(
                  ["git", "rev-parse", "--show-toplevel"],
                  cwd=dir, dry=True)[0].strip('\n')
  
          # Raise an exception if --fmf-id uses w/o url and git root
          # doesn't exist for discovered plan
          if self.opt('fmf_id'):
              def assert_git_url(plan_name=None):
                  try:
                      subprocess.run(
                          'git rev-parse --show-toplevel'.split(),
                          stdout=subprocess.PIPE,
                          stderr=subprocess.DEVNULL,
                          check=True)
                  except subprocess.CalledProcessError:
                      raise tmt.utils.DiscoverError(
                          f"`tmt run discover --fmf-id` without `url` option in "
                          f"plan `{plan_name}` can be used only within"
                          f" git repo.")
              # It covers only one case, when there is:
              # 1) no --url on CLI
              # 2) plan w/o url exists in test run
              if not self.opt('url'):
                  try:
                      fmf_tree = fmf.Tree(os.getcwd())
                  except fmf.utils.RootError:
                      raise tmt.utils.DiscoverError(
                          "No metadata found in the current directory. "
                          "Use 'tmt init' to get started.")
                  for i, attr in enumerate(fmf_tree.climb()):
                      try:
                          plan_url = attr.data.get('discover').get('url')
                          plan_name = attr.name
                          if not plan_url:
                              assert_git_url(plan_name)
                      except AttributeError:
                          pass
              # All other cases are covered by this condition
              if not url:
                  assert_git_url(self.step.plan.name)
  
          # Clone provided git repository (if url given) with disabled
          # prompt to ignore possibly missing or private repositories
          if url:
              self.info('url', url, 'green')
              self.debug(f"Clone '{url}' to '{self.testdir}'.")
              # Shallow clone to speed up testing and
              # minimize data transfers if ref is not provided
              git_clone(url, self.testdir, self, env={"GIT_ASKPASS": "echo"}, shallow=ref is None)
              git_root = self.testdir
          # Copy git repository root to workdir
          else:
              fmf_root = path or self.step.plan.my_run.tree.root
              requires_git = self.opt('sync-repo') or any(
                  self.get(opt) for opt in self._REQUIRES_GIT)
              # Path for distgit sources cannot be checked until the
              # tarball is extracted
              if path and not os.path.isdir(path) and not dist_git_source:
                  raise tmt.utils.DiscoverError(
                      f"Provided path '{path}' is not a directory.")
              if dist_git_source:
                  # Ensure we're in a git repo when extracting dist-git sources
                  try:
                      git_root = get_git_root(self.step.plan.my_run.tree.root)
                  except tmt.utils.RunError:
                      raise tmt.utils.DiscoverError(
                          f"{self.step.plan.my_run.tree.root} is not a git repo")
              else:
                  if fmf_root is None:
                      raise tmt.utils.DiscoverError(
                          "No metadata found in the current directory.")
                  # Check git repository root (use fmf root if not found)
                  try:
                      git_root = get_git_root(fmf_root)
                  except tmt.utils.RunError:
                      self.debug(f"Git root not found, using '{fmf_root}.'")
                      git_root = fmf_root
                  # Set path to relative path from the git root to fmf root
                  path = os.path.relpath(
                      fmf_root, git_root if requires_git else fmf_root)
  
              # And finally copy the git/fmf root directory to testdir
              # (for dist-git case only when merge explicitly requested)
              directory = git_root if requires_git else fmf_root
              self.info('directory', directory, 'green')
              if not dist_git_source or dist_git_merge:
                  self.debug(f"Copy '{directory}' to '{self.testdir}'.")
                  if not self.opt('dry'):
                      shutil.copytree(directory, self.testdir, symlinks=True)
  
          # Checkout revision if requested
          if ref:
              self.info('ref', ref, 'green')
              self.debug(f"Checkout ref '{ref}'.")
              self.run(
                  ['git', 'checkout', '-f', str(ref)],
                  cwd=self.testdir)
  
          # Show current commit hash if inside a git repository
          if os.path.isdir(self.testdir):
              try:
                  hash_, _ = self.run(["git", "rev-parse", "--short", "HEAD"],
                                      cwd=self.testdir)
                  self.verbose('hash', hash_.strip(), 'green')
              except (tmt.utils.RunError, AttributeError):
                  pass
  
          # Fetch and extract distgit sources
          if dist_git_source:
              try:
                  self.extract_distgit_source(
                      git_root, sourcedir, self.get('dist-git-type'))
              except Exception as error:
                  raise tmt.utils.DiscoverError(
                      "Failed to process 'dist-git-source'.", original=error)
  
              # Check what should be extracted from the sources
              if dist_git_extract:
                  if dist_git_extract == '/':
                      dist_git_extract = sourcedir
                  else:
                      try:
                          dist_git_extract = glob.glob(os.path.join(
                              sourcedir, dist_git_extract.lstrip('/')))[0]
                      except IndexError:
                          raise tmt.utils.DiscoverError(
                              f"Couldn't glob '{dist_git_extract}' "
                              f"within extracted sources.")
  
              # Check sources for the fmf root, copy git root if not found
              if not dist_git_init and not dist_git_extract:
                  try:
                      top_fmf_root = tmt.utils.find_fmf_root(sourcedir)[0]
                  except tmt.utils.MetadataError:
                      dist_git_extract = sourcedir
                      if not dist_git_merge:
                          self.warn(
                              "Extracted sources do not contain fmf root, "
                              "merging with plan data. Avoid this warning by "
                              "explicit use of the '--dist-git-merge' option.")
                          self.debug(f"Copy '{git_root}' to '{self.testdir}'.")
                          if not self.opt('dry'):
                              shutil.copytree(
                                  git_root, self.testdir, symlinks=True)
  
              # Initialize or remove fmf root
              if dist_git_init:
                  if not dist_git_extract:
                      dist_git_extract = sourcedir
                  if not self.opt('dry'):
                      fmf.Tree.init(dist_git_extract)
              elif dist_git_remove_fmf_root:
                  if not self.opt('dry'):
                      shutil.rmtree(os.path.join(
                          dist_git_extract or top_fmf_root, '.fmf'))
                  if not dist_git_extract:
                      dist_git_extract = sourcedir
  
              # Now can safely default to top_fmf_root
              if not dist_git_extract:
                  dist_git_extract = top_fmf_root
  
              # Now copy dist_git_extract into tests
              if not self.opt('dry'):
                  tmt.utils.copytree(
                      dist_git_extract,
                      self.testdir,
                      symlinks=True,
                      dirs_exist_ok=True)
  
          # Adjust path and optionally show
          if path is None or path == '.':
              path = ''
          else:
              self.info('path', path, 'green')
  
          # Prepare the whole tree path and test path prefix
          tree_path = os.path.join(self.testdir, path.lstrip('/'))
          if not os.path.isdir(tree_path) and not self.opt('dry'):
              raise tmt.utils.DiscoverError(
                  f"Metadata tree path '{path}' not found.")
          prefix_path = os.path.join('/tests', path.lstrip('/'))
  
          # Show filters and test names if provided
          # Check the 'test --filter' option first, then from discover
          filters = list(tmt.base.Test._opt('filters') or self.get('filter', []))
          for filter_ in filters:
              self.info('filter', filter_, 'green')
          # Names of tests selected by --test option
          names = self.get('test', [])
          if names:
              self.info('tests', fmf.utils.listed(names), 'green')
          # Check the 'test --link' option first, then from discover
          links = list(tmt.base.Test._opt('link') or self.get('link', []))
          for link_ in links:
              self.info('link', link_, 'green')
  
          excludes = list(tmt.base.Test._opt('exclude')
                          or self.get('exclude', []))
  
          # Filter only modified tests if requested
          modified_only = self.get('modified-only')
          modified_url = self.get('modified-url')
          if modified_url:
              self.info('modified-url', modified_url, 'green')
              self.debug(f"Fetch also '{modified_url}' as 'reference'.")
              self.run(['git', 'remote', 'add', 'reference', modified_url],
                       cwd=self.testdir)
              self.run(['git', 'fetch', 'reference'], cwd=self.testdir)
          if modified_only:
              modified_ref = self.get(
                  'modified-ref', tmt.utils.default_branch(self.testdir))
              self.info('modified-ref', modified_ref, 'green')
              output = self.run(
                  ['git', 'log', '--format=', '--stat', '--name-only',
                   f"{modified_ref}..HEAD"], cwd=self.testdir)[0]
              modified = set(
                  f"^/{re.escape(name)}"
                  for name in map(os.path.dirname, output.split('\n')) if name)
              self.debug(f"Limit to modified test dirs: {modified}", level=3)
              names.extend(modified)
  
          # Initialize the metadata tree, search for available tests
          self.debug(f"Check metadata tree in '{tree_path}'.")
          if self.opt('dry'):
              self._tests = []
              return
          tree = tmt.Tree(path=tree_path, context=self.step.plan._fmf_context())
          self._tests = tree.tests(
              filters=filters,
              names=names,
              conditions=["manual is False"],
              unique=False,
              links=links,
              excludes=excludes)
  
          # Prefix tests and handle library requires
          for test in self._tests:
              # Prefix test path with 'tests' and possible 'path' prefix
              test.path = os.path.join(prefix_path, test.path.lstrip('/'))
              # Check for possible required beakerlib libraries
              if test.require or test.recommend:
                  test.require, test.recommend, _ = tmt.beakerlib.dependencies(
                      test.require, test.recommend, parent=self)
  
          # Add TMT_SOURCE_DIR variable for each test
          if dist_git_source:
              for test in self._tests:
                  test.environment['TMT_SOURCE_DIR'] = sourcedir
  
      def tests(self):
          """ Return all discovered tests """
          return self._tests
  
files: tmt/steps/discover/fmf.py
files: import glob
  import os
  import re
  import shutil
  import subprocess
  
  import click
  import fmf
  
  import tmt
  import tmt.beakerlib
  import tmt.steps
  import tmt.steps.discover
  from tmt.utils import git_clone
  
  
  @tmt.steps.provides_method('fmf')
  class DiscoverFmf(tmt.steps.discover.DiscoverPlugin):
      """
      Discover available tests from fmf metadata
  
      By default all available tests from the current repository are used
      so the minimal configuration looks like this:
  
          discover:
              how: fmf
  
      Full config example:
  
          discover:
              how: fmf
              url: https://github.com/teemtee/tmt
              ref: main
              path: /fmf/root
              test: /tests/basic
              filter: 'tier: 1'
  
      If no 'ref' is provided, the default branch from the origin is used.
  
      For DistGit repo one can extract source tarball first and discover
      tests from it by using 'distgit-source: true'. It can be used
      together with 'ref', 'path' and 'url', However 'ref' is not possible
      without using 'url'.
  
          discover:
              how: fmf
              dist-git-source: true
  
      Related config options (all optional):
      * dist-git-merge - set to True if you want to copy in extracted
        sources to the local repo
      * dist-git-init - set to True and 'fmf init' will be called inside
        extracted sources (at dist-git-extract or top directory)
      * dist-git-extract - directory (glob supported) to copy from
        extracted sources (defaults to inner fmf-root)
      * dist-git-remove-fmf-root - set to True to remove fmf root from
        extracted sources
  
      Selecting tests containing specified link is possible using 'link'
      option accepting RELATION:TARGET format of values. Regular
      expressions are supported for both relation and target part of the
      value. Relation can be omitted to target match any relation.
  
          discover:
              how: fmf
              link: verifies:.*issue/850$
  
      It is also possible to limit tests only to those that have changed
      in git since a given revision. This can be particularly useful when
      testing changes to tests themselves (e.g. in a pull request CI).
  
      Related config options (all optional):
      * modified-only - set to True if you want to filter modified tests
      * modified-url - fetched as "reference" remote in the test dir
      * modified-ref - the ref to compare against
  
      Example to compare local repo against upstream 'main' branch:
  
          discover:
              how: fmf
              modified-only: True
              modified-url: https://github.com/teemtee/tmt
              modified-ref: reference/main
  
      Note that internally the modified tests are appended to the list
      specified via 'test', so those tests will also be selected even if
      not modified.
      """
  
      # Supported keys
      _keys = [
          "url", "ref", "path", "test", "link", "filter",
          "modified-only", "modified-url", "modified-ref",
          "dist-git-init", "dist-git-remove-fmf-root", "dist-git-merge",
          "dist-git-extract",
          "fmf-id", "exclude"]
  
      REF_OPTION = click.option(
          '-r', '--ref', metavar='REVISION',
          help='Branch, tag or commit specifying the git revision.')
      TEST_OPTION = click.option(
          '-t', '--test', metavar='NAMES', multiple=True,
          help='Select tests by name.')
      FILTER_OPTION = click.option(
          '-F', '--filter', metavar='FILTERS', multiple=True,
          help='Include only tests matching the filter.')
      EXCLUDE_OPTION = click.option(
          '-x', '--exclude', metavar='[REGEXP]', multiple=True,
          help="Exclude a regular expression from search result.")
  
      # Options which require .git to be present for their functionality
      _REQUIRES_GIT = (
          "ref",
          "modified-url",
          "modified-only",
          "fmf-id",
          )
  
      @classmethod
      def options(cls, how=None):
          """ Prepare command line options for given method """
          return [
              click.option(
                  '-u', '--url', metavar='REPOSITORY',
                  help='URL of the git repository with fmf metadata.'),
              cls.REF_OPTION,
              click.option(
                  '--modified-url', metavar='REPOSITORY',
                  help='URL of the reference git repository with fmf metadata.'),
              click.option(
                  '--modified-ref', metavar='REVISION',
                  help='Branch, tag or commit specifying the reference git '
                  'revision (if not provided, the default branch is used).'),
              click.option(
                  '-m', '--modified-only', is_flag=True,
                  help='If set, select only tests modified '
                  'since reference revision.'),
              click.option(
                  '-p', '--path', metavar='ROOT',
                  help='Path to the metadata tree root.'),
              cls.TEST_OPTION,
              click.option(
                  '--link', metavar="RELATION:TARGET", multiple=True,
                  help="Filter by linked objects (regular expressions are "
                       "supported for both relation and target)."),
              cls.FILTER_OPTION,
              cls.EXCLUDE_OPTION,
              click.option(
                  '--fmf-id', default=False, is_flag=True,
                  help='Show fmf identifiers for tests discovered in plan.'),
              click.option(
                  '--sync-repo', default=False, is_flag=True,
                  help='Force the sync of the whole git repo. By default, the '
                       'repo is copied only if the used options require it.'),
              click.option(
                  '--dist-git-init', is_flag=True,
                  help='Initialize fmf root inside extracted sources '
                       '(at dist-git-extract or top directory).'),
              click.option(
                  '--dist-git-remove-fmf-root', is_flag=True,
                  help='Remove fmf root from extracted source '
                       '(top one or selected by copy-path, '
                       'happens before dist-git-extract.'),
              click.option(
                  '--dist-git-merge', is_flag=True,
                  help='Merge copied sources and plan fmf root.'),
              click.option(
                  '--dist-git-extract',
                  help='What to copy from extracted sources, globbing is '
                       'supported. Defaults to the top fmf root if it is '
                       'present, otherwise top directory (shortcut "/").'
                  ),
              ] + super().options(how)
  
      def wake(self, keys=None):
          """ Wake up the plugin, process data, apply options """
          # Handle backward-compatible stuff
          if 'repository' in self.data:
              self.data['url'] = self.data.pop('repository')
          if 'revision' in self.data:
              self.data['ref'] = self.data.pop('revision')
  
          # Make sure that 'exclude', 'filter' and 'test' keys are lists
          tmt.utils.listify(self.data, keys=["exclude", "filter", "test"])
  
          # Process command line options, apply defaults
          super().wake(keys=keys)
  
      @property
      def is_in_standalone_mode(self):
          """ Enable standalone mode when listing fmf ids """
          if self.opt('fmf_id'):
              return True
          return super().is_in_standalone_mode
  
      def go(self):
          """ Discover available tests """
          super(DiscoverFmf, self).go()
  
          # Check url and path, prepare test directory
          url = self.get('url')
          path = self.get('path')
          # Save the test directory so that others can reference it
          ref = self.get('ref')
          self.testdir = os.path.join(self.workdir, 'tests')
          sourcedir = os.path.join(self.workdir, 'source')
          dist_git_source = self.get('dist-git-source', False)
          dist_git_merge = self.get('dist-git-merge', False)
          dist_git_init = self.get('dist-git-init', False)
          dist_git_extract = self.get('dist-git-extract', None)
          dist_git_remove_fmf_root = self.get('dist-git-remove-fmf-root', False)
  
          # Self checks
          if dist_git_source and not dist_git_merge and (ref or url):
              raise tmt.utils.DiscoverError(
                  "Cannot manipulate with dist-git without "
                  "the `--dist-git-merge` option.")
  
          def get_git_root(dir):
              return self.run(
                  ["git", "rev-parse", "--show-toplevel"],
                  cwd=dir, dry=True)[0].strip('\n')
  
          # Raise an exception if --fmf-id uses w/o url and git root
          # doesn't exist for discovered plan
          if self.opt('fmf_id'):
              def assert_git_url(plan_name=None):
                  try:
                      subprocess.run(
                          'git rev-parse --show-toplevel'.split(),
                          stdout=subprocess.PIPE,
                          stderr=subprocess.DEVNULL,
                          check=True)
                  except subprocess.CalledProcessError:
                      raise tmt.utils.DiscoverError(
                          f"`tmt run discover --fmf-id` without `url` option in "
                          f"plan `{plan_name}` can be used only within"
                          f" git repo.")
              # It covers only one case, when there is:
              # 1) no --url on CLI
              # 2) plan w/o url exists in test run
              if not self.opt('url'):
                  try:
                      fmf_tree = fmf.Tree(os.getcwd())
                  except fmf.utils.RootError:
                      raise tmt.utils.DiscoverError(
                          "No metadata found in the current directory. "
                          "Use 'tmt init' to get started.")
                  for i, attr in enumerate(fmf_tree.climb()):
                      try:
                          plan_url = attr.data.get('discover').get('url')
                          plan_name = attr.name
                          if not plan_url:
                              assert_git_url(plan_name)
                      except AttributeError:
                          pass
              # All other cases are covered by this condition
              if not url:
                  assert_git_url(self.step.plan.name)
  
          # Clone provided git repository (if url given) with disabled
          # prompt to ignore possibly missing or private repositories
          if url:
              self.info('url', url, 'green')
              self.debug(f"Clone '{url}' to '{self.testdir}'.")
              # Shallow clone to speed up testing and
              # minimize data transfers if ref is not provided
              git_clone(url, self.testdir, self, env={"GIT_ASKPASS": "echo"}, shallow=ref is None)
              git_root = self.testdir
          # Copy git repository root to workdir
          else:
              fmf_root = path or self.step.plan.my_run.tree.root
              requires_git = self.opt('sync-repo') or any(
                  self.get(opt) for opt in self._REQUIRES_GIT)
              # Path for distgit sources cannot be checked until the
              # tarball is extracted
              if path and not os.path.isdir(path) and not dist_git_source:
                  raise tmt.utils.DiscoverError(
                      f"Provided path '{path}' is not a directory.")
              if dist_git_source:
                  # Ensure we're in a git repo when extracting dist-git sources
                  try:
                      git_root = get_git_root(self.step.plan.my_run.tree.root)
                  except tmt.utils.RunError:
                      raise tmt.utils.DiscoverError(
                          f"{self.step.plan.my_run.tree.root} is not a git repo")
              else:
                  if fmf_root is None:
                      raise tmt.utils.DiscoverError(
                          "No metadata found in the current directory.")
                  # Check git repository root (use fmf root if not found)
                  try:
                      git_root = get_git_root(fmf_root)
                  except tmt.utils.RunError:
                      self.debug(f"Git root not found, using '{fmf_root}.'")
                      git_root = fmf_root
                  # Set path to relative path from the git root to fmf root
                  path = os.path.relpath(
                      fmf_root, git_root if requires_git else fmf_root)
  
              # And finally copy the git/fmf root directory to testdir
              # (for dist-git case only when merge explicitly requested)
              directory = git_root if requires_git else fmf_root
              self.info('directory', directory, 'green')
              if not dist_git_source or dist_git_merge:
                  self.debug(f"Copy '{directory}' to '{self.testdir}'.")
                  if not self.opt('dry'):
                      shutil.copytree(directory, self.testdir, symlinks=True)
  
          # Checkout revision if requested
          if ref:
              self.info('ref', ref, 'green')
              self.debug(f"Checkout ref '{ref}'.")
              self.run(
                  ['git', 'checkout', '-f', str(ref)],
                  cwd=self.testdir)
  
          # Show current commit hash if inside a git repository
          if os.path.isdir(self.testdir):
              try:
                  hash_, _ = self.run(["git", "rev-parse", "--short", "HEAD"],
                                      cwd=self.testdir)
                  self.verbose('hash', hash_.strip(), 'green')
              except (tmt.utils.RunError, AttributeError):
                  pass
  
          # Fetch and extract distgit sources
          if dist_git_source:
              try:
                  self.extract_distgit_source(
                      git_root, sourcedir, self.get('dist-git-type'))
              except Exception as error:
                  raise tmt.utils.DiscoverError(
                      "Failed to process 'dist-git-source'.", original=error)
  
              # Check what should be extracted from the sources
              if dist_git_extract:
                  if dist_git_extract == '/':
                      dist_git_extract = sourcedir
                  else:
                      try:
                          dist_git_extract = glob.glob(os.path.join(
                              sourcedir, dist_git_extract.lstrip('/')))[0]
                      except IndexError:
                          raise tmt.utils.DiscoverError(
                              f"Couldn't glob '{dist_git_extract}' "
                              f"within extracted sources.")
  
              # Check sources for the fmf root, copy git root if not found
              if not dist_git_init and not dist_git_extract:
                  try:
                      top_fmf_root = tmt.utils.find_fmf_root(sourcedir)[0]
                  except tmt.utils.MetadataError:
                      dist_git_extract = sourcedir
                      if not dist_git_merge:
                          self.warn(
                              "Extracted sources do not contain fmf root, "
                              "merging with plan data. Avoid this warning by "
                              "explicit use of the '--dist-git-merge' option.")
                          self.debug(f"Copy '{git_root}' to '{self.testdir}'.")
                          if not self.opt('dry'):
                              shutil.copytree(
                                  git_root, self.testdir, symlinks=True)
  
              # Initialize or remove fmf root
              if dist_git_init:
                  if not dist_git_extract:
                      dist_git_extract = sourcedir
                  if not self.opt('dry'):
                      fmf.Tree.init(dist_git_extract)
              elif dist_git_remove_fmf_root:
                  if not self.opt('dry'):
                      shutil.rmtree(os.path.join(
                          dist_git_extract or top_fmf_root, '.fmf'))
                  if not dist_git_extract:
                      dist_git_extract = sourcedir
  
              # Now can safely default to top_fmf_root
              if not dist_git_extract:
                  dist_git_extract = top_fmf_root
  
              # Now copy dist_git_extract into tests
              if not self.opt('dry'):
                  tmt.utils.copytree(
                      dist_git_extract,
                      self.testdir,
                      symlinks=True,
                      dirs_exist_ok=True)
  
          # Adjust path and optionally show
          if path is None or path == '.':
              path = ''
          else:
              self.info('path', path, 'green')
  
          # Prepare the whole tree path and test path prefix
          tree_path = os.path.join(self.testdir, path.lstrip('/'))
          if not os.path.isdir(tree_path) and not self.opt('dry'):
              raise tmt.utils.DiscoverError(
                  f"Metadata tree path '{path}' not found.")
          prefix_path = os.path.join('/tests', path.lstrip('/'))
  
          # Show filters and test names if provided
          # Check the 'test --filter' option first, then from discover
          filters = list(tmt.base.Test._opt('filters') or self.get('filter', []))
          for filter_ in filters:
              self.info('filter', filter_, 'green')
          # Names of tests selected by --test option
          names = self.get('test', [])
          if names:
              self.info('tests', fmf.utils.listed(names), 'green')
          # Check the 'test --link' option first, then from discover
          links = list(tmt.base.Test._opt('link') or self.get('link', []))
          for link_ in links:
              self.info('link', link_, 'green')
  
          excludes = list(tmt.base.Test._opt('exclude')
                          or self.get('exclude', []))
  
          # Filter only modified tests if requested
          modified_only = self.get('modified-only')
          modified_url = self.get('modified-url')
          if modified_url:
              self.info('modified-url', modified_url, 'green')
              self.debug(f"Fetch also '{modified_url}' as 'reference'.")
              self.run(['git', 'remote', 'add', 'reference', modified_url],
                       cwd=self.testdir)
              self.run(['git', 'fetch', 'reference'], cwd=self.testdir)
          if modified_only:
              modified_ref = self.get(
                  'modified-ref', tmt.utils.default_branch(self.testdir))
              self.info('modified-ref', modified_ref, 'green')
              output = self.run(
                  ['git', 'log', '--format=', '--stat', '--name-only',
                   f"{modified_ref}..HEAD"], cwd=self.testdir)[0]
              modified = set(
                  f"^/{re.escape(name)}"
                  for name in map(os.path.dirname, output.split('\n')) if name)
              self.debug(f"Limit to modified test dirs: {modified}", level=3)
              names.extend(modified)
  
          # Initialize the metadata tree, search for available tests
          self.debug(f"Check metadata tree in '{tree_path}'.")
          if self.opt('dry'):
              self._tests = []
              return
          tree = tmt.Tree(path=tree_path, context=self.step.plan._fmf_context())
          self._tests = tree.tests(
              filters=filters,
              names=names,
              conditions=["manual is False"],
              unique=False,
              links=links,
              excludes=excludes)
  
          # Prefix tests and handle library requires
          for test in self._tests:
              # Prefix test path with 'tests' and possible 'path' prefix
              test.path = os.path.join(prefix_path, test.path.lstrip('/'))
              # Check for possible required beakerlib libraries
              if test.require or test.recommend:
                  test.require, test.recommend, _ = tmt.beakerlib.dependencies(
                      test.require, test.recommend, parent=self)
  
          # Add TMT_SOURCE_DIR variable for each test
          if dist_git_source:
              for test in self._tests:
                  test.environment['TMT_SOURCE_DIR'] = sourcedir
  
      def tests(self):
          """ Return all discovered tests """
          return self._tests
  
files: tmt/steps/discover/fmf.py
files: import copy
  import os
  from typing import TYPE_CHECKING, List, Optional
  
  import click
  import fmf
  
  import tmt
  import tmt.steps
  import tmt.steps.discover
  
  if TYPE_CHECKING:
      import tmt.base
  
  
  @tmt.steps.provides_method('shell')
  class DiscoverShell(tmt.steps.discover.DiscoverPlugin):
      """
      Use provided list of shell script tests
  
      List of test cases to be executed can be defined manually directly
      in the plan as a list of dictionaries containing test name, actual
      test script and optionally a path to the test. Example config:
  
      discover:
          how: shell
          tests:
          - name: /help/main
            test: tmt --help
          - name: /help/test
            test: tmt test --help
          - name: /help/smoke
            test: ./smoke.sh
            path: /tests/shell
  
      For DistGit repo one can extract source tarball and use its code.
      It is extracted to TMT_SOURCE_DIR however no patches are applied
      (only source tarball is extracted).
  
      discover:
          how: shell
          dist-git-source: true
          tests:
          - name: /upstream
            test: cd $TMT_SOURCE_DIR/*/tests && make test
      """
  
      def show(self, keys: Optional[List[str]] = None) -> None:
          """ Show config details """
          super().show([])
          tests = self.get('tests')
          if tests:
              test_names = [test['name'] for test in tests]
              click.echo(tmt.utils.format('tests', test_names))
  
      def wake(self) -> None:
          """ Wake up the plugin, process data, apply options """
          super().wake()
          # Check provided tests, default to an empty list
          if 'tests' not in self.data:
              self.data['tests'] = []
          self._tests: List[tmt.base.Test] = []
  
      def go(self) -> None:
          """ Discover available tests """
          super(DiscoverShell, self).go()
          tests = fmf.Tree(dict(summary='tests'))
  
          # dist-git related
          assert self.workdir is not None
          sourcedir = os.path.join(self.workdir, 'source')
          dist_git_source = self.get('dist-git-source', False)
  
          # Check and process each defined shell test
          assert self.data['tests'] is not None
          for data in self.data['tests']:
              # Create data copy (we want to keep original data for save()
              data = copy.deepcopy(data)
              # Extract name, make sure it is present
              try:
                  name = data.pop('name')
              except KeyError:
                  raise tmt.utils.SpecificationError(
                      f"Missing test name in '{self.step.plan.name}'.")
              # Make sure that the test script is defined
              if 'test' not in data:
                  raise tmt.utils.SpecificationError(
                      f"Missing test script in '{self.step.plan.name}'.")
              # Prepare path to the test working directory (tree root by default)
              try:
                  data['path'] = f"/tests{data['path']}"
              except KeyError:
                  data['path'] = "/tests"
              # Apply default test duration unless provided
              if 'duration' not in data:
                  data['duration'] = tmt.base.DEFAULT_TEST_DURATION_L2
              # Add source dir path variable
              if dist_git_source:
                  data.setdefault('environment', {})[
                      'TMT_SOURCE_DIR'] = sourcedir
              # Create a simple fmf node, adjust its name
              tests.child(name, data)
  
          # Symlink tests directory to the plan work tree
          testdir = os.path.join(self.workdir, "tests")
          relative_path = os.path.relpath(self.step.plan.worktree, self.workdir)
          os.symlink(relative_path, testdir)
  
          if dist_git_source:
              try:
                  run_result = self.run(
                      ["git", "rev-parse", "--show-toplevel"],
                      cwd=self.step.plan.my_run.tree.root,
                      dry=True)[0]
                  assert run_result is not None
                  git_root = run_result.strip('\n')
              except tmt.utils.RunError:
                  raise tmt.utils.DiscoverError(
                      f"Directory '{self.step.plan.my_run.tree.root}' "
                      f"is not a git repository.")
              try:
                  self.extract_distgit_source(
                      git_root, sourcedir, self.get('dist-git-type'))
              except Exception as error:
                  raise tmt.utils.DiscoverError(
                      "Failed to process 'dist-git-source'.", original=error)
  
          # Use a tmt.Tree to apply possible command line filters
          tests = tmt.Tree(tree=tests).tests(conditions=["manual is False"])
          self._tests = tests
  
      def tests(self) -> List[tmt.base.Test]:
          return self._tests
  
files: tmt/steps/discover/shell.py
files: import copy
  import os
  from typing import TYPE_CHECKING, List, Optional
  
  import click
  import fmf
  
  import tmt
  import tmt.steps
  import tmt.steps.discover
  
  if TYPE_CHECKING:
      import tmt.base
  
  
  @tmt.steps.provides_method('shell')
  class DiscoverShell(tmt.steps.discover.DiscoverPlugin):
      """
      Use provided list of shell script tests
  
      List of test cases to be executed can be defined manually directly
      in the plan as a list of dictionaries containing test name, actual
      test script and optionally a path to the test. Example config:
  
      discover:
          how: shell
          tests:
          - name: /help/main
            test: tmt --help
          - name: /help/test
            test: tmt test --help
          - name: /help/smoke
            test: ./smoke.sh
            path: /tests/shell
  
      For DistGit repo one can extract source tarball and use its code.
      It is extracted to TMT_SOURCE_DIR however no patches are applied
      (only source tarball is extracted).
  
      discover:
          how: shell
          dist-git-source: true
          tests:
          - name: /upstream
            test: cd $TMT_SOURCE_DIR/*/tests && make test
      """
  
      def show(self, keys: Optional[List[str]] = None) -> None:
          """ Show config details """
          super().show([])
          tests = self.get('tests')
          if tests:
              test_names = [test['name'] for test in tests]
              click.echo(tmt.utils.format('tests', test_names))
  
      def wake(self, keys: Optional[List[str]] = None) -> None:
          """ Wake up the plugin, process data, apply options """
          super().wake(keys=keys)
          # Check provided tests, default to an empty list
          if 'tests' not in self.data:
              self.data['tests'] = []
          self._tests: List[tmt.base.Test] = []
  
      def go(self) -> None:
          """ Discover available tests """
          super(DiscoverShell, self).go()
          tests = fmf.Tree(dict(summary='tests'))
  
          # dist-git related
          assert self.workdir is not None
          sourcedir = os.path.join(self.workdir, 'source')
          dist_git_source = self.get('dist-git-source', False)
  
          # Check and process each defined shell test
          assert self.data['tests'] is not None
          for data in self.data['tests']:
              # Create data copy (we want to keep original data for save()
              data = copy.deepcopy(data)
              # Extract name, make sure it is present
              try:
                  name = data.pop('name')
              except KeyError:
                  raise tmt.utils.SpecificationError(
                      f"Missing test name in '{self.step.plan.name}'.")
              # Make sure that the test script is defined
              if 'test' not in data:
                  raise tmt.utils.SpecificationError(
                      f"Missing test script in '{self.step.plan.name}'.")
              # Prepare path to the test working directory (tree root by default)
              try:
                  data['path'] = f"/tests{data['path']}"
              except KeyError:
                  data['path'] = "/tests"
              # Apply default test duration unless provided
              if 'duration' not in data:
                  data['duration'] = tmt.base.DEFAULT_TEST_DURATION_L2
              # Add source dir path variable
              if dist_git_source:
                  data.setdefault('environment', {})[
                      'TMT_SOURCE_DIR'] = sourcedir
              # Create a simple fmf node, adjust its name
              tests.child(name, data)
  
          # Symlink tests directory to the plan work tree
          testdir = os.path.join(self.workdir, "tests")
          relative_path = os.path.relpath(self.step.plan.worktree, self.workdir)
          os.symlink(relative_path, testdir)
  
          if dist_git_source:
              try:
                  run_result = self.run(
                      ["git", "rev-parse", "--show-toplevel"],
                      cwd=self.step.plan.my_run.tree.root,
                      dry=True)[0]
                  assert run_result is not None
                  git_root = run_result.strip('\n')
              except tmt.utils.RunError:
                  raise tmt.utils.DiscoverError(
                      f"Directory '{self.step.plan.my_run.tree.root}' "
                      f"is not a git repository.")
              try:
                  self.extract_distgit_source(
                      git_root, sourcedir, self.get('dist-git-type'))
              except Exception as error:
                  raise tmt.utils.DiscoverError(
                      "Failed to process 'dist-git-source'.", original=error)
  
          # Use a tmt.Tree to apply possible command line filters
          tests = tmt.Tree(tree=tests).tests(conditions=["manual is False"])
          self._tests = tests
  
      def tests(self) -> List[tmt.base.Test]:
          return self._tests
  
files: tmt/steps/discover/shell.py
files: import json
  import os
  import sys
  import time
  
  import click
  
  import tmt
  import tmt.steps
  import tmt.steps.execute
  import tmt.utils
  from tmt.steps.execute import TEST_OUTPUT_FILENAME, Script
  
  # Script handling reboots, in restraint compatible fashion
  TMT_REBOOT_SCRIPT = Script("/usr/local/bin/tmt-reboot",
                             aliases=[
                                 "/usr/local/bin/rstrnt-reboot",
                                 "/usr/local/bin/rhts-reboot"],
                             related_variables=[
                                 "TMT_REBOOT_COUNT",
                                 "REBOOTCOUNT",
                                 "RSTRNT_REBOOTCOUNT"]
                             )
  
  # Script handling result reporting, in restraint compatible fashion
  TMT_REPORT_RESULT_SCRIPT = Script("/usr/local/bin/tmt-report-result",
                                    aliases=[
                                        "/usr/local/bin/rstrnt-report-result",
                                        "/usr/local/bin/rhts-report-result"],
                                    related_variables=[]
                                    )
  
  # Script for archiving a file, usable for BEAKERLIB_COMMAND_SUBMIT_LOG
  TMT_FILE_SUBMIT_SCRIPT = Script("/usr/local/bin/tmt-file-submit",
                                  aliases=[
                                      "/usr/local/bin/rstrnt-report-log",
                                      "/usr/local/bin/rhts-submit-log",
                                      "/usr/local/bin/rhts_submit_log"],
                                  related_variables=[]
                                  )
  
  # Script handling text execution abortion, in restraint compatible fashion
  TMT_ABORT_SCRIPT = Script("/usr/local/bin/tmt-abort",
                            aliases=[
                                "/usr/local/bin/rstrnt-abort",
                                "/usr/local/bin/rhts-abort"],
                            related_variables=[]
                            )
  
  # File for requesting reboot
  REBOOT_REQUEST_FILENAME = "reboot_request"
  
  # List of all available scripts
  SCRIPTS = (TMT_FILE_SUBMIT_SCRIPT,
             TMT_REBOOT_SCRIPT,
             TMT_REPORT_RESULT_SCRIPT,
             TMT_ABORT_SCRIPT
             )
  
  
  @tmt.steps.provides_method('tmt')
  @tmt.steps.provides_method('shell.tmt', order=80)
  @tmt.steps.provides_method('beakerlib.tmt', order=80)
  class ExecuteInternal(tmt.steps.execute.ExecutePlugin):
      """
      Use the internal tmt executor to execute tests
  
      The internal tmt executor runs tests on the guest one by one, shows
      testing progress and supports interactive debugging as well. Test
      result is based on the script exit code (for shell tests) or the
      results file (for beakerlib tests).
      """
  
      # Supported keys
      _keys = ["script", "interactive"]
  
      def __init__(self, *args, **kwargs):
          super().__init__(*args, **kwargs)
          self._previous_progress_message = ""
          self.scripts = SCRIPTS
  
      @classmethod
      def options(cls, how=None):
          """ Prepare command line options for given method """
          options = []
          # Shell script as a test
          options.append(click.option(
              '-s', '--script', metavar='SCRIPT', multiple=True,
              help='Shell script to be executed as a test.'))
          # Interactive mode
          options.append(click.option(
              '-i', '--interactive', is_flag=True,
              help='Run in interactive mode, do not capture output.'))
          # Disable interactive progress bar
          options.append(click.option(
              '--no-progress-bar', is_flag=True,
              help='Disable interactive progress bar showing the current test.'))
          return options + super().options(how)
  
      def wake(self):
          """ Wake up the plugin, process data, apply options """
          super().wake()
          # Make sure that script is a list
          tmt.utils.listify(self.data, keys=['script'])
  
      # TODO: consider switching to utils.updatable_message() - might need more
      # work, since use of _show_progress is split over several methods.
      def _show_progress(self, progress, test_name, finish=False):
          """
          Show an interactive progress bar in non-verbose mode.
  
          If the output is not an interactive terminal, or progress bar is
          disabled using an option, just output the message as info without
          utilising \r. If finish is True, overwrite the previous progress bar.
          """
          # Verbose mode outputs other information, using \r to
          # create a status bar wouldn't work.
          if self.opt('verbose'):
              return
  
          # No progress if terminal not attached or explicitly disabled
          if not sys.stdout.isatty() or self.opt('no-progress-bar'):
              return
  
          # For debug mode show just an info message (unless finishing)
          message = f"{test_name} [{progress}]" if not finish else ""
          if self.opt('debug'):
              if not finish:
                  self.info(message, shift=1)
              return
  
          # Show progress bar in an interactive shell.
          # We need to completely override the previous message, add
          # spaces if necessary.
          message = message.ljust(len(self._previous_progress_message))
          self._previous_progress_message = message
          message = self._indent('progress', message, color='cyan')
          sys.stdout.write(f"\r{message}")
          if finish:
              # The progress has been overwritten, return back to the start
              sys.stdout.write("\r")
              self._previous_progress_message = ""
          sys.stdout.flush()
  
      def _test_environment(self, test, extra_environment):
          """ Return test environment """
          data_directory = self.data_path(test, full=True, create=True)
  
          environment = extra_environment.copy()
          environment.update(test.environment)
          environment["TMT_TREE"] = self.parent.plan.worktree
          environment["TMT_TEST_DATA"] = os.path.join(
              data_directory, tmt.steps.execute.TEST_DATA)
          environment["TMT_REBOOT_REQUEST"] = os.path.join(
              data_directory,
              tmt.steps.execute.TEST_DATA,
              REBOOT_REQUEST_FILENAME)
          # Set all supported reboot variables
          for reboot_variable in TMT_REBOOT_SCRIPT.related_variables:
              environment[reboot_variable] = str(test._reboot_count)
          # Variables related to beakerlib tests
          if test.framework == 'beakerlib':
              environment['BEAKERLIB_DIR'] = data_directory
              environment['BEAKERLIB_COMMAND_SUBMIT_LOG'] = (
                  f"bash {TMT_FILE_SUBMIT_SCRIPT.path}")
  
          return environment
  
      def execute(self, test, guest, extra_environment):
          """ Run test on the guest """
          self.debug(f"Execute '{test.name}' as a '{test.framework}' test.")
  
          # Test will be executed in it's own directory, relative to the workdir
          workdir = os.path.join(self.discover.workdir, test.path.lstrip('/'))
          self.debug(f"Use workdir '{workdir}'.", level=3)
  
          # Create data directory, prepare test environment
          environment = self._test_environment(test, extra_environment)
  
          # Prepare the test command (use default options for shell tests)
          if test.framework == "shell":
              command = f"{tmt.utils.SHELL_OPTIONS}; {test.test}"
          else:
              command = test.test
  
          # Prepare custom function to log output in verbose mode
          def log(key, value=None, color=None, shift=1, level=1):
              self.verbose(key, value, color, shift=2, level=3)
  
          # Execute the test, save the output and return code
          start = time.time()
          try:
              stdout, stderr = guest.execute(
                  command, cwd=workdir, env=environment,
                  join=True, interactive=self.get('interactive'), log=log,
                  timeout=tmt.utils.duration_to_seconds(test.duration))
              test.returncode = 0
          except tmt.utils.RunError as error:
              stdout = error.stdout
              test.returncode = error.returncode
              if test.returncode == tmt.utils.PROCESS_TIMEOUT:
                  self.debug(f"Test duration '{test.duration}' exceeded.")
          end = time.time()
          self.write(
              self.data_path(test, TEST_OUTPUT_FILENAME, full=True),
              stdout or '', mode='a', level=3)
          test.real_duration = self.test_duration(start, end)
  
      def check(self, test):
          """ Check the test result """
          self.debug(f"Check result of '{test.name}'.")
          if test.framework == 'beakerlib':
              return self.check_beakerlib(test)
          else:
              try:
                  return self.check_result_file(test)
              except tmt.utils.FileError:
                  return self.check_shell(test)
  
      def _will_reboot(self, test):
          """ True if reboot is requested """
          return os.path.exists(self._reboot_request_path(test))
  
      def _reboot_request_path(self, test):
          """ Return reboot_request """
          reboot_request_path = os.path.join(
              self.data_path(test, full=True),
              tmt.steps.execute.TEST_DATA,
              REBOOT_REQUEST_FILENAME)
          return reboot_request_path
  
      def _handle_reboot(self, test, guest):
          """
          Reboot the guest if the test requested it.
  
          Check for presence of a file signalling reboot request
          and orchestrate the reboot if it was requested. Also increment
          REBOOTCOUNT variable, reset it to 0 if no reboot was requested
          (going forward to the next test). Return whether reboot was done.
          """
          if self._will_reboot(test):
              test._reboot_count += 1
              self.debug(f"Reboot during test '{test}' "
                         f"with reboot count {test._reboot_count}.")
              reboot_request_path = self._reboot_request_path(test)
              test_data = os.path.join(
                  self.data_path(test, full=True),
                  tmt.steps.execute.TEST_DATA)
              with open(reboot_request_path, 'r') as reboot_file:
                  reboot_data = json.loads(reboot_file.read())
              reboot_command = reboot_data.get('command')
              try:
                  timeout = int(reboot_data.get('timeout'))
              except ValueError:
                  timeout = None
              # Reset the file
              os.remove(reboot_request_path)
              guest.push(test_data)
              try:
                  guest.reboot(command=reboot_command, timeout=timeout)
              except tmt.utils.RunError:
                  self.fail(
                      f"Failed to reboot guest using the "
                      f"custom command '{reboot_command}'.")
                  raise
              except tmt.utils.ProvisionError:
                  self.warn(
                      "Guest does not support soft reboot, "
                      "trying hard reboot.")
                  guest.reboot(hard=True, timeout=timeout)
              return True
          return False
  
      def go(self, guest):
          """ Execute available tests """
          super().go(guest)
          self._results = []
  
          # Nothing to do in dry mode
          if self.opt('dry'):
              self._results = []
              return
  
          self._run_tests(guest)
  
      def _run_tests(self, guest, extra_environment=None):
          """ Execute tests on provided guest """
          extra_environment = extra_environment or {}
  
          # Prepare tests and helper scripts, check options
          tests = self.prepare_tests()
          exit_first = self.get('exit-first', default=False)
  
          # Prepare scripts, except localhost guest
          if not guest.localhost:
              self.prepare_scripts(guest)
  
          # Push workdir to guest and execute tests
          guest.push()
          # We cannot use enumerate here due to continue in the code
          index = 0
          while index < len(tests):
              test = tests[index]
              if not hasattr(test, "_reboot_count"):
                  test._reboot_count = 0
  
              progress = f"{index + 1}/{len(tests)}"
              self._show_progress(progress, test.name)
              self.verbose(
                  'test', test.summary or test.name, color='cyan', shift=1, level=2)
  
              self.execute(
                  test, guest,
                  extra_environment=extra_environment)
  
              # Pull test logs from the guest, exclude beakerlib backups
              if test.framework == "beakerlib":
                  exclude = [
                      "--exclude",
                      self.data_path(test, "backup*", full=True)]
              else:
                  exclude = None
              guest.pull(
                  source=self.data_path(test, full=True),
                  extend_options=exclude)
  
              result = self.check(test)
              duration = click.style(test.real_duration, fg='cyan')
              shift = 1 if self.opt('verbose') < 2 else 2
  
              # Handle reboot, abort, exit-first
              if self._will_reboot(test):
                  # Output before the reboot
                  self.verbose(
                      f"{duration} {test.name} [{progress}]", shift=shift)
                  if self._handle_reboot(test, guest):
                      continue
              abort = self.check_abort_file(test)
              if abort:
                  result.note = 'aborted'
              self._results.append(result)
              self.verbose(
                  f"{duration} {result.show()} [{progress}]", shift=shift)
              if (abort or exit_first and
                      result.result not in ('pass', 'info')):
                  # Clear the progress bar before outputting
                  self._show_progress('', '', True)
                  what_happened = "aborted" if abort else "failed"
                  self.warn(
                      f'Test {test.name} {what_happened}, stopping execution.')
                  break
              index += 1
          # Overwrite the progress bar, the test data is irrelevant
          self._show_progress('', '', True)
  
          # Pull artifacts created in the plan data directory
          self.debug("Pull the plan data directory.", level=2)
          guest.pull(source=self.step.plan.data_directory)
  
      def results(self):
          """ Return test results """
          return self._results
  
      def requires(self):
          """ Return list of required packages """
          # FIXME Remove when we drop support for the old execution methods
          return ['beakerlib'] if self.step._framework == 'beakerlib' else []
  
files: tmt/steps/execute/internal.py
files: import json
  import os
  import sys
  import time
  
  import click
  
  import tmt
  import tmt.steps
  import tmt.steps.execute
  import tmt.utils
  from tmt.steps.execute import TEST_OUTPUT_FILENAME, Script
  
  # Script handling reboots, in restraint compatible fashion
  TMT_REBOOT_SCRIPT = Script("/usr/local/bin/tmt-reboot",
                             aliases=[
                                 "/usr/local/bin/rstrnt-reboot",
                                 "/usr/local/bin/rhts-reboot"],
                             related_variables=[
                                 "TMT_REBOOT_COUNT",
                                 "REBOOTCOUNT",
                                 "RSTRNT_REBOOTCOUNT"]
                             )
  
  # Script handling result reporting, in restraint compatible fashion
  TMT_REPORT_RESULT_SCRIPT = Script("/usr/local/bin/tmt-report-result",
                                    aliases=[
                                        "/usr/local/bin/rstrnt-report-result",
                                        "/usr/local/bin/rhts-report-result"],
                                    related_variables=[]
                                    )
  
  # Script for archiving a file, usable for BEAKERLIB_COMMAND_SUBMIT_LOG
  TMT_FILE_SUBMIT_SCRIPT = Script("/usr/local/bin/tmt-file-submit",
                                  aliases=[
                                      "/usr/local/bin/rstrnt-report-log",
                                      "/usr/local/bin/rhts-submit-log",
                                      "/usr/local/bin/rhts_submit_log"],
                                  related_variables=[]
                                  )
  
  # Script handling text execution abortion, in restraint compatible fashion
  TMT_ABORT_SCRIPT = Script("/usr/local/bin/tmt-abort",
                            aliases=[
                                "/usr/local/bin/rstrnt-abort",
                                "/usr/local/bin/rhts-abort"],
                            related_variables=[]
                            )
  
  # File for requesting reboot
  REBOOT_REQUEST_FILENAME = "reboot_request"
  
  # List of all available scripts
  SCRIPTS = (TMT_FILE_SUBMIT_SCRIPT,
             TMT_REBOOT_SCRIPT,
             TMT_REPORT_RESULT_SCRIPT,
             TMT_ABORT_SCRIPT
             )
  
  
  @tmt.steps.provides_method('tmt')
  @tmt.steps.provides_method('shell.tmt', order=80)
  @tmt.steps.provides_method('beakerlib.tmt', order=80)
  class ExecuteInternal(tmt.steps.execute.ExecutePlugin):
      """
      Use the internal tmt executor to execute tests
  
      The internal tmt executor runs tests on the guest one by one, shows
      testing progress and supports interactive debugging as well. Test
      result is based on the script exit code (for shell tests) or the
      results file (for beakerlib tests).
      """
  
      # Supported keys
      _keys = ["script", "interactive"]
  
      def __init__(self, *args, **kwargs):
          super().__init__(*args, **kwargs)
          self._previous_progress_message = ""
          self.scripts = SCRIPTS
  
      @classmethod
      def options(cls, how=None):
          """ Prepare command line options for given method """
          options = []
          # Shell script as a test
          options.append(click.option(
              '-s', '--script', metavar='SCRIPT', multiple=True,
              help='Shell script to be executed as a test.'))
          # Interactive mode
          options.append(click.option(
              '-i', '--interactive', is_flag=True,
              help='Run in interactive mode, do not capture output.'))
          # Disable interactive progress bar
          options.append(click.option(
              '--no-progress-bar', is_flag=True,
              help='Disable interactive progress bar showing the current test.'))
          return options + super().options(how)
  
      def wake(self, keys=None):
          """ Wake up the plugin, process data, apply options """
          super().wake(keys=keys)
          # Make sure that script is a list
          tmt.utils.listify(self.data, keys=['script'])
  
      # TODO: consider switching to utils.updatable_message() - might need more
      # work, since use of _show_progress is split over several methods.
      def _show_progress(self, progress, test_name, finish=False):
          """
          Show an interactive progress bar in non-verbose mode.
  
          If the output is not an interactive terminal, or progress bar is
          disabled using an option, just output the message as info without
          utilising \r. If finish is True, overwrite the previous progress bar.
          """
          # Verbose mode outputs other information, using \r to
          # create a status bar wouldn't work.
          if self.opt('verbose'):
              return
  
          # No progress if terminal not attached or explicitly disabled
          if not sys.stdout.isatty() or self.opt('no-progress-bar'):
              return
  
          # For debug mode show just an info message (unless finishing)
          message = f"{test_name} [{progress}]" if not finish else ""
          if self.opt('debug'):
              if not finish:
                  self.info(message, shift=1)
              return
  
          # Show progress bar in an interactive shell.
          # We need to completely override the previous message, add
          # spaces if necessary.
          message = message.ljust(len(self._previous_progress_message))
          self._previous_progress_message = message
          message = self._indent('progress', message, color='cyan')
          sys.stdout.write(f"\r{message}")
          if finish:
              # The progress has been overwritten, return back to the start
              sys.stdout.write("\r")
              self._previous_progress_message = ""
          sys.stdout.flush()
  
      def _test_environment(self, test, extra_environment):
          """ Return test environment """
          data_directory = self.data_path(test, full=True, create=True)
  
          environment = extra_environment.copy()
          environment.update(test.environment)
          environment["TMT_TREE"] = self.parent.plan.worktree
          environment["TMT_TEST_DATA"] = os.path.join(
              data_directory, tmt.steps.execute.TEST_DATA)
          environment["TMT_REBOOT_REQUEST"] = os.path.join(
              data_directory,
              tmt.steps.execute.TEST_DATA,
              REBOOT_REQUEST_FILENAME)
          # Set all supported reboot variables
          for reboot_variable in TMT_REBOOT_SCRIPT.related_variables:
              environment[reboot_variable] = str(test._reboot_count)
          # Variables related to beakerlib tests
          if test.framework == 'beakerlib':
              environment['BEAKERLIB_DIR'] = data_directory
              environment['BEAKERLIB_COMMAND_SUBMIT_LOG'] = (
                  f"bash {TMT_FILE_SUBMIT_SCRIPT.path}")
  
          return environment
  
      def execute(self, test, guest, extra_environment):
          """ Run test on the guest """
          self.debug(f"Execute '{test.name}' as a '{test.framework}' test.")
  
          # Test will be executed in it's own directory, relative to the workdir
          workdir = os.path.join(self.discover.workdir, test.path.lstrip('/'))
          self.debug(f"Use workdir '{workdir}'.", level=3)
  
          # Create data directory, prepare test environment
          environment = self._test_environment(test, extra_environment)
  
          # Prepare the test command (use default options for shell tests)
          if test.framework == "shell":
              command = f"{tmt.utils.SHELL_OPTIONS}; {test.test}"
          else:
              command = test.test
  
          # Prepare custom function to log output in verbose mode
          def log(key, value=None, color=None, shift=1, level=1):
              self.verbose(key, value, color, shift=2, level=3)
  
          # Execute the test, save the output and return code
          start = time.time()
          try:
              stdout, stderr = guest.execute(
                  command, cwd=workdir, env=environment,
                  join=True, interactive=self.get('interactive'), log=log,
                  timeout=tmt.utils.duration_to_seconds(test.duration))
              test.returncode = 0
          except tmt.utils.RunError as error:
              stdout = error.stdout
              test.returncode = error.returncode
              if test.returncode == tmt.utils.PROCESS_TIMEOUT:
                  self.debug(f"Test duration '{test.duration}' exceeded.")
          end = time.time()
          self.write(
              self.data_path(test, TEST_OUTPUT_FILENAME, full=True),
              stdout or '', mode='a', level=3)
          test.real_duration = self.test_duration(start, end)
  
      def check(self, test):
          """ Check the test result """
          self.debug(f"Check result of '{test.name}'.")
          if test.framework == 'beakerlib':
              return self.check_beakerlib(test)
          else:
              try:
                  return self.check_result_file(test)
              except tmt.utils.FileError:
                  return self.check_shell(test)
  
      def _will_reboot(self, test):
          """ True if reboot is requested """
          return os.path.exists(self._reboot_request_path(test))
  
      def _reboot_request_path(self, test):
          """ Return reboot_request """
          reboot_request_path = os.path.join(
              self.data_path(test, full=True),
              tmt.steps.execute.TEST_DATA,
              REBOOT_REQUEST_FILENAME)
          return reboot_request_path
  
      def _handle_reboot(self, test, guest):
          """
          Reboot the guest if the test requested it.
  
          Check for presence of a file signalling reboot request
          and orchestrate the reboot if it was requested. Also increment
          REBOOTCOUNT variable, reset it to 0 if no reboot was requested
          (going forward to the next test). Return whether reboot was done.
          """
          if self._will_reboot(test):
              test._reboot_count += 1
              self.debug(f"Reboot during test '{test}' "
                         f"with reboot count {test._reboot_count}.")
              reboot_request_path = self._reboot_request_path(test)
              test_data = os.path.join(
                  self.data_path(test, full=True),
                  tmt.steps.execute.TEST_DATA)
              with open(reboot_request_path, 'r') as reboot_file:
                  reboot_data = json.loads(reboot_file.read())
              reboot_command = reboot_data.get('command')
              try:
                  timeout = int(reboot_data.get('timeout'))
              except ValueError:
                  timeout = None
              # Reset the file
              os.remove(reboot_request_path)
              guest.push(test_data)
              try:
                  guest.reboot(command=reboot_command, timeout=timeout)
              except tmt.utils.RunError:
                  self.fail(
                      f"Failed to reboot guest using the "
                      f"custom command '{reboot_command}'.")
                  raise
              except tmt.utils.ProvisionError:
                  self.warn(
                      "Guest does not support soft reboot, "
                      "trying hard reboot.")
                  guest.reboot(hard=True, timeout=timeout)
              return True
          return False
  
      def go(self, guest):
          """ Execute available tests """
          super().go(guest)
          self._results = []
  
          # Nothing to do in dry mode
          if self.opt('dry'):
              self._results = []
              return
  
          self._run_tests(guest)
  
      def _run_tests(self, guest, extra_environment=None):
          """ Execute tests on provided guest """
          extra_environment = extra_environment or {}
  
          # Prepare tests and helper scripts, check options
          tests = self.prepare_tests()
          exit_first = self.get('exit-first', default=False)
  
          # Prepare scripts, except localhost guest
          if not guest.localhost:
              self.prepare_scripts(guest)
  
          # Push workdir to guest and execute tests
          guest.push()
          # We cannot use enumerate here due to continue in the code
          index = 0
          while index < len(tests):
              test = tests[index]
              if not hasattr(test, "_reboot_count"):
                  test._reboot_count = 0
  
              progress = f"{index + 1}/{len(tests)}"
              self._show_progress(progress, test.name)
              self.verbose(
                  'test', test.summary or test.name, color='cyan', shift=1, level=2)
  
              self.execute(
                  test, guest,
                  extra_environment=extra_environment)
  
              # Pull test logs from the guest, exclude beakerlib backups
              if test.framework == "beakerlib":
                  exclude = [
                      "--exclude",
                      self.data_path(test, "backup*", full=True)]
              else:
                  exclude = None
              guest.pull(
                  source=self.data_path(test, full=True),
                  extend_options=exclude)
  
              result = self.check(test)
              duration = click.style(test.real_duration, fg='cyan')
              shift = 1 if self.opt('verbose') < 2 else 2
  
              # Handle reboot, abort, exit-first
              if self._will_reboot(test):
                  # Output before the reboot
                  self.verbose(
                      f"{duration} {test.name} [{progress}]", shift=shift)
                  if self._handle_reboot(test, guest):
                      continue
              abort = self.check_abort_file(test)
              if abort:
                  result.note = 'aborted'
              self._results.append(result)
              self.verbose(
                  f"{duration} {result.show()} [{progress}]", shift=shift)
              if (abort or exit_first and
                      result.result not in ('pass', 'info')):
                  # Clear the progress bar before outputting
                  self._show_progress('', '', True)
                  what_happened = "aborted" if abort else "failed"
                  self.warn(
                      f'Test {test.name} {what_happened}, stopping execution.')
                  break
              index += 1
          # Overwrite the progress bar, the test data is irrelevant
          self._show_progress('', '', True)
  
          # Pull artifacts created in the plan data directory
          self.debug("Pull the plan data directory.", level=2)
          guest.pull(source=self.step.plan.data_directory)
  
      def results(self):
          """ Return test results """
          return self._results
  
      def requires(self):
          """ Return list of required packages """
          # FIXME Remove when we drop support for the old execution methods
          return ['beakerlib'] if self.step._framework == 'beakerlib' else []
  
files: tmt/steps/execute/internal.py
files: from typing import Any, List, Optional
  
  import click
  import fmf
  
  import tmt
  import tmt.steps
  import tmt.steps.finish
  from tmt.steps.provision import Guest
  
  
  @tmt.steps.provides_method('shell')
  class FinishShell(tmt.steps.finish.FinishPlugin):
      """
      Perform finishing tasks using shell scripts
  
      Example config:
  
      finish:
          how: shell
          script:
              - upload-logs.sh || true
              - rm -rf /tmp/temporary-files
  
      Use the 'order' attribute to select in which order finishing tasks
      should happen if there are multiple configs. Default order is '50'.
      """
  
      # Supported keys
      _keys = ["script"]
  
      @classmethod
      def options(cls, how: Optional[str] = None) -> List[tmt.options.ClickOptionDecoratorType]:
          """ Finish command line options """
          options = super().options(how)
          options[:0] = [
              click.option(
                  '-s', '--script', metavar='SCRIPT',
                  help='Shell script to be executed.')
              ]
          return options
  
      def default(self, option: str, default: Optional[Any] = None) -> Any:
          """ Return default data for given option """
          if option == 'script':
              return []
          return default
  
      # TODO: use better types once superclass gains its annotations
      def wake(self) -> None:
          """ Wake up the plugin, process data, apply options """
          super().wake()
  
          # Convert to list if single script provided
          tmt.utils.listify(self.data, keys=['script'])
  
      def go(self, guest: Guest) -> None:
          """ Perform finishing tasks on given guest """
          super().go(guest)
  
          # Give a short summary
          scripts = self.get('script')
          overview = fmf.utils.listed(scripts, 'script')
          self.info('overview', f'{overview} found', 'green')
  
          # Execute each script on the guest
          for script in scripts:
              self.verbose('script', script, 'green')
              guest.execute(script, cwd=self.step.plan.worktree)
  
files: tmt/steps/finish/shell.py
files: from typing import Any, List, Optional
  
  import click
  import fmf
  
  import tmt
  import tmt.steps
  import tmt.steps.finish
  from tmt.steps.provision import Guest
  
  
  @tmt.steps.provides_method('shell')
  class FinishShell(tmt.steps.finish.FinishPlugin):
      """
      Perform finishing tasks using shell scripts
  
      Example config:
  
      finish:
          how: shell
          script:
              - upload-logs.sh || true
              - rm -rf /tmp/temporary-files
  
      Use the 'order' attribute to select in which order finishing tasks
      should happen if there are multiple configs. Default order is '50'.
      """
  
      # Supported keys
      _keys = ["script"]
  
      @classmethod
      def options(cls, how: Optional[str] = None) -> List[tmt.options.ClickOptionDecoratorType]:
          """ Finish command line options """
          options = super().options(how)
          options[:0] = [
              click.option(
                  '-s', '--script', metavar='SCRIPT',
                  help='Shell script to be executed.')
              ]
          return options
  
      def default(self, option: str, default: Optional[Any] = None) -> Any:
          """ Return default data for given option """
          if option == 'script':
              return []
          return default
  
      # TODO: use better types once superclass gains its annotations
      def wake(self, keys: Optional[List[str]] = None) -> None:
          """ Wake up the plugin, process data, apply options """
          super().wake(keys=keys)
  
          # Convert to list if single script provided
          tmt.utils.listify(self.data, keys=['script'])
  
      def go(self, guest: Guest) -> None:
          """ Perform finishing tasks on given guest """
          super().go(guest)
  
          # Give a short summary
          scripts = self.get('script')
          overview = fmf.utils.listed(scripts, 'script')
          self.info('overview', f'{overview} found', 'green')
  
          # Execute each script on the guest
          for script in scripts:
              self.verbose('script', script, 'green')
              guest.execute(script, cwd=self.step.plan.worktree)
  
files: tmt/steps/finish/shell.py
files: import sys
  from typing import Any, List, Optional
  
  import click
  
  import tmt
  import tmt.steps
  import tmt.steps.prepare
  import tmt.utils
  from tmt.steps import Step
  from tmt.steps.provision import Guest
  
  if sys.version_info >= (3, 8):
      from typing import TypedDict
  else:
      from typing_extensions import TypedDict
  
  StepDataType = TypedDict(
      'StepDataType',
      {
          'playbook': List[str],
          'playbooks': List[str]
          }
      )
  
  
  @tmt.steps.provides_method('ansible')
  class PrepareAnsible(tmt.steps.prepare.PreparePlugin):  # type: ignore[misc]
      """
      Prepare guest using ansible
  
      Single playbook config:
  
          prepare:
              how: ansible
              playbook: ansible/packages.yml
  
      Multiple playbooks config:
  
          prepare:
              how: ansible
              playbook:
                - playbook/one.yml
                - playbook/two.yml
                - playbook/three.yml
              extra-args: '-vvv'
  
      The playbook path should be relative to the metadata tree root.
      Use 'order' attribute to select in which order preparation should
      happen if there are multiple configs. Default order is '50'.
      Default order of required packages installation is '70'.
      """
  
      # Supported keys
      _keys = ["playbook", "extra-args"]
  
      def __init__(self, step: Step, data: StepDataType) -> None:
          """ Store plugin name, data and parent step """
          super().__init__(step, data)
          # Rename plural playbooks to singular
          if 'playbooks' in self.data:
              self.data['playbook'] = self.data.pop('playbooks')
  
      # TODO: fix types once superclass gains its annotations
      @classmethod
      def options(cls, how: Optional[str] = None) -> Any:
          """ Prepare command line options """
          return [
              click.option(
                  '-p', '--playbook', metavar='PLAYBOOK', multiple=True,
                  help='Path to an ansible playbook to run.'),
              click.option(
                  '--extra-args', metavar='EXTRA-ARGS',
                  help='Optional arguments for ansible-playbook.')
              ] + super().options(how)
  
      def default(self, option: str, default: Optional[Any] = None) -> Any:
          """ Return default data for given option """
          if option == 'playbook':
              return []
          return default
  
      # TODO: use better types once superclass gains its annotations
      def wake(self) -> None:
          """ Wake up the plugin, process data, apply options """
          super().wake()
  
          # Convert to list if necessary
          tmt.utils.listify(self.data, keys=['playbook'])
  
      def go(self, guest: Guest) -> None:
          """ Prepare the guests """
          super().go(guest)
  
          # Apply each playbook on the guest
          for playbook in self.get('playbook'):
              self.info('playbook', playbook, 'green')
              guest.ansible(playbook, self.get('extra-args'))
  
files: tmt/steps/prepare/ansible.py
files: import sys
  from typing import Any, List, Optional
  
  import click
  
  import tmt
  import tmt.steps
  import tmt.steps.prepare
  import tmt.utils
  from tmt.steps import Step
  from tmt.steps.provision import Guest
  
  if sys.version_info >= (3, 8):
      from typing import TypedDict
  else:
      from typing_extensions import TypedDict
  
  StepDataType = TypedDict(
      'StepDataType',
      {
          'playbook': List[str],
          'playbooks': List[str]
          }
      )
  
  
  @tmt.steps.provides_method('ansible')
  class PrepareAnsible(tmt.steps.prepare.PreparePlugin):  # type: ignore[misc]
      """
      Prepare guest using ansible
  
      Single playbook config:
  
          prepare:
              how: ansible
              playbook: ansible/packages.yml
  
      Multiple playbooks config:
  
          prepare:
              how: ansible
              playbook:
                - playbook/one.yml
                - playbook/two.yml
                - playbook/three.yml
              extra-args: '-vvv'
  
      The playbook path should be relative to the metadata tree root.
      Use 'order' attribute to select in which order preparation should
      happen if there are multiple configs. Default order is '50'.
      Default order of required packages installation is '70'.
      """
  
      # Supported keys
      _keys = ["playbook", "extra-args"]
  
      def __init__(self, step: Step, data: StepDataType) -> None:
          """ Store plugin name, data and parent step """
          super().__init__(step, data)
          # Rename plural playbooks to singular
          if 'playbooks' in self.data:
              self.data['playbook'] = self.data.pop('playbooks')
  
      # TODO: fix types once superclass gains its annotations
      @classmethod
      def options(cls, how: Optional[str] = None) -> Any:
          """ Prepare command line options """
          return [
              click.option(
                  '-p', '--playbook', metavar='PLAYBOOK', multiple=True,
                  help='Path to an ansible playbook to run.'),
              click.option(
                  '--extra-args', metavar='EXTRA-ARGS',
                  help='Optional arguments for ansible-playbook.')
              ] + super().options(how)
  
      def default(self, option: str, default: Optional[Any] = None) -> Any:
          """ Return default data for given option """
          if option == 'playbook':
              return []
          return default
  
      # TODO: use better types once superclass gains its annotations
      def wake(self, keys: Optional[List[str]] = None) -> None:
          """ Wake up the plugin, process data, apply options """
          super().wake(keys=keys)
  
          # Convert to list if necessary
          tmt.utils.listify(self.data, keys=['playbook'])
  
      def go(self, guest: Guest) -> None:
          """ Prepare the guests """
          super().go(guest)
  
          # Apply each playbook on the guest
          for playbook in self.get('playbook'):
              self.info('playbook', playbook, 'green')
              guest.ansible(playbook, self.get('extra-args'))
  
files: tmt/steps/prepare/ansible.py
files: import os
  import re
  import shutil
  
  import click
  import fmf
  
  import tmt
  import tmt.steps
  import tmt.steps.prepare
  
  COPR_URL = 'https://copr.fedorainfracloud.org/coprs'
  
  
  class InstallBase(tmt.utils.Common):
      """ Base class for installation implementations """
  
      # Each installer knows its package manager and copr plugin
      package_manager = None
      copr_plugin = None
  
      def __init__(self, parent, guest):
          """ Initialize installation data """
          super().__init__(parent=parent, relative_indent=0)
          self.guest = guest
  
          # Get package related data from the plugin
          self.packages = self.parent.get("package", [])
          self.directories = self.parent.get("directory", [])
          self.exclude = self.parent.get("exclude", [])
          if not self.packages and not self.directories:
              self.debug("No packages for installation found.", level=3)
  
          # Prepare package lists and installation command
          self.prepare_packages()
          self.prepare_sudo()
          self.prepare_command()
  
      def prepare_packages(self):
          """ Process package names and directories """
          self.local_packages = []
          self.debuginfo_packages = []
          self.repository_packages = []
  
          # Detect local, debuginfo and repository packages
          for package in self.packages:
              if package.endswith('.rpm'):
                  self.local_packages.append(package)
              elif re.search(r"-debug(info|source)(\.|$)", package):
                  # Strip the '-debuginfo' string from package name
                  # (installing with it doesn't work on RHEL7)
                  package = re.sub(r"-debuginfo((?=\.)|$)", "", package)
                  self.debuginfo_packages.append(package)
              else:
                  self.repository_packages.append(package)
  
          # Check rpm packages in local directories
          for directory in self.directories:
              self.info('directory', directory, 'green')
              if not os.path.isdir(directory):
                  raise tmt.utils.PrepareError(f"Packages directory '{directory}' not found.")
              for filename in os.listdir(directory):
                  if filename.endswith('.rpm'):
                      self.debug(f"Found rpm '{filename}'.", level=3)
                      self.local_packages.append(os.path.join(directory, filename))
  
      def prepare_sudo(self):
          """ Check if sudo is needed for installation """
          self.debug('Check if sudo is necessary.', level=2)
          user = self.guest.execute('whoami')[0].strip()
          self.sudo = '' if user == 'root' else 'sudo '
  
      def prepare_command(self):
          """ Prepare installation command"""
          raise NotImplementedError
  
      def prepare_repository(self, **kwargs):
          """ Configure additional repository """
          raise NotImplementedError
  
      def list_packages(self, package_list, title):
          """ Show package info and return quoted package names """
          # Show a brief summary by default
          if not self.opt('verbose'):
              summary = fmf.utils.listed(package_list, max=3)
              self.info(title, summary, 'green')
          # Provide a full list of packages in verbose mode
          else:
              summary = fmf.utils.listed(package_list, 'package')
              self.info(title, summary + ' requested', 'green')
              for package in sorted(package_list):
                  self.verbose(package, shift=1)
          # Return quoted package names
          return " ".join([tmt.utils.quote(package) for package in package_list])
  
      def enable_copr_epel6(self, copr):
          """ Manually enable copr repositories for epel6 """
          # Parse the copr repo name
          matched = re.match("^(@)?([^/]+)/([^/]+)$", copr)
          if not matched:
              raise tmt.utils.PrepareError(f"Invalid copr repository '{copr}'.")
          group, name, project = matched.groups()
          group = 'group_' if group else ''
          # Prepare the repo file url
          parts = [COPR_URL] + (['g'] if group else [])
          parts += [name, project, 'repo', 'epel-6']
          parts += [f"{group}{name}-{project}-epel-6.repo"]
          url = '/'.join(parts)
          # Download the repo file on guest
          try:
              self.guest.execute(f'curl -LOf {url}', cwd='/etc/yum.repos.d')
          except tmt.utils.RunError as error:
              if 'not found' in error.stderr.lower():
                  raise tmt.utils.PrepareError(
                      f"Copr repository '{copr}' not found.")
              raise
  
      def enable_copr(self):
          """ Enable requested copr repositories """
          coprs = self.parent.get('copr')
          if not coprs:
              return
          # Try to install copr plugin
          self.debug('Make sure the copr plugin is available.')
          try:
              self.guest.execute(
                  f'rpm -q {self.copr_plugin} || {self.command} install -y {self.copr_plugin}')
          # Enable repositories manually for epel6
          except tmt.utils.RunError:
              for copr in coprs:
                  self.info('copr', copr, 'green')
                  self.enable_copr_epel6(copr)
          # Enable repositories using copr plugin
          else:
              for copr in coprs:
                  self.info('copr', copr, 'green')
                  self.guest.execute(f'{self.command} copr enable -y {copr}')
  
      def prepare_install_local(self):
          """ Copy packages to the test system """
          self.rpms_directory = os.path.join(self.parent.step.workdir, 'rpms')
          os.makedirs(self.rpms_directory)
  
          # Copy local packages into workdir, push to guests
          for package in self.local_packages:
              self.verbose(os.path.basename(package), shift=1)
              self.debug(f"Copy '{package}' to '{self.rpms_directory}'.", level=3)
              shutil.copy(package, self.rpms_directory)
          self.guest.push()
  
      def install_from_repository(self):
          """ Default base install method for packages from repositories """
          pass
  
      def install_local(self):
          """ Default base install method for local packages """
          pass
  
      def install_debuginfo(self):
          """ Default base install method for debuginfo packages """
          pass
  
      def install(self):
          """ Perform the actual installation """
          if self.local_packages:
              self.prepare_install_local()
              self.install_local()
          if self.repository_packages:
              self.install_from_repository()
          if self.debuginfo_packages:
              self.install_debuginfo()
  
  
  class InstallDnf(InstallBase):
      """ Install packages using dnf """
  
      package_manager = "dnf"
      copr_plugin = "dnf-plugins-core"
  
      def prepare_command(self):
          """ Prepare installation command """
          self.options = '-y'
          self.skip = ' --skip-broken' if self.parent.get('missing') == 'skip' else ''
          for package in self.exclude:
              self.options += " --exclude " + tmt.utils.quote(package)
  
          self.command = f"{self.sudo}{self.package_manager}{self.skip}"
          self.debug(f"Using '{self.command}' for all package operations.")
          self.debug(f"Options for package operations are '{self.options}'")
  
      def install_local(self):
          """ Install copied local packages """
          # Use both dnf install/reinstall to get all packages refreshed
          # FIXME Simplify this once BZ#1831022 is fixed/implemeted.
          self.guest.execute(f"{self.command} install {self.options} {self.rpms_directory}/*")
          self.guest.execute(f"{self.command} reinstall {self.options} {self.rpms_directory}/*")
          summary = fmf.utils.listed(self.local_packages, 'local package')
          self.info('total', f"{summary} installed", 'green')
  
      def install_from_repository(self):
          """ Install packages from the repository """
          packages = self.list_packages(self.repository_packages, title="package")
          check = f'rpm -q --whatprovides {packages}'
          # Check and install
          self.guest.execute(f"{check} || {self.command} install {self.options} {packages}")
  
      def install_debuginfo(self):
          """ Install debuginfo packages """
          packages = self.list_packages(self.debuginfo_packages, title="debuginfo")
          # Make sure debuginfo-install is present on the target system
          self.guest.execute(f"{self.command} install -y /usr/bin/debuginfo-install")
          self.guest.execute(f"debuginfo-install -y {packages}")
  
  
  class InstallYum(InstallDnf):
      """ Install packages using yum """
  
      package_manager = "yum"
      copr_plugin = "yum-plugin-copr"
  
      def install_from_repository(self):
          """ Install packages from the repository """
          packages = self.list_packages(self.repository_packages, title="package")
          # Extra ignore/check for yum to workaround BZ#1920176
          check = f'rpm -q --whatprovides {packages}'
          final_check = " || true" if self.skip else f" && {check}"
          # Check and install
          self.guest.execute(
              f"{check} || {self.command} install {self.options} {packages}{final_check}")
  
  
  class InstallRpmOstree(InstallBase):
      """ Install packages using rpm-ostree """
  
      package_manager = "rpm-ostree"
      copr_plugin = "dnf-plugins-core"
  
      def sort_packages(self):
          """ Identify required and recommended packages """
          self.recommended_packages = []
          self.required_packages = []
          for package in self.repository_packages:
              try:
                  output = self.guest.execute(f"rpm -q --whatprovides '{package}'")
                  self.debug(f"Package '{output.stdout.strip()}' already installed.")
              except tmt.utils.RunError:
                  if self.skip:
                      self.recommended_packages.append(package)
                  else:
                      self.required_packages.append(package)
  
      def prepare_command(self):
          """ Prepare installation command for rpm-ostree"""
          self.skip = True if self.parent.get('missing') == 'skip' else False
          self.command = f"{self.sudo}rpm-ostree"
          self.options = '--apply-live --idempotent --allow-inactive'
          for package in self.exclude:
              # exclude not supported in rpm-ostree
              self.warn(f"there is no support for rpm-ostree exclude. "
                        f"Package '{package}' may still be installed.")
          self.debug(f"Using '{self.command}' for all package operations.")
          self.debug(f"Options for package operations are '{self.options}'.")
  
      def install_debuginfo(self):
          """ Install debuginfo packages """
          self.warn("Installation of debuginfo packages not supported yet.")
  
      def install_local(self):
          """ Install copied local packages """
          local_packages_installed = []
          for package in self.local_packages:
              try:
                  self.guest.execute(
                      f"{self.command} install {self.options} "
                      f"{self.rpms_directory}/{os.path.basename(package)}")
                  local_packages_installed.append(package)
              except tmt.utils.RunError as error:
                  self.warn(f"Local package '{package}' not installed: {error.stderr}")
          summary = fmf.utils.listed(local_packages_installed, 'local package')
          self.info('total', f"{summary} installed", 'green')
  
      def install_from_repository(self):
          """ Install packages from the repository """
          self.sort_packages()
  
          # Install recommended packages
          if self.recommended_packages:
              self.list_packages(self.recommended_packages, title="package")
              for package in self.recommended_packages:
                  try:
                      self.guest.execute(f"{self.command} install {self.options} '{package}'")
                  except tmt.utils.RunError as error:
                      if "error: Packages not found" in error.stderr and self.skip:
                          self.warn(f"No match for recommended package '{package}'.")
                          continue
                      raise
  
          # Install required packages
          if self.required_packages:
              packages = self.list_packages(self.required_packages, title="package")
              self.guest.execute(f"{self.command} install {self.options} {packages}")
  
  
  @tmt.steps.provides_method('install')
  class PrepareInstall(tmt.steps.prepare.PreparePlugin):
      """
      Install packages on the guest
  
      Example config:
  
          prepare:
              how: install
              copr: psss/tmt
              package: tmt-all
              missing: fail
  
      Use 'copr' for enabling desired copr repository and 'missing' to
      choose if missing packages should be silently ignored (skip) or a
      preparation error should be reported (fail), which is the default.
  
      In addition to package name you can also use one or more paths to
      local rpm files to be installed:
  
          prepare:
              how: install
              package:
                  - tmp/RPMS/noarch/tmt-0.15-1.fc31.noarch.rpm
                  - tmp/RPMS/noarch/python3-tmt-0.15-1.fc31.noarch.rpm
  
      Use 'directory' to install all packages from given folder and
      'exclude' to skip selected packages:
  
          prepare:
              how: install
              directory: tmp/RPMS/noarch
              exclude: tmt-provision-virtual
  
      Use 'order' attribute to select in which order preparation should
      happen if there are multiple configs. Default order is '50'.
      Default order of required packages installation is '70'.
  
      Note: When testing ostree booted deployments tmt will use
      `rpm-ostree` as the package manager to perform the installation of
      requested packages. The current limitations of the rpm-ostree
      implementation are:
  
          Cannot install new version of already installed local rpm.
          No support for installing debuginfo packages at this time.
      """
  
      # Supported keys
      _keys = ["package", "directory", "copr", "exclude", "missing"]
  
      @classmethod
      def options(cls, how=None):
          """ Prepare command line options """
          return [
              click.option(
                  '-p', '--package', metavar='PACKAGE', multiple=True,
                  help='Package name or path to rpm to be installed.'),
              click.option(
                  '-D', '--directory', metavar='PATH', multiple=True,
                  help='Path to a local directory with rpm packages.'),
              click.option(
                  '-c', '--copr', metavar='REPO', multiple=True,
                  help='Copr repository to be enabled.'),
              click.option(
                  '-x', '--exclude', metavar='PACKAGE', multiple=True,
                  help='Packages to be skipped during installation.'),
              click.option(
                  '-m', '--missing', metavar='ACTION',
                  type=click.Choice(['fail', 'skip']),
                  help='Action on missing packages, fail (default) or skip.'),
              ] + super().options(how)
  
      def default(self, option, default=None):
          """ Return default data for given option """
          if option == 'missing':
              return 'fail'
          if option == 'exclude':
              return []
          return default
  
      def wake(self):
          """ Wake up the plugin, process data, apply options """
          super().wake()
          # Convert to list if necessary
          tmt.utils.listify(
              self.data, split=True,
              keys=['package', 'directory', 'copr', 'exclude'])
  
      def go(self, guest):
          """ Perform preparation for the guests"""
          super().go(guest)
  
          # Nothing to do in dry mode
          if self.opt('dry'):
              return
  
          # Pick the right implementation
          try:
              guest.execute('stat /run/ostree-booted')
              installer = InstallRpmOstree(parent=self, guest=guest)
          except tmt.utils.RunError:
              try:
                  guest.execute('rpm -q dnf')
                  installer = InstallDnf(parent=self, guest=guest)
              except tmt.utils.RunError:
                  installer = InstallYum(parent=self, guest=guest)
  
          # Enable copr repositories and install packages
          installer.enable_copr()
          installer.install()
  
files: tmt/steps/prepare/install.py
files: import os
  import re
  import shutil
  
  import click
  import fmf
  
  import tmt
  import tmt.steps
  import tmt.steps.prepare
  
  COPR_URL = 'https://copr.fedorainfracloud.org/coprs'
  
  
  class InstallBase(tmt.utils.Common):
      """ Base class for installation implementations """
  
      # Each installer knows its package manager and copr plugin
      package_manager = None
      copr_plugin = None
  
      def __init__(self, parent, guest):
          """ Initialize installation data """
          super().__init__(parent=parent, relative_indent=0)
          self.guest = guest
  
          # Get package related data from the plugin
          self.packages = self.parent.get("package", [])
          self.directories = self.parent.get("directory", [])
          self.exclude = self.parent.get("exclude", [])
          if not self.packages and not self.directories:
              self.debug("No packages for installation found.", level=3)
  
          # Prepare package lists and installation command
          self.prepare_packages()
          self.prepare_sudo()
          self.prepare_command()
  
      def prepare_packages(self):
          """ Process package names and directories """
          self.local_packages = []
          self.debuginfo_packages = []
          self.repository_packages = []
  
          # Detect local, debuginfo and repository packages
          for package in self.packages:
              if package.endswith('.rpm'):
                  self.local_packages.append(package)
              elif re.search(r"-debug(info|source)(\.|$)", package):
                  # Strip the '-debuginfo' string from package name
                  # (installing with it doesn't work on RHEL7)
                  package = re.sub(r"-debuginfo((?=\.)|$)", "", package)
                  self.debuginfo_packages.append(package)
              else:
                  self.repository_packages.append(package)
  
          # Check rpm packages in local directories
          for directory in self.directories:
              self.info('directory', directory, 'green')
              if not os.path.isdir(directory):
                  raise tmt.utils.PrepareError(f"Packages directory '{directory}' not found.")
              for filename in os.listdir(directory):
                  if filename.endswith('.rpm'):
                      self.debug(f"Found rpm '{filename}'.", level=3)
                      self.local_packages.append(os.path.join(directory, filename))
  
      def prepare_sudo(self):
          """ Check if sudo is needed for installation """
          self.debug('Check if sudo is necessary.', level=2)
          user = self.guest.execute('whoami')[0].strip()
          self.sudo = '' if user == 'root' else 'sudo '
  
      def prepare_command(self):
          """ Prepare installation command"""
          raise NotImplementedError
  
      def prepare_repository(self, **kwargs):
          """ Configure additional repository """
          raise NotImplementedError
  
      def list_packages(self, package_list, title):
          """ Show package info and return quoted package names """
          # Show a brief summary by default
          if not self.opt('verbose'):
              summary = fmf.utils.listed(package_list, max=3)
              self.info(title, summary, 'green')
          # Provide a full list of packages in verbose mode
          else:
              summary = fmf.utils.listed(package_list, 'package')
              self.info(title, summary + ' requested', 'green')
              for package in sorted(package_list):
                  self.verbose(package, shift=1)
          # Return quoted package names
          return " ".join([tmt.utils.quote(package) for package in package_list])
  
      def enable_copr_epel6(self, copr):
          """ Manually enable copr repositories for epel6 """
          # Parse the copr repo name
          matched = re.match("^(@)?([^/]+)/([^/]+)$", copr)
          if not matched:
              raise tmt.utils.PrepareError(f"Invalid copr repository '{copr}'.")
          group, name, project = matched.groups()
          group = 'group_' if group else ''
          # Prepare the repo file url
          parts = [COPR_URL] + (['g'] if group else [])
          parts += [name, project, 'repo', 'epel-6']
          parts += [f"{group}{name}-{project}-epel-6.repo"]
          url = '/'.join(parts)
          # Download the repo file on guest
          try:
              self.guest.execute(f'curl -LOf {url}', cwd='/etc/yum.repos.d')
          except tmt.utils.RunError as error:
              if 'not found' in error.stderr.lower():
                  raise tmt.utils.PrepareError(
                      f"Copr repository '{copr}' not found.")
              raise
  
      def enable_copr(self):
          """ Enable requested copr repositories """
          coprs = self.parent.get('copr')
          if not coprs:
              return
          # Try to install copr plugin
          self.debug('Make sure the copr plugin is available.')
          try:
              self.guest.execute(
                  f'rpm -q {self.copr_plugin} || {self.command} install -y {self.copr_plugin}')
          # Enable repositories manually for epel6
          except tmt.utils.RunError:
              for copr in coprs:
                  self.info('copr', copr, 'green')
                  self.enable_copr_epel6(copr)
          # Enable repositories using copr plugin
          else:
              for copr in coprs:
                  self.info('copr', copr, 'green')
                  self.guest.execute(f'{self.command} copr enable -y {copr}')
  
      def prepare_install_local(self):
          """ Copy packages to the test system """
          self.rpms_directory = os.path.join(self.parent.step.workdir, 'rpms')
          os.makedirs(self.rpms_directory)
  
          # Copy local packages into workdir, push to guests
          for package in self.local_packages:
              self.verbose(os.path.basename(package), shift=1)
              self.debug(f"Copy '{package}' to '{self.rpms_directory}'.", level=3)
              shutil.copy(package, self.rpms_directory)
          self.guest.push()
  
      def install_from_repository(self):
          """ Default base install method for packages from repositories """
          pass
  
      def install_local(self):
          """ Default base install method for local packages """
          pass
  
      def install_debuginfo(self):
          """ Default base install method for debuginfo packages """
          pass
  
      def install(self):
          """ Perform the actual installation """
          if self.local_packages:
              self.prepare_install_local()
              self.install_local()
          if self.repository_packages:
              self.install_from_repository()
          if self.debuginfo_packages:
              self.install_debuginfo()
  
  
  class InstallDnf(InstallBase):
      """ Install packages using dnf """
  
      package_manager = "dnf"
      copr_plugin = "dnf-plugins-core"
  
      def prepare_command(self):
          """ Prepare installation command """
          self.options = '-y'
          self.skip = ' --skip-broken' if self.parent.get('missing') == 'skip' else ''
          for package in self.exclude:
              self.options += " --exclude " + tmt.utils.quote(package)
  
          self.command = f"{self.sudo}{self.package_manager}{self.skip}"
          self.debug(f"Using '{self.command}' for all package operations.")
          self.debug(f"Options for package operations are '{self.options}'")
  
      def install_local(self):
          """ Install copied local packages """
          # Use both dnf install/reinstall to get all packages refreshed
          # FIXME Simplify this once BZ#1831022 is fixed/implemeted.
          self.guest.execute(f"{self.command} install {self.options} {self.rpms_directory}/*")
          self.guest.execute(f"{self.command} reinstall {self.options} {self.rpms_directory}/*")
          summary = fmf.utils.listed(self.local_packages, 'local package')
          self.info('total', f"{summary} installed", 'green')
  
      def install_from_repository(self):
          """ Install packages from the repository """
          packages = self.list_packages(self.repository_packages, title="package")
          check = f'rpm -q --whatprovides {packages}'
          # Check and install
          self.guest.execute(f"{check} || {self.command} install {self.options} {packages}")
  
      def install_debuginfo(self):
          """ Install debuginfo packages """
          packages = self.list_packages(self.debuginfo_packages, title="debuginfo")
          # Make sure debuginfo-install is present on the target system
          self.guest.execute(f"{self.command} install -y /usr/bin/debuginfo-install")
          self.guest.execute(f"debuginfo-install -y {packages}")
  
  
  class InstallYum(InstallDnf):
      """ Install packages using yum """
  
      package_manager = "yum"
      copr_plugin = "yum-plugin-copr"
  
      def install_from_repository(self):
          """ Install packages from the repository """
          packages = self.list_packages(self.repository_packages, title="package")
          # Extra ignore/check for yum to workaround BZ#1920176
          check = f'rpm -q --whatprovides {packages}'
          final_check = " || true" if self.skip else f" && {check}"
          # Check and install
          self.guest.execute(
              f"{check} || {self.command} install {self.options} {packages}{final_check}")
  
  
  class InstallRpmOstree(InstallBase):
      """ Install packages using rpm-ostree """
  
      package_manager = "rpm-ostree"
      copr_plugin = "dnf-plugins-core"
  
      def sort_packages(self):
          """ Identify required and recommended packages """
          self.recommended_packages = []
          self.required_packages = []
          for package in self.repository_packages:
              try:
                  output = self.guest.execute(f"rpm -q --whatprovides '{package}'")
                  self.debug(f"Package '{output.stdout.strip()}' already installed.")
              except tmt.utils.RunError:
                  if self.skip:
                      self.recommended_packages.append(package)
                  else:
                      self.required_packages.append(package)
  
      def prepare_command(self):
          """ Prepare installation command for rpm-ostree"""
          self.skip = True if self.parent.get('missing') == 'skip' else False
          self.command = f"{self.sudo}rpm-ostree"
          self.options = '--apply-live --idempotent --allow-inactive'
          for package in self.exclude:
              # exclude not supported in rpm-ostree
              self.warn(f"there is no support for rpm-ostree exclude. "
                        f"Package '{package}' may still be installed.")
          self.debug(f"Using '{self.command}' for all package operations.")
          self.debug(f"Options for package operations are '{self.options}'.")
  
      def install_debuginfo(self):
          """ Install debuginfo packages """
          self.warn("Installation of debuginfo packages not supported yet.")
  
      def install_local(self):
          """ Install copied local packages """
          local_packages_installed = []
          for package in self.local_packages:
              try:
                  self.guest.execute(
                      f"{self.command} install {self.options} "
                      f"{self.rpms_directory}/{os.path.basename(package)}")
                  local_packages_installed.append(package)
              except tmt.utils.RunError as error:
                  self.warn(f"Local package '{package}' not installed: {error.stderr}")
          summary = fmf.utils.listed(local_packages_installed, 'local package')
          self.info('total', f"{summary} installed", 'green')
  
      def install_from_repository(self):
          """ Install packages from the repository """
          self.sort_packages()
  
          # Install recommended packages
          if self.recommended_packages:
              self.list_packages(self.recommended_packages, title="package")
              for package in self.recommended_packages:
                  try:
                      self.guest.execute(f"{self.command} install {self.options} '{package}'")
                  except tmt.utils.RunError as error:
                      if "error: Packages not found" in error.stderr and self.skip:
                          self.warn(f"No match for recommended package '{package}'.")
                          continue
                      raise
  
          # Install required packages
          if self.required_packages:
              packages = self.list_packages(self.required_packages, title="package")
              self.guest.execute(f"{self.command} install {self.options} {packages}")
  
  
  @tmt.steps.provides_method('install')
  class PrepareInstall(tmt.steps.prepare.PreparePlugin):
      """
      Install packages on the guest
  
      Example config:
  
          prepare:
              how: install
              copr: psss/tmt
              package: tmt-all
              missing: fail
  
      Use 'copr' for enabling desired copr repository and 'missing' to
      choose if missing packages should be silently ignored (skip) or a
      preparation error should be reported (fail), which is the default.
  
      In addition to package name you can also use one or more paths to
      local rpm files to be installed:
  
          prepare:
              how: install
              package:
                  - tmp/RPMS/noarch/tmt-0.15-1.fc31.noarch.rpm
                  - tmp/RPMS/noarch/python3-tmt-0.15-1.fc31.noarch.rpm
  
      Use 'directory' to install all packages from given folder and
      'exclude' to skip selected packages:
  
          prepare:
              how: install
              directory: tmp/RPMS/noarch
              exclude: tmt-provision-virtual
  
      Use 'order' attribute to select in which order preparation should
      happen if there are multiple configs. Default order is '50'.
      Default order of required packages installation is '70'.
  
      Note: When testing ostree booted deployments tmt will use
      `rpm-ostree` as the package manager to perform the installation of
      requested packages. The current limitations of the rpm-ostree
      implementation are:
  
          Cannot install new version of already installed local rpm.
          No support for installing debuginfo packages at this time.
      """
  
      # Supported keys
      _keys = ["package", "directory", "copr", "exclude", "missing"]
  
      @classmethod
      def options(cls, how=None):
          """ Prepare command line options """
          return [
              click.option(
                  '-p', '--package', metavar='PACKAGE', multiple=True,
                  help='Package name or path to rpm to be installed.'),
              click.option(
                  '-D', '--directory', metavar='PATH', multiple=True,
                  help='Path to a local directory with rpm packages.'),
              click.option(
                  '-c', '--copr', metavar='REPO', multiple=True,
                  help='Copr repository to be enabled.'),
              click.option(
                  '-x', '--exclude', metavar='PACKAGE', multiple=True,
                  help='Packages to be skipped during installation.'),
              click.option(
                  '-m', '--missing', metavar='ACTION',
                  type=click.Choice(['fail', 'skip']),
                  help='Action on missing packages, fail (default) or skip.'),
              ] + super().options(how)
  
      def default(self, option, default=None):
          """ Return default data for given option """
          if option == 'missing':
              return 'fail'
          if option == 'exclude':
              return []
          return default
  
      def wake(self, keys=None):
          """ Wake up the plugin, process data, apply options """
          super().wake(keys=keys)
          # Convert to list if necessary
          tmt.utils.listify(
              self.data, split=True,
              keys=['package', 'directory', 'copr', 'exclude'])
  
      def go(self, guest):
          """ Perform preparation for the guests"""
          super().go(guest)
  
          # Nothing to do in dry mode
          if self.opt('dry'):
              return
  
          # Pick the right implementation
          try:
              guest.execute('stat /run/ostree-booted')
              installer = InstallRpmOstree(parent=self, guest=guest)
          except tmt.utils.RunError:
              try:
                  guest.execute('rpm -q dnf')
                  installer = InstallDnf(parent=self, guest=guest)
              except tmt.utils.RunError:
                  installer = InstallYum(parent=self, guest=guest)
  
          # Enable copr repositories and install packages
          installer.enable_copr()
          installer.install()
  
files: tmt/steps/prepare/install.py
files: from typing import Any, Optional
  
  import click
  import fmf
  
  import tmt
  import tmt.steps
  import tmt.steps.prepare
  import tmt.utils
  from tmt.steps.provision import Guest
  
  
  # TODO: drop ignore once type annotations between modules enabled
  @tmt.steps.provides_method('shell')
  class PrepareShell(tmt.steps.prepare.PreparePlugin):  # type: ignore[misc]
      """
      Prepare guest using shell scripts
  
      Example config:
  
      prepare:
          how: shell
          script:
          - sudo dnf install -y 'dnf-command(copr)'
          - sudo dnf copr enable -y psss/tmt
          - sudo dnf install -y tmt
  
      Use 'order' attribute to select in which order preparation should
      happen if there are multiple configs. Default order is '50'.
      Default order of required packages installation is '70'.
      """
  
      # Supported keys
      _keys = ["script"]
  
      @classmethod
      def options(cls, how: Optional[str] = None) -> Any:
          """ Prepare command line options """
          return [
              click.option(
                  '-s', '--script', metavar='SCRIPT',
                  help='Shell script to be executed.')
              ] + super().options(how)
  
      def default(self, option: str, default: Optional[Any] = None) -> Any:
          """ Return default data for given option """
          if option == 'script':
              return []
          return default
  
      def wake(self) -> None:
          """ Wake up the plugin, process data, apply options """
          super().wake()
  
          # Convert to list if single script provided
          tmt.utils.listify(self.data, keys=['script'])
  
      def go(self, guest: Guest) -> None:
          """ Prepare the guests """
          super().go(guest)
  
          # Give a short summary
          scripts = self.get('script')
          overview = fmf.utils.listed(scripts, 'script')
          self.info('overview', f'{overview} found', 'green')
  
          # Execute each script on the guest (with default shell options)
          for script in scripts:
              self.verbose('script', script, 'green')
              script_with_options = f'{tmt.utils.SHELL_OPTIONS}; {script}'
              guest.execute(script_with_options, cwd=self.step.plan.worktree)
  
files: tmt/steps/prepare/shell.py
files: from typing import Any, List, Optional
  
  import click
  import fmf
  
  import tmt
  import tmt.steps
  import tmt.steps.prepare
  import tmt.utils
  from tmt.steps.provision import Guest
  
  
  # TODO: drop ignore once type annotations between modules enabled
  @tmt.steps.provides_method('shell')
  class PrepareShell(tmt.steps.prepare.PreparePlugin):  # type: ignore[misc]
      """
      Prepare guest using shell scripts
  
      Example config:
  
      prepare:
          how: shell
          script:
          - sudo dnf install -y 'dnf-command(copr)'
          - sudo dnf copr enable -y psss/tmt
          - sudo dnf install -y tmt
  
      Use 'order' attribute to select in which order preparation should
      happen if there are multiple configs. Default order is '50'.
      Default order of required packages installation is '70'.
      """
  
      # Supported keys
      _keys = ["script"]
  
      @classmethod
      def options(cls, how: Optional[str] = None) -> Any:
          """ Prepare command line options """
          return [
              click.option(
                  '-s', '--script', metavar='SCRIPT',
                  help='Shell script to be executed.')
              ] + super().options(how)
  
      def default(self, option: str, default: Optional[Any] = None) -> Any:
          """ Return default data for given option """
          if option == 'script':
              return []
          return default
  
      def wake(self, keys: Optional[List[str]] = None) -> None:
          """ Wake up the plugin, process data, apply options """
          super().wake(keys=keys)
  
          # Convert to list if single script provided
          tmt.utils.listify(self.data, keys=['script'])
  
      def go(self, guest: Guest) -> None:
          """ Prepare the guests """
          super().go(guest)
  
          # Give a short summary
          scripts = self.get('script')
          overview = fmf.utils.listed(scripts, 'script')
          self.info('overview', f'{overview} found', 'green')
  
          # Execute each script on the guest (with default shell options)
          for script in scripts:
              self.verbose('script', script, 'green')
              script_with_options = f'{tmt.utils.SHELL_OPTIONS}; {script}'
              guest.execute(script_with_options, cwd=self.step.plan.worktree)
  
files: tmt/steps/prepare/shell.py
files: import collections
  import dataclasses
  import datetime
  import enum
  import os
  import random
  import re
  import shlex
  import string
  import subprocess
  import tempfile
  from shlex import quote
  from typing import Any, Dict, List, Optional, Type, Union, cast
  
  import click
  import fmf
  
  import tmt
  import tmt.plugins
  import tmt.steps
  import tmt.utils
  from tmt.steps import Action
  
  # Timeout in seconds of waiting for a connection after reboot
  CONNECTION_TIMEOUT = 5 * 60
  
  # When waiting for guest to recover from reboot, try re-connecting every
  # this many seconds.
  RECONNECT_WAIT_TICK = 5
  RECONNECT_WAIT_TICK_INCREASE = 1.0
  
  # Default rsync options
  DEFAULT_RSYNC_OPTIONS = [
      "-R", "-r", "-z", "--links", "--safe-links", "--delete"]
  
  DEFAULT_RSYNC_PUSH_OPTIONS = ["-R", "-r", "-z", "--links", "--safe-links", "--delete"]
  DEFAULT_RSYNC_PULL_OPTIONS = ["-R", "-r", "-z", "--links", "--safe-links", "--protect-args"]
  
  
  class CheckRsyncOutcome(enum.Enum):
      ALREADY_INSTALLED = 'already-installed'
      INSTALLED = 'installed'
  
  
  class Provision(tmt.steps.Step):
      """ Provision an environment for testing or use localhost. """
  
      # Default implementation for provision is a virtual machine
      DEFAULT_HOW = 'virtual'
  
      def __init__(self, plan: 'tmt.Plan', data: tmt.steps.StepData) -> None:
          """ Initialize provision step data """
          super().__init__(plan=plan, data=data)
          # Check that the names are unique
          names = [data['name'] for data in self.data]
          names_count: Dict[str, int] = collections.defaultdict(int)
          for name in names:
              names_count[name] += 1
          if len(self.data) > 1 and len(self.data) != len(names_count):
              duplicate = [k for k, v in names_count.items() if v > 1]
              duplicate_string = ', '.join(duplicate)
              raise tmt.utils.GeneralError(
                  f"Provision step names must be unique for multihost testing. "
                  f"Duplicate names: {duplicate_string} in plan '{plan.name}'.")
          # List of provisioned guests and loaded guest data
          self._guests: List['Guest'] = []
          self._guest_data: Dict[str, 'GuestData'] = {}
          self.is_multihost = False
  
      def load(self) -> None:
          """ Load guest data from the workdir """
          super().load()
          try:
              raw_guest_data = tmt.utils.yaml_to_dict(self.read('guests.yaml'))
  
              self._guest_data = {
                  name: tmt.utils.SerializableContainer.unserialize(guest_data)
                  for name, guest_data in raw_guest_data.items()
                  }
  
          except tmt.utils.FileError:
              self.debug('Provisioned guests not found.', level=2)
  
      def save(self) -> None:
          """ Save guest data to the workdir """
          super().save()
          try:
              raw_guest_data = {guest.name: guest.save().to_serialized()
                                for guest in self.guests()}
  
              self.write('guests.yaml', tmt.utils.dict_to_yaml(raw_guest_data))
          except tmt.utils.FileError:
              self.debug('Failed to save provisioned guests.')
  
      def wake(self) -> None:
          """ Wake up the step (process workdir and command line) """
          super().wake()
  
          # Choose the right plugin and wake it up
          for data in self.data:
              # TODO: with generic BasePlugin, delegate() should return more fitting type,
              # not the base class.
              plugin = cast(ProvisionPlugin, ProvisionPlugin.delegate(self, data))
              self._phases.append(plugin)
              # If guest data loaded, perform a complete wake up
              plugin.wake(data=self._guest_data.get(plugin.name))
  
              guest = plugin.guest()
              if guest:
                  self._guests.append(guest)
  
          # Nothing more to do if already done
          if self.status() == 'done':
              self.debug(
                  'Provision wake up complete (already done before).', level=2)
          # Save status and step data (now we know what to do)
          else:
              self.status('todo')
              self.save()
  
      def show(self) -> None:
          """ Show discover details """
          for data in self.data:
              ProvisionPlugin.delegate(self, data).show()
  
      def summary(self) -> None:
          """ Give a concise summary of the provisioning """
          # Summary of provisioned guests
          guests = fmf.utils.listed(self.guests(), 'guest')
          self.info('summary', f'{guests} provisioned', 'green', shift=1)
          # Guest list in verbose mode
          for guest in self.guests():
              if not guest.name.startswith(tmt.utils.DEFAULT_NAME):
                  self.verbose(guest.name, color='red', shift=2)
  
      def go(self) -> None:
          """ Provision all guests"""
          super().go()
  
          # Nothing more to do if already done
          if self.status() == 'done':
              self.info('status', 'done', 'green', shift=1)
              self.summary()
              self.actions()
              return
  
          # Provision guests
          self._guests = []
          save = True
          self.is_multihost = sum([isinstance(phase, ProvisionPlugin)
                                  for phase in self.phases()]) > 1
          try:
              for phase in self.phases(classes=(Action, ProvisionPlugin)):
                  try:
                      if isinstance(phase, Action):
                          phase.go()
  
                      elif isinstance(phase, ProvisionPlugin):
                          phase.go()
  
                          guest = phase.guest()
                          if guest:
                              guest.details()
  
                      if self.is_multihost:
                          self.info('')
                  finally:
                      if isinstance(phase, ProvisionPlugin):
                          guest = phase.guest()
                          if guest:
                              self._guests.append(guest)
  
              # Give a summary, update status and save
              self.summary()
              self.status('done')
          except (SystemExit, tmt.utils.SpecificationError) as error:
              # A plugin will only raise SystemExit if the exit is really desired
              # and no other actions should be done. An example of this is
              # listing available images. In such case, the workdir is deleted
              # as it's redundant and save() would throw an error.
              save = False
              raise error
          finally:
              if save:
                  self.save()
  
      def guests(self) -> List['Guest']:
          """ Return the list of all provisioned guests """
          return self._guests
  
      def requires(self) -> List[str]:
          """
          Packages required by all enabled provision plugins
  
          Return a list of packages which need to be installed on the
          provisioned guest so that the workdir can be synced to it.
          Used by the prepare step.
          """
          requires = set()
          for plugin in self.phases(classes=ProvisionPlugin):
              requires.update(plugin.requires())
          return list(requires)
  
  
  class ProvisionPlugin(tmt.steps.GuestlessPlugin):
      """ Common parent of provision plugins """
  
      # Default implementation for provision is a virtual machine
      how = 'virtual'
  
      # List of all supported methods aggregated from all plugins of the same step.
      _supported_methods: List[tmt.steps.Method] = []
  
      # Common keys for all provision step implementations
      _common_keys = ['role']
  
      @classmethod
      def base_command(
              cls,
              usage: str,
              method_class: Optional[Type[click.Command]] = None) -> click.Command:
          """ Create base click command (common for all provision plugins) """
  
          # Prepare general usage message for the step
          if method_class:
              usage = Provision.usage(method_overview=usage)
  
          # Create the command
          @click.command(cls=method_class, help=usage)
          @click.pass_context
          @click.option(
              '-h', '--how', metavar='METHOD',
              help='Use specified method for provisioning.')
          def provision(context: click.Context, **kwargs: Any) -> None:
              context.obj.steps.add('provision')
              Provision._save_context(context)
  
          return provision
  
      def wake(self, data: Optional['GuestData'] = None) -> None:
          """
          Wake up the plugin
  
          Override data with command line options.
          Wake up the guest based on provided guest data.
          """
          super().wake()
  
      def guest(self) -> Optional['Guest']:
          """
          Return provisioned guest
  
          Each ProvisionPlugin has to implement this method.
          Should return a provisioned Guest() instance.
          """
          raise NotImplementedError()
  
      def requires(self) -> List[str]:
          """ List of required packages needed for workdir sync """
          return Guest.requires()
  
      @classmethod
      def clean_images(cls, clean: 'tmt.base.Clean', dry: bool) -> None:
          """ Remove the images of one particular plugin """
  
  
  @dataclasses.dataclass
  class GuestData(tmt.utils.SerializableContainer):
      """
      Keys necessary to describe, create, save and restore a guest.
  
      Very basic set of keys shared across all known guest classes.
      """
  
      # guest role in the multihost scenario
      role: Optional[str] = None
      # hostname or ip address
      guest: Optional[str] = None
  
  
  class Guest(tmt.utils.Common):
      """
      Guest provisioned for test execution
  
      A base class for guest-like classes. Provides some of the basic methods
      and functionality, but note some of the methods are left intentionally
      empty. These do not have valid implementation on this level, and it's up
      to Guest subclasses to provide one working in their respective
      infrastructure.
  
      The following keys are expected in the 'data' container::
  
          role ....... guest role in the multihost scenario
          guest ...... name, hostname or ip address
  
      These are by default imported into instance attributes.
      """
  
      # Used by save() to construct the correct container for keys.
      _data_class = GuestData
  
      role: Optional[str]
      guest: Optional[str]
  
      # Flag to indicate localhost guest, requires special handling
      localhost = False
  
      # TODO: do we need this list? Can whatever code is using it use _data_class directly?
      # List of supported keys
      # (used for import/export to/from attributes during load and save)
      @property
      def _keys(self) -> List[str]:
          return list(self._data_class.keys())
  
      def __init__(self,
                   data: GuestData,
                   name: Optional[str] = None,
                   parent: Optional[tmt.utils.Common] = None) -> None:
          """ Initialize guest data """
          super().__init__(parent, name)
  
          self.load(data)
  
      def _random_name(self, prefix: str = '', length: int = 16) -> str:
          """ Generate a random name """
          # Append at least 5 random characters
          min_random_part = max(5, length - len(prefix))
          name = prefix + ''.join(
              random.choices(string.ascii_letters, k=min_random_part))
          # Return tail (containing random characters) of name
          return name[-length:]
  
      def _tmt_name(self) -> str:
          """ Generate a name prefixed with tmt run id """
          # TODO: with generic Step and Common, this should not be needed
          parent = cast(Provision, self.parent)
  
          _, run_id = os.path.split(parent.plan.my_run.workdir)
          return self._random_name(prefix="tmt-{0}-".format(run_id[-3:]))
  
      def load(self, data: GuestData) -> None:
          """
          Load guest data into object attributes for easy access
  
          Called during guest object initialization. Takes care of storing
          all supported keys (see class attribute _keys for the list) from
          provided data to the guest object attributes. Child classes can
          extend it to make additional guest attributes easily available.
  
          Data dictionary can contain guest information from both command
          line options / L2 metadata / user configuration and wake up data
          stored by the save() method below.
          """
          data.inject_to(self)
  
      def save(self) -> GuestData:
          """
          Save guest data for future wake up
  
          Export all essential guest data into a dictionary which will be
          stored in the `guests.yaml` file for possible future wake up of
          the guest. Everything needed to attach to a running instance
          should be added into the data dictionary by child classes.
          """
          return self._data_class.extract_from(self)
  
      def wake(self) -> None:
          """
          Wake up the guest
  
          Perform any actions necessary after step wake up to be able to
          attach to a running guest instance and execute commands. Called
          after load() is completed so all guest data should be prepared.
          """
          self.debug(f"Doing nothing to wake up guest '{self.guest}'.")
  
      def start(self) -> None:
          """
          Start the guest
  
          Get a new guest instance running. This should include preparing
          any configuration necessary to get it started. Called after
          load() is completed so all guest data should be available.
          """
          self.debug(f"Doing nothing to start guest '{self.guest}'.")
  
      def details(self) -> None:
          """ Show guest details such as distro and kernel """
          # Skip distro & kernel check in dry mode
          if self.opt('dry'):
              return
  
          # A small helper to make the repeated run & extract combo easier on eyes.
          def _fetch_detail(command: str, pattern: str) -> str:
              output = self.execute(command)
  
              if not output.stdout:
                  raise tmt.utils.RunError(
                      'command produced no usable output',
                      command,
                      0,
                      output.stdout,
                      output.stderr)
  
              match = re.search(pattern, output.stdout)
  
              if not match:
                  raise tmt.utils.RunError(
                      'command produced no usable output',
                      command,
                      0,
                      output.stdout,
                      output.stderr)
  
              return match.group(1)
  
          # Distro
          distro_commands = [
              # check os-release first)
              ('cat /etc/os-release', r'PRETTY_NAME="(.*)"'),
              # Check for lsb-release
              ('cat /etc/lsb-release', r'DISTRIB_DESCRIPTION="(.*)"'),
              # Check for redhat-release
              ('cat /etc/redhat-release', r'(.*)')
              ]
  
          for command, pattern in distro_commands:
              try:
                  distro = _fetch_detail(command, pattern)
  
              except tmt.utils.RunError:
                  continue
  
          else:
              distro = None
  
          # Handle standard cloud images message when connecting
          if distro is not None and 'Please login as the user' in distro:
              raise tmt.utils.GeneralError(
                  f'Login to the guest failed.\n{distro}')
  
          if distro:
              self.info('distro', distro, 'green')
  
          # Kernel
          kernel = _fetch_detail('uname -r', r'(.+)')
          self.verbose('kernel', kernel, 'green')
  
      def _ansible_verbosity(self) -> List[str]:
          """ Prepare verbose level based on the --debug option count """
          if self.opt('debug') < 3:
              return []
          else:
              return ['-' + (self.opt('debug') - 2) * 'v']
  
      @staticmethod
      def _ansible_extra_args(extra_args: str) -> List[str]:
          """ Prepare extra arguments for ansible-playbook"""
          if extra_args is None:
              return []
          else:
              return shlex.split(str(extra_args))
  
      def _ansible_summary(self, output: Optional[str]) -> None:
          """ Check the output for ansible result summary numbers """
          if not output:
              return
          keys = 'ok changed unreachable failed skipped rescued ignored'.split()
          for key in keys:
              matched = re.search(rf'^.*\s:\s.*{key}=(\d+).*$', output, re.M)
              if matched and int(matched.group(1)) > 0:
                  tasks = fmf.utils.listed(matched.group(1), 'task')
                  self.verbose(key, tasks, 'green')
  
      def _ansible_playbook_path(self, playbook: str) -> str:
          """ Prepare full ansible playbook path """
          # Playbook paths should be relative to the metadata tree root
          self.debug(f"Applying playbook '{playbook}' on guest '{self.guest}'.")
          # TODO: with generic Step and Common, this should not be needed
          parent = cast(Provision, self.parent)
          playbook = os.path.join(parent.plan.my_run.tree.root, playbook)
          self.debug(f"Playbook full path: '{playbook}'", level=2)
          return playbook
  
      def _prepare_environment(
          self,
          execute_environment: Optional[tmt.utils.EnvironmentType] = None
              ) -> tmt.utils.EnvironmentType:
          """ Prepare dict of environment variables """
          # Prepare environment variables so they can be correctly passed
          # to shell. Create a copy to prevent modifying source.
          environment: tmt.utils.EnvironmentType = dict()
          environment.update(execute_environment or dict())
          # Plan environment and variables provided on the command line
          # override environment provided to execute().
          # TODO: with generic Step and Common, this should not be needed
          parent = cast(Provision, self.parent)
          environment.update(parent.plan.environment)
          return environment
  
      @staticmethod
      def _export_environment(environment: tmt.utils.EnvironmentType) -> str:
          """ Prepare shell export of environment variables """
          if not environment:
              return ""
          return f'export {" ".join(tmt.utils.shell_variables(environment))}; '
  
      def ansible(self, playbook: str, extra_args: Optional[str] = None) -> None:
          """ Prepare guest using ansible playbook """
  
          raise NotImplementedError()
  
      def execute(self, command: Union[str, List[str]], **kwargs: Any) -> tmt.utils.CommandOutput:
          """
          Execute command on the guest
  
          command ... string or list of command arguments (required)
          env ....... dictionary with environment variables
          cwd ....... working directory to be entered before execution
  
          If the command is provided as a list, it will be space-joined.
          If necessary, quote escaping has to be handled by the caller.
          """
  
          raise NotImplementedError()
  
      def push(self,
               source: Optional[str] = None,
               destination: Optional[str] = None,
               options: Optional[List[str]] = None) -> None:
          """
          Push files to the guest
          """
  
          raise NotImplementedError()
  
      def pull(self,
               source: Optional[str] = None,
               destination: Optional[str] = None,
               options: Optional[List[str]] = None,
               extend_options: Optional[List[str]] = None) -> None:
          """
          Pull files from the guest
          """
  
          raise NotImplementedError()
  
      def stop(self) -> None:
          """
          Stop the guest
  
          Shut down a running guest instance so that it does not consume
          any memory or cpu resources. If needed, perform any actions
          necessary to store the instance status to disk.
          """
  
          raise NotImplementedError()
  
      def reboot(
              self,
              hard: bool = False,
              command: Optional[str] = None,
              timeout: Optional[int] = None) -> bool:
          """
          Reboot the guest, return True if successful
  
          Parameter 'hard' set to True means that guest should be
          rebooted by way which is not clean in sense that data can be
          lost. When set to False reboot should be done gracefully.
  
          Use the 'command' parameter to specify a custom reboot command
          instead of the default 'reboot'.
  
          Parameter 'timeout' can be used to specify time (in seconds) to
          wait for the guest to come back up after rebooting.
          """
  
          raise NotImplementedError()
  
      def reconnect(
              self,
              timeout: Optional[int] = None,
              tick: float = RECONNECT_WAIT_TICK,
              tick_increase: float = RECONNECT_WAIT_TICK_INCREASE
              ) -> bool:
          """
          Ensure the connection to the guest is working
  
          The default timeout is 5 minutes. Custom number of seconds can be
          provided in the `timeout` parameter. This may be useful when long
          operations (such as system upgrade) are performed.
          """
          # The default is handled here rather than in the argument so that
          # the caller can pass in None as an argument (i.e. don't care value)
          timeout = timeout or CONNECTION_TIMEOUT
          self.debug("Wait for a connection to the guest.")
  
          def try_whoami() -> None:
              try:
                  self.execute('whoami')
  
              except tmt.utils.RunError:
                  raise tmt.utils.WaitingIncomplete()
  
          try:
              tmt.utils.wait(
                  self,
                  try_whoami,
                  datetime.timedelta(seconds=timeout),
                  tick=tick,
                  tick_increase=tick_increase)
  
          except tmt.utils.WaitingTimedOutError:
              self.debug("Connection to guest failed after reboot.")
              return False
  
          return True
  
      def remove(self) -> None:
          """
          Remove the guest
  
          Completely remove all guest instance data so that it does not
          consume any disk resources.
          """
          self.debug(f"Doing nothing to remove guest '{self.guest}'.")
  
      def _check_rsync(self) -> CheckRsyncOutcome:
          """
          Make sure that rsync is installed on the guest
  
          On read-only distros install it under the '/root/pkg' directory.
          Returns 'already installed' when rsync is already present.
          """
  
          # Check for rsync (nothing to do if already installed)
          self.debug("Ensure that rsync is installed on the guest.")
          try:
              self.execute("rsync --version")
              return CheckRsyncOutcome.ALREADY_INSTALLED
          except tmt.utils.RunError:
              pass
  
          # Check the package manager
          self.debug("Check the package manager.")
          try:
              self.execute("dnf --version")
              package_manager = "dnf"
          except tmt.utils.RunError:
              package_manager = "yum"
  
          # Install under '/root/pkg' for read-only distros
          # (for now the check is based on 'rpm-ostree' presence)
          # FIXME: Find a better way how to detect read-only distros
          self.debug("Check for a read-only distro.")
          try:
              self.execute("rpm-ostree --version")
              readonly = (
                  " --installroot=/root/pkg --releasever / "
                  "&& ln -sf /root/pkg/bin/rsync /usr/local/bin/rsync")
          except tmt.utils.RunError:
              readonly = ""
  
          # Install the rsync
          self.execute(f"{package_manager} install -y rsync" + readonly)
  
          return CheckRsyncOutcome.INSTALLED
  
      @classmethod
      def requires(cls) -> List[str]:
          """ No extra requires needed """
          return []
  
  
  @dataclasses.dataclass
  class GuestSshData(GuestData):
      """
      Keys necessary to describe, create, save and restore a guest with SSH
      capability.
  
      Derived from GuestData, this class adds keys relevant for guests that can be
      reached over SSH.
      """
  
      # port to connect to
      port: Optional[int] = None
      # user name to log in
      user: Optional[str] = None
      # path to the private key
      key: List[str] = dataclasses.field(default_factory=list)
      # password
      password: Optional[str] = None
  
  
  class GuestSsh(Guest):
      """
      Guest provisioned for test execution, capable of accepting SSH connections
  
      The following keys are expected in the 'data' dictionary::
  
          role ....... guest role in the multihost scenario (inherited)
          guest ...... hostname or ip address (inherited)
          port ....... port to connect to
          user ....... user name to log in
          key ........ path to the private key (str or list)
          password ... password
  
      These are by default imported into instance attributes.
      """
  
      _data_class = GuestSshData
  
      port: Optional[int]
      user: Optional[str]
      key: List[str]
      password: Optional[str]
  
      # Master ssh connection process and socket path
      _ssh_master_process: Optional['subprocess.Popen[bytes]'] = None
      _ssh_socket_path: Optional[str] = None
  
      def _ssh_guest(self) -> str:
          """ Return user@guest """
          return f'{self.user}@{self.guest}'
  
      def _ssh_socket(self) -> str:
          """ Prepare path to the master connection socket """
          if not self._ssh_socket_path:
              # Use '/run/user/uid' if it exists, '/tmp' otherwise
              run_dir = f"/run/user/{os.getuid()}"
              if os.path.isdir(run_dir):
                  socket_dir = os.path.join(run_dir, "tmt")
              else:
                  socket_dir = "/tmp"
              os.makedirs(socket_dir, exist_ok=True)
              self._ssh_socket_path = tempfile.mktemp(dir=socket_dir)
          return self._ssh_socket_path
  
      def _ssh_options(self, join: bool = False) -> Union[str, List[str]]:
          """ Return common ssh options (list or joined) """
          options = [
              '-oForwardX11=no',
              '-oStrictHostKeyChecking=no',
              '-oUserKnownHostsFile=/dev/null',
              ]
          if self.key or self.password:
              # Skip ssh-agent (it adds additional identities)
              options.append('-oIdentitiesOnly=yes')
          if self.port:
              options.append(f'-p{self.port}')
          if self.key:
              for key in self.key:
                  options.append(f'-i{shlex.quote(key) if join else key}')
          if self.password:
              options.extend(['-oPasswordAuthentication=yes'])
  
          # Use the shared master connection
          options.append(f'-S{self._ssh_socket()}')
  
          return ' '.join(options) if join else options
  
      def _ssh_master_connection(self, command: List[str]) -> None:
          """ Check/create the master ssh connection """
          if self._ssh_master_process:
              return
  
          # TODO: _ssh_options() returns two incompatible types, we need to chose the right one.
          ssh_options = self._ssh_options()
          assert isinstance(ssh_options, list)
  
          command = command + ssh_options + ["-MNnT", self._ssh_guest()]
          self.debug(f"Create the master ssh connection: {' '.join(command)}")
          self._ssh_master_process = subprocess.Popen(
              command,
              stdin=subprocess.DEVNULL,
              stdout=subprocess.DEVNULL,
              stderr=subprocess.DEVNULL)
  
      def _ssh_command(self, join: bool = False) -> Union[str, List[str]]:
          """ Prepare an ssh command line for execution (list or joined) """
          command = []
          if self.password:
              password = shlex.quote(self.password) if join else self.password
              command.extend(["sshpass", "-p", password])
          command.append("ssh")
  
          # Check the master connection
          self._ssh_master_connection(command)
  
          if join:
              # TODO: _ssh_options() returns two incompatible types, we need to chose the right one.
              joined_ssh_options = self._ssh_options(join=True)
              assert isinstance(joined_ssh_options, str)
  
              return " ".join(command) + " " + joined_ssh_options
          else:
              # TODO: _ssh_options() returns two incompatible types, we need to chose the right one.
              ssh_options = self._ssh_options()
              assert isinstance(ssh_options, list)
  
              return command + ssh_options
  
      def ansible(self, playbook: str, extra_args: Optional[str] = None) -> None:
          """ Prepare guest using ansible playbook """
          playbook = self._ansible_playbook_path(playbook)
  
          # TODO: _ssh_options() returns two incompatible types, we need to chose the right one.
          joined_ssh_options = self._ssh_options(join=True)
          assert isinstance(joined_ssh_options, str)
  
          ansible_command = [
              'ansible-playbook'
              ] + self._ansible_verbosity()
  
          if extra_args:
              ansible_command += self._ansible_extra_args(extra_args)
  
          ansible_command += [
              f'--ssh-common-args={joined_ssh_options}',
              '-i', f'{self._ssh_guest()},',
              playbook
              ]
  
          # TODO: with generic Step and Common, this should not be needed
          parent = cast(Provision, self.parent)
  
          stdout, stderr = self.run(
              ansible_command,
              cwd=parent.plan.worktree,
              env=self._prepare_environment())
          self._ansible_summary(stdout)
  
      def execute(self, command: Union[str, List[str]], **kwargs: Any) -> tmt.utils.CommandOutput:
          """
          Execute command on the guest
  
          command ... string or list of command arguments (required)
          env ....... dictionary with environment variables
          cwd ....... working directory to be entered before execution
  
          If the command is provided as a list, it will be space-joined.
          If necessary, quote escaping has to be handled by the caller.
          """
          # Abort if guest is unavailable
          if self.guest is None:
              if not self.opt('dry'):
                  raise tmt.utils.GeneralError('The guest is not available.')
  
          # Prepare the export of environment variables
          environment = self._export_environment(
              self._prepare_environment(kwargs.get('env', dict())))
  
          # Change to given directory on guest if cwd provided
          directory = kwargs.get('cwd') or ''
          if directory:
              directory = f"cd {quote(directory)}; "
  
          # Run in interactive mode if requested
          interactive = ['-t'] if kwargs.get('interactive') else []
  
          # Prepare command and run it
          if isinstance(command, (list, tuple)):
              command = ' '.join(command)
          self.debug(f"Execute command '{command}' on guest '{self.guest}'.")
  
          # TODO: _ssh_command() returns two incompatible types, we need to chose the right one.
          ssh_command = self._ssh_command()
          assert isinstance(ssh_command, list)
  
          command = (
              ssh_command + interactive + [self._ssh_guest()] +
              [f'{environment}{directory}{command}'])
          return self.run(command, **kwargs)
  
      def push(self,
               source: Optional[str] = None,
               destination: Optional[str] = None,
               options: Optional[List[str]] = None) -> None:
          """
          Push files to the guest
  
          By default the whole plan workdir is synced to the same location
          on the guest. Use the 'source' and 'destination' to sync custom
          location and the 'options' parametr to modify default options
          which are '-Rrz --links --safe-links --delete'.
          """
          # Abort if guest is unavailable
          if self.guest is None:
              if not self.opt('dry'):
                  raise tmt.utils.GeneralError('The guest is not available.')
  
          # Prepare options and the push command
          options = options or DEFAULT_RSYNC_PUSH_OPTIONS
          if destination is None:
              destination = "/"
          if source is None:
              # TODO: with generic Step and Common, this should not be needed
              parent = cast(Provision, self.parent)
  
              assert parent.plan.workdir is not None
  
              source = parent.plan.workdir
              self.debug(f"Push workdir to guest '{self.guest}'.")
          else:
              self.debug(f"Copy '{source}' to '{destination}' on the guest.")
  
          def rsync() -> None:
              """ Run the rsync command """
              # In closure, mypy has hard times to reason about the state of used variables.
              assert options
              assert source
              assert destination
  
              # TODO: _ssh_command() returns two incompatible types, we need to chose the right one.
              joined_ssh_command = self._ssh_command(join=True)
              assert isinstance(joined_ssh_command, str)
  
              self.run(
                  ["rsync"] + options
                  + ["-e", joined_ssh_command]
                  + [source, f"{self._ssh_guest()}:{destination}"])
  
          # Try to push twice, check for rsync after the first failure
          try:
              rsync()
          except tmt.utils.RunError:
              try:
                  if self._check_rsync() == CheckRsyncOutcome.ALREADY_INSTALLED:
                      raise
                  rsync()
              except tmt.utils.RunError:
                  # Provide a reasonable error to the user
                  self.fail(
                      f"Failed to push workdir to the guest. This usually means "
                      f"that login as '{self.user}' to the guest does not work.")
                  raise
  
      def pull(self,
               source: Optional[str] = None,
               destination: Optional[str] = None,
               options: Optional[List[str]] = None,
               extend_options: Optional[List[str]] = None) -> None:
          f"""
          Pull files from the guest
  
          By default the whole plan workdir is synced from the same
          location on the guest. Use the 'source' and 'destination' to
          sync custom location, the 'options' parameter to modify
          default options '{" ".join(DEFAULT_RSYNC_PULL_OPTIONS)}'
          and 'extend_options' to extend them (e.g. by exclude).
          """
          # Abort if guest is unavailable
          if self.guest is None:
              if not self.opt('dry'):
                  raise tmt.utils.GeneralError('The guest is not available.')
  
          # Prepare options and the pull command
          options = options or DEFAULT_RSYNC_PULL_OPTIONS
          if extend_options is not None:
              options.extend(extend_options)
          if destination is None:
              destination = "/"
          if source is None:
              # TODO: with generic Step and Common, this should not be needed
              parent = cast(Provision, self.parent)
  
              assert parent.plan.workdir is not None
  
              source = parent.plan.workdir
              self.debug(f"Pull workdir from guest '{self.guest}'.")
          else:
              self.debug(f"Copy '{source}' from the guest to '{destination}'.")
  
          def rsync() -> None:
              """ Run the rsync command """
              # In closure, mypy has hard times to reason about the state of used variables.
              assert options
              assert source
              assert destination
  
              # TODO: _ssh_command() returns two incompatible types, we need to chose the right one.
              joined_ssh_command = self._ssh_command(join=True)
              assert isinstance(joined_ssh_command, str)
  
              self.run(
                  ["rsync"] + options
                  + ["-e", joined_ssh_command]
                  + [f"{self._ssh_guest()}:{source}", destination])
  
          # Try to pull twice, check for rsync after the first failure
          try:
              rsync()
          except tmt.utils.RunError:
              try:
                  if self._check_rsync() == CheckRsyncOutcome.ALREADY_INSTALLED:
                      raise
                  rsync()
              except tmt.utils.RunError:
                  # Provide a reasonable error to the user
                  self.fail(
                      f"Failed to pull workdir from the guest. "
                      f"This usually means that login as '{self.user}' "
                      f"to the guest does not work.")
                  raise
  
      def stop(self) -> None:
          """
          Stop the guest
  
          Shut down a running guest instance so that it does not consume
          any memory or cpu resources. If needed, perform any actions
          necessary to store the instance status to disk.
          """
  
          # Close the master ssh connection
          if self._ssh_master_process:
              self.debug("Close the master ssh connection.", level=3)
              try:
                  self._ssh_master_process.terminate()
                  self._ssh_master_process.wait(timeout=3)
              except subprocess.TimeoutExpired:
                  pass
  
          # Remove the ssh socket
          if self._ssh_socket_path and os.path.exists(self._ssh_socket_path):
              self.debug(
                  f"Remove ssh socket '{self._ssh_socket_path}'.", level=3)
              try:
                  os.unlink(self._ssh_socket_path)
              except OSError as error:
                  self.debug(f"Failed to remove the socket: {error}", level=3)
  
      def reboot(
              self,
              hard: bool = False,
              command: Optional[str] = None,
              timeout: Optional[int] = None,
              tick: float = tmt.utils.DEFAULT_WAIT_TICK,
              tick_increase: float = tmt.utils.DEFAULT_WAIT_TICK_INCREASE) -> bool:
          """
          Reboot the guest, return True if reconnect was successful
  
          Parameter 'hard' set to True means that guest should be
          rebooted by way which is not clean in sense that data can be
          lost. When set to False reboot should be done gracefully.
  
          Use the 'command' parameter to specify a custom reboot command
          instead of the default 'reboot'.
          """
  
          if hard:
              raise tmt.utils.ProvisionError(
                  "Method does not support hard reboot.")
  
          command = command or "reboot"
          self.debug(f"Reboot using the command '{command}'.")
  
          re_boot_time = re.compile(r'btime\s+(\d+)')
  
          def get_boot_time() -> int:
              """ Reads btime from /proc/stat """
              stdout = self.execute(["cat", "/proc/stat"]).stdout
              assert stdout
  
              match = re_boot_time.search(stdout)
  
              if match is None:
                  raise tmt.utils.ProvisionError('Failed to retrieve boot time from guest')
  
              return int(match.group(1))
  
          current_boot_time = get_boot_time()
  
          try:
              self.execute(command)
          except tmt.utils.RunError as error:
              # Connection can be closed by the remote host even before the
              # reboot command is completed. Let's ignore such errors.
              if error.returncode == 255:
                  self.debug(
                      "Seems the connection was closed too fast, ignoring.")
              else:
                  raise
  
          # Wait until we get new boot time, connection will drop and will be
          # unreachable for some time
          def check_boot_time() -> None:
              try:
                  new_boot_time = get_boot_time()
  
                  if new_boot_time != current_boot_time:
                      # Different boot time and we are reconnected
                      return
  
                  # Same boot time, reboot didn't happen yet, retrying
                  raise tmt.utils.WaitingIncomplete()
  
              except tmt.utils.RunError:
                  self.debug('Failed to connect to the guest.')
                  raise tmt.utils.WaitingIncomplete()
  
          timeout = timeout or CONNECTION_TIMEOUT
  
          try:
              tmt.utils.wait(
                  self,
                  check_boot_time,
                  datetime.timedelta(seconds=timeout),
                  tick=tick,
                  tick_increase=tick_increase)
  
          except tmt.utils.WaitingTimedOutError:
              self.debug("Connection to guest failed after reboot.")
              return False
  
          self.debug("Connection to guest succeeded after reboot.")
          return True
  
      def remove(self) -> None:
          """
          Remove the guest
  
          Completely remove all guest instance data so that it does not
          consume any disk resources.
          """
          self.debug(f"Doing nothing to remove guest '{self.guest}'.")
  
      def _check_rsync(self) -> CheckRsyncOutcome:
          """
          Make sure that rsync is installed on the guest
  
          On read-only distros install it under the '/root/pkg' directory.
          Returns 'already installed' when rsync is already present.
          """
  
          # Check for rsync (nothing to do if already installed)
          self.debug("Ensure that rsync is installed on the guest.")
          try:
              self.execute("rsync --version")
              return CheckRsyncOutcome.ALREADY_INSTALLED
          except tmt.utils.RunError:
              pass
  
          # Check the package manager
          self.debug("Check the package manager.")
          try:
              self.execute("dnf --version")
              package_manager = "dnf"
          except tmt.utils.RunError:
              package_manager = "yum"
  
          # Install under '/root/pkg' for read-only distros
          # (for now the check is based on 'rpm-ostree' presence)
          # FIXME: Find a better way how to detect read-only distros
          self.debug("Check for a read-only distro.")
          try:
              self.execute("rpm-ostree --version")
              readonly = (
                  " --installroot=/root/pkg --releasever / "
                  "&& ln -sf /root/pkg/bin/rsync /usr/local/bin/rsync")
          except tmt.utils.RunError:
              readonly = ""
  
          # Install the rsync
          self.execute(f"{package_manager} install -y rsync" + readonly)
  
          return CheckRsyncOutcome.INSTALLED
  
      @classmethod
      def requires(cls) -> List[str]:
          """ No extra requires needed """
          return []
  
files: tmt/steps/provision/__init__.py
files: import collections
  import dataclasses
  import datetime
  import enum
  import os
  import random
  import re
  import shlex
  import string
  import subprocess
  import tempfile
  from shlex import quote
  from typing import Any, Dict, List, Optional, Type, Union, cast
  
  import click
  import fmf
  
  import tmt
  import tmt.plugins
  import tmt.steps
  import tmt.utils
  from tmt.steps import Action
  
  # Timeout in seconds of waiting for a connection after reboot
  CONNECTION_TIMEOUT = 5 * 60
  
  # When waiting for guest to recover from reboot, try re-connecting every
  # this many seconds.
  RECONNECT_WAIT_TICK = 5
  RECONNECT_WAIT_TICK_INCREASE = 1.0
  
  # Default rsync options
  DEFAULT_RSYNC_OPTIONS = [
      "-R", "-r", "-z", "--links", "--safe-links", "--delete"]
  
  DEFAULT_RSYNC_PUSH_OPTIONS = ["-R", "-r", "-z", "--links", "--safe-links", "--delete"]
  DEFAULT_RSYNC_PULL_OPTIONS = ["-R", "-r", "-z", "--links", "--safe-links", "--protect-args"]
  
  
  class CheckRsyncOutcome(enum.Enum):
      ALREADY_INSTALLED = 'already-installed'
      INSTALLED = 'installed'
  
  
  class Provision(tmt.steps.Step):
      """ Provision an environment for testing or use localhost. """
  
      # Default implementation for provision is a virtual machine
      DEFAULT_HOW = 'virtual'
  
      def __init__(self, plan: 'tmt.Plan', data: tmt.steps.StepData) -> None:
          """ Initialize provision step data """
          super().__init__(plan=plan, data=data)
          # Check that the names are unique
          names = [data['name'] for data in self.data]
          names_count: Dict[str, int] = collections.defaultdict(int)
          for name in names:
              names_count[name] += 1
          if len(self.data) > 1 and len(self.data) != len(names_count):
              duplicate = [k for k, v in names_count.items() if v > 1]
              duplicate_string = ', '.join(duplicate)
              raise tmt.utils.GeneralError(
                  f"Provision step names must be unique for multihost testing. "
                  f"Duplicate names: {duplicate_string} in plan '{plan.name}'.")
          # List of provisioned guests and loaded guest data
          self._guests: List['Guest'] = []
          self._guest_data: Dict[str, 'GuestData'] = {}
          self.is_multihost = False
  
      def load(self) -> None:
          """ Load guest data from the workdir """
          super().load()
          try:
              raw_guest_data = tmt.utils.yaml_to_dict(self.read('guests.yaml'))
  
              self._guest_data = {
                  name: tmt.utils.SerializableContainer.unserialize(guest_data)
                  for name, guest_data in raw_guest_data.items()
                  }
  
          except tmt.utils.FileError:
              self.debug('Provisioned guests not found.', level=2)
  
      def save(self) -> None:
          """ Save guest data to the workdir """
          super().save()
          try:
              raw_guest_data = {guest.name: guest.save().to_serialized()
                                for guest in self.guests()}
  
              self.write('guests.yaml', tmt.utils.dict_to_yaml(raw_guest_data))
          except tmt.utils.FileError:
              self.debug('Failed to save provisioned guests.')
  
      def wake(self) -> None:
          """ Wake up the step (process workdir and command line) """
          super().wake()
  
          # Choose the right plugin and wake it up
          for data in self.data:
              # TODO: with generic BasePlugin, delegate() should return more fitting type,
              # not the base class.
              plugin = cast(ProvisionPlugin, ProvisionPlugin.delegate(self, data))
              self._phases.append(plugin)
              # If guest data loaded, perform a complete wake up
              plugin.wake(data=self._guest_data.get(plugin.name))
  
              guest = plugin.guest()
              if guest:
                  self._guests.append(guest)
  
          # Nothing more to do if already done
          if self.status() == 'done':
              self.debug(
                  'Provision wake up complete (already done before).', level=2)
          # Save status and step data (now we know what to do)
          else:
              self.status('todo')
              self.save()
  
      def show(self) -> None:
          """ Show discover details """
          for data in self.data:
              ProvisionPlugin.delegate(self, data).show()
  
      def summary(self) -> None:
          """ Give a concise summary of the provisioning """
          # Summary of provisioned guests
          guests = fmf.utils.listed(self.guests(), 'guest')
          self.info('summary', f'{guests} provisioned', 'green', shift=1)
          # Guest list in verbose mode
          for guest in self.guests():
              if not guest.name.startswith(tmt.utils.DEFAULT_NAME):
                  self.verbose(guest.name, color='red', shift=2)
  
      def go(self) -> None:
          """ Provision all guests"""
          super().go()
  
          # Nothing more to do if already done
          if self.status() == 'done':
              self.info('status', 'done', 'green', shift=1)
              self.summary()
              self.actions()
              return
  
          # Provision guests
          self._guests = []
          save = True
          self.is_multihost = sum([isinstance(phase, ProvisionPlugin)
                                  for phase in self.phases()]) > 1
          try:
              for phase in self.phases(classes=(Action, ProvisionPlugin)):
                  try:
                      if isinstance(phase, Action):
                          phase.go()
  
                      elif isinstance(phase, ProvisionPlugin):
                          phase.go()
  
                          guest = phase.guest()
                          if guest:
                              guest.details()
  
                      if self.is_multihost:
                          self.info('')
                  finally:
                      if isinstance(phase, ProvisionPlugin):
                          guest = phase.guest()
                          if guest:
                              self._guests.append(guest)
  
              # Give a summary, update status and save
              self.summary()
              self.status('done')
          except (SystemExit, tmt.utils.SpecificationError) as error:
              # A plugin will only raise SystemExit if the exit is really desired
              # and no other actions should be done. An example of this is
              # listing available images. In such case, the workdir is deleted
              # as it's redundant and save() would throw an error.
              save = False
              raise error
          finally:
              if save:
                  self.save()
  
      def guests(self) -> List['Guest']:
          """ Return the list of all provisioned guests """
          return self._guests
  
      def requires(self) -> List[str]:
          """
          Packages required by all enabled provision plugins
  
          Return a list of packages which need to be installed on the
          provisioned guest so that the workdir can be synced to it.
          Used by the prepare step.
          """
          requires = set()
          for plugin in self.phases(classes=ProvisionPlugin):
              requires.update(plugin.requires())
          return list(requires)
  
  
  class ProvisionPlugin(tmt.steps.GuestlessPlugin):
      """ Common parent of provision plugins """
  
      # Default implementation for provision is a virtual machine
      how = 'virtual'
  
      # List of all supported methods aggregated from all plugins of the same step.
      _supported_methods: List[tmt.steps.Method] = []
  
      # Common keys for all provision step implementations
      _common_keys = ['role']
  
      @classmethod
      def base_command(
              cls,
              usage: str,
              method_class: Optional[Type[click.Command]] = None) -> click.Command:
          """ Create base click command (common for all provision plugins) """
  
          # Prepare general usage message for the step
          if method_class:
              usage = Provision.usage(method_overview=usage)
  
          # Create the command
          @click.command(cls=method_class, help=usage)
          @click.pass_context
          @click.option(
              '-h', '--how', metavar='METHOD',
              help='Use specified method for provisioning.')
          def provision(context: click.Context, **kwargs: Any) -> None:
              context.obj.steps.add('provision')
              Provision._save_context(context)
  
          return provision
  
      def wake(self, keys: Optional[List[str]] = None, data: Optional['GuestData'] = None) -> None:
          """
          Wake up the plugin
  
          Override data with command line options.
          Wake up the guest based on provided guest data.
          """
          super().wake(keys=keys)
  
      def guest(self) -> Optional['Guest']:
          """
          Return provisioned guest
  
          Each ProvisionPlugin has to implement this method.
          Should return a provisioned Guest() instance.
          """
          raise NotImplementedError()
  
      def requires(self) -> List[str]:
          """ List of required packages needed for workdir sync """
          return Guest.requires()
  
      @classmethod
      def clean_images(cls, clean: 'tmt.base.Clean', dry: bool) -> None:
          """ Remove the images of one particular plugin """
  
  
  @dataclasses.dataclass
  class GuestData(tmt.utils.SerializableContainer):
      """
      Keys necessary to describe, create, save and restore a guest.
  
      Very basic set of keys shared across all known guest classes.
      """
  
      # guest role in the multihost scenario
      role: Optional[str] = None
      # hostname or ip address
      guest: Optional[str] = None
  
  
  class Guest(tmt.utils.Common):
      """
      Guest provisioned for test execution
  
      A base class for guest-like classes. Provides some of the basic methods
      and functionality, but note some of the methods are left intentionally
      empty. These do not have valid implementation on this level, and it's up
      to Guest subclasses to provide one working in their respective
      infrastructure.
  
      The following keys are expected in the 'data' container::
  
          role ....... guest role in the multihost scenario
          guest ...... name, hostname or ip address
  
      These are by default imported into instance attributes.
      """
  
      # Used by save() to construct the correct container for keys.
      _data_class = GuestData
  
      role: Optional[str]
      guest: Optional[str]
  
      # Flag to indicate localhost guest, requires special handling
      localhost = False
  
      # TODO: do we need this list? Can whatever code is using it use _data_class directly?
      # List of supported keys
      # (used for import/export to/from attributes during load and save)
      @property
      def _keys(self) -> List[str]:
          return list(self._data_class.keys())
  
      def __init__(self,
                   data: GuestData,
                   name: Optional[str] = None,
                   parent: Optional[tmt.utils.Common] = None) -> None:
          """ Initialize guest data """
          super().__init__(parent, name)
  
          self.load(data)
  
      def _random_name(self, prefix: str = '', length: int = 16) -> str:
          """ Generate a random name """
          # Append at least 5 random characters
          min_random_part = max(5, length - len(prefix))
          name = prefix + ''.join(
              random.choices(string.ascii_letters, k=min_random_part))
          # Return tail (containing random characters) of name
          return name[-length:]
  
      def _tmt_name(self) -> str:
          """ Generate a name prefixed with tmt run id """
          # TODO: with generic Step and Common, this should not be needed
          parent = cast(Provision, self.parent)
  
          _, run_id = os.path.split(parent.plan.my_run.workdir)
          return self._random_name(prefix="tmt-{0}-".format(run_id[-3:]))
  
      def load(self, data: GuestData) -> None:
          """
          Load guest data into object attributes for easy access
  
          Called during guest object initialization. Takes care of storing
          all supported keys (see class attribute _keys for the list) from
          provided data to the guest object attributes. Child classes can
          extend it to make additional guest attributes easily available.
  
          Data dictionary can contain guest information from both command
          line options / L2 metadata / user configuration and wake up data
          stored by the save() method below.
          """
          data.inject_to(self)
  
      def save(self) -> GuestData:
          """
          Save guest data for future wake up
  
          Export all essential guest data into a dictionary which will be
          stored in the `guests.yaml` file for possible future wake up of
          the guest. Everything needed to attach to a running instance
          should be added into the data dictionary by child classes.
          """
          return self._data_class.extract_from(self)
  
      def wake(self) -> None:
          """
          Wake up the guest
  
          Perform any actions necessary after step wake up to be able to
          attach to a running guest instance and execute commands. Called
          after load() is completed so all guest data should be prepared.
          """
          self.debug(f"Doing nothing to wake up guest '{self.guest}'.")
  
      def start(self) -> None:
          """
          Start the guest
  
          Get a new guest instance running. This should include preparing
          any configuration necessary to get it started. Called after
          load() is completed so all guest data should be available.
          """
          self.debug(f"Doing nothing to start guest '{self.guest}'.")
  
      def details(self) -> None:
          """ Show guest details such as distro and kernel """
          # Skip distro & kernel check in dry mode
          if self.opt('dry'):
              return
  
          # A small helper to make the repeated run & extract combo easier on eyes.
          def _fetch_detail(command: str, pattern: str) -> str:
              output = self.execute(command)
  
              if not output.stdout:
                  raise tmt.utils.RunError(
                      'command produced no usable output',
                      command,
                      0,
                      output.stdout,
                      output.stderr)
  
              match = re.search(pattern, output.stdout)
  
              if not match:
                  raise tmt.utils.RunError(
                      'command produced no usable output',
                      command,
                      0,
                      output.stdout,
                      output.stderr)
  
              return match.group(1)
  
          # Distro
          distro_commands = [
              # check os-release first)
              ('cat /etc/os-release', r'PRETTY_NAME="(.*)"'),
              # Check for lsb-release
              ('cat /etc/lsb-release', r'DISTRIB_DESCRIPTION="(.*)"'),
              # Check for redhat-release
              ('cat /etc/redhat-release', r'(.*)')
              ]
  
          for command, pattern in distro_commands:
              try:
                  distro = _fetch_detail(command, pattern)
  
              except tmt.utils.RunError:
                  continue
  
          else:
              distro = None
  
          # Handle standard cloud images message when connecting
          if distro is not None and 'Please login as the user' in distro:
              raise tmt.utils.GeneralError(
                  f'Login to the guest failed.\n{distro}')
  
          if distro:
              self.info('distro', distro, 'green')
  
          # Kernel
          kernel = _fetch_detail('uname -r', r'(.+)')
          self.verbose('kernel', kernel, 'green')
  
      def _ansible_verbosity(self) -> List[str]:
          """ Prepare verbose level based on the --debug option count """
          if self.opt('debug') < 3:
              return []
          else:
              return ['-' + (self.opt('debug') - 2) * 'v']
  
      @staticmethod
      def _ansible_extra_args(extra_args: str) -> List[str]:
          """ Prepare extra arguments for ansible-playbook"""
          if extra_args is None:
              return []
          else:
              return shlex.split(str(extra_args))
  
      def _ansible_summary(self, output: Optional[str]) -> None:
          """ Check the output for ansible result summary numbers """
          if not output:
              return
          keys = 'ok changed unreachable failed skipped rescued ignored'.split()
          for key in keys:
              matched = re.search(rf'^.*\s:\s.*{key}=(\d+).*$', output, re.M)
              if matched and int(matched.group(1)) > 0:
                  tasks = fmf.utils.listed(matched.group(1), 'task')
                  self.verbose(key, tasks, 'green')
  
      def _ansible_playbook_path(self, playbook: str) -> str:
          """ Prepare full ansible playbook path """
          # Playbook paths should be relative to the metadata tree root
          self.debug(f"Applying playbook '{playbook}' on guest '{self.guest}'.")
          # TODO: with generic Step and Common, this should not be needed
          parent = cast(Provision, self.parent)
          playbook = os.path.join(parent.plan.my_run.tree.root, playbook)
          self.debug(f"Playbook full path: '{playbook}'", level=2)
          return playbook
  
      def _prepare_environment(
          self,
          execute_environment: Optional[tmt.utils.EnvironmentType] = None
              ) -> tmt.utils.EnvironmentType:
          """ Prepare dict of environment variables """
          # Prepare environment variables so they can be correctly passed
          # to shell. Create a copy to prevent modifying source.
          environment: tmt.utils.EnvironmentType = dict()
          environment.update(execute_environment or dict())
          # Plan environment and variables provided on the command line
          # override environment provided to execute().
          # TODO: with generic Step and Common, this should not be needed
          parent = cast(Provision, self.parent)
          environment.update(parent.plan.environment)
          return environment
  
      @staticmethod
      def _export_environment(environment: tmt.utils.EnvironmentType) -> str:
          """ Prepare shell export of environment variables """
          if not environment:
              return ""
          return f'export {" ".join(tmt.utils.shell_variables(environment))}; '
  
      def ansible(self, playbook: str, extra_args: Optional[str] = None) -> None:
          """ Prepare guest using ansible playbook """
  
          raise NotImplementedError()
  
      def execute(self, command: Union[str, List[str]], **kwargs: Any) -> tmt.utils.CommandOutput:
          """
          Execute command on the guest
  
          command ... string or list of command arguments (required)
          env ....... dictionary with environment variables
          cwd ....... working directory to be entered before execution
  
          If the command is provided as a list, it will be space-joined.
          If necessary, quote escaping has to be handled by the caller.
          """
  
          raise NotImplementedError()
  
      def push(self,
               source: Optional[str] = None,
               destination: Optional[str] = None,
               options: Optional[List[str]] = None) -> None:
          """
          Push files to the guest
          """
  
          raise NotImplementedError()
  
      def pull(self,
               source: Optional[str] = None,
               destination: Optional[str] = None,
               options: Optional[List[str]] = None,
               extend_options: Optional[List[str]] = None) -> None:
          """
          Pull files from the guest
          """
  
          raise NotImplementedError()
  
      def stop(self) -> None:
          """
          Stop the guest
  
          Shut down a running guest instance so that it does not consume
          any memory or cpu resources. If needed, perform any actions
          necessary to store the instance status to disk.
          """
  
          raise NotImplementedError()
  
      def reboot(
              self,
              hard: bool = False,
              command: Optional[str] = None,
              timeout: Optional[int] = None) -> bool:
          """
          Reboot the guest, return True if successful
  
          Parameter 'hard' set to True means that guest should be
          rebooted by way which is not clean in sense that data can be
          lost. When set to False reboot should be done gracefully.
  
          Use the 'command' parameter to specify a custom reboot command
          instead of the default 'reboot'.
  
          Parameter 'timeout' can be used to specify time (in seconds) to
          wait for the guest to come back up after rebooting.
          """
  
          raise NotImplementedError()
  
      def reconnect(
              self,
              timeout: Optional[int] = None,
              tick: float = RECONNECT_WAIT_TICK,
              tick_increase: float = RECONNECT_WAIT_TICK_INCREASE
              ) -> bool:
          """
          Ensure the connection to the guest is working
  
          The default timeout is 5 minutes. Custom number of seconds can be
          provided in the `timeout` parameter. This may be useful when long
          operations (such as system upgrade) are performed.
          """
          # The default is handled here rather than in the argument so that
          # the caller can pass in None as an argument (i.e. don't care value)
          timeout = timeout or CONNECTION_TIMEOUT
          self.debug("Wait for a connection to the guest.")
  
          def try_whoami() -> None:
              try:
                  self.execute('whoami')
  
              except tmt.utils.RunError:
                  raise tmt.utils.WaitingIncomplete()
  
          try:
              tmt.utils.wait(
                  self,
                  try_whoami,
                  datetime.timedelta(seconds=timeout),
                  tick=tick,
                  tick_increase=tick_increase)
  
          except tmt.utils.WaitingTimedOutError:
              self.debug("Connection to guest failed after reboot.")
              return False
  
          return True
  
      def remove(self) -> None:
          """
          Remove the guest
  
          Completely remove all guest instance data so that it does not
          consume any disk resources.
          """
          self.debug(f"Doing nothing to remove guest '{self.guest}'.")
  
      def _check_rsync(self) -> CheckRsyncOutcome:
          """
          Make sure that rsync is installed on the guest
  
          On read-only distros install it under the '/root/pkg' directory.
          Returns 'already installed' when rsync is already present.
          """
  
          # Check for rsync (nothing to do if already installed)
          self.debug("Ensure that rsync is installed on the guest.")
          try:
              self.execute("rsync --version")
              return CheckRsyncOutcome.ALREADY_INSTALLED
          except tmt.utils.RunError:
              pass
  
          # Check the package manager
          self.debug("Check the package manager.")
          try:
              self.execute("dnf --version")
              package_manager = "dnf"
          except tmt.utils.RunError:
              package_manager = "yum"
  
          # Install under '/root/pkg' for read-only distros
          # (for now the check is based on 'rpm-ostree' presence)
          # FIXME: Find a better way how to detect read-only distros
          self.debug("Check for a read-only distro.")
          try:
              self.execute("rpm-ostree --version")
              readonly = (
                  " --installroot=/root/pkg --releasever / "
                  "&& ln -sf /root/pkg/bin/rsync /usr/local/bin/rsync")
          except tmt.utils.RunError:
              readonly = ""
  
          # Install the rsync
          self.execute(f"{package_manager} install -y rsync" + readonly)
  
          return CheckRsyncOutcome.INSTALLED
  
      @classmethod
      def requires(cls) -> List[str]:
          """ No extra requires needed """
          return []
  
  
  @dataclasses.dataclass
  class GuestSshData(GuestData):
      """
      Keys necessary to describe, create, save and restore a guest with SSH
      capability.
  
      Derived from GuestData, this class adds keys relevant for guests that can be
      reached over SSH.
      """
  
      # port to connect to
      port: Optional[int] = None
      # user name to log in
      user: Optional[str] = None
      # path to the private key
      key: List[str] = dataclasses.field(default_factory=list)
      # password
      password: Optional[str] = None
  
  
  class GuestSsh(Guest):
      """
      Guest provisioned for test execution, capable of accepting SSH connections
  
      The following keys are expected in the 'data' dictionary::
  
          role ....... guest role in the multihost scenario (inherited)
          guest ...... hostname or ip address (inherited)
          port ....... port to connect to
          user ....... user name to log in
          key ........ path to the private key (str or list)
          password ... password
  
      These are by default imported into instance attributes.
      """
  
      _data_class = GuestSshData
  
      port: Optional[int]
      user: Optional[str]
      key: List[str]
      password: Optional[str]
  
      # Master ssh connection process and socket path
      _ssh_master_process: Optional['subprocess.Popen[bytes]'] = None
      _ssh_socket_path: Optional[str] = None
  
      def _ssh_guest(self) -> str:
          """ Return user@guest """
          return f'{self.user}@{self.guest}'
  
      def _ssh_socket(self) -> str:
          """ Prepare path to the master connection socket """
          if not self._ssh_socket_path:
              # Use '/run/user/uid' if it exists, '/tmp' otherwise
              run_dir = f"/run/user/{os.getuid()}"
              if os.path.isdir(run_dir):
                  socket_dir = os.path.join(run_dir, "tmt")
              else:
                  socket_dir = "/tmp"
              os.makedirs(socket_dir, exist_ok=True)
              self._ssh_socket_path = tempfile.mktemp(dir=socket_dir)
          return self._ssh_socket_path
  
      def _ssh_options(self, join: bool = False) -> Union[str, List[str]]:
          """ Return common ssh options (list or joined) """
          options = [
              '-oForwardX11=no',
              '-oStrictHostKeyChecking=no',
              '-oUserKnownHostsFile=/dev/null',
              ]
          if self.key or self.password:
              # Skip ssh-agent (it adds additional identities)
              options.append('-oIdentitiesOnly=yes')
          if self.port:
              options.append(f'-p{self.port}')
          if self.key:
              for key in self.key:
                  options.append(f'-i{shlex.quote(key) if join else key}')
          if self.password:
              options.extend(['-oPasswordAuthentication=yes'])
  
          # Use the shared master connection
          options.append(f'-S{self._ssh_socket()}')
  
          return ' '.join(options) if join else options
  
      def _ssh_master_connection(self, command: List[str]) -> None:
          """ Check/create the master ssh connection """
          if self._ssh_master_process:
              return
  
          # TODO: _ssh_options() returns two incompatible types, we need to chose the right one.
          ssh_options = self._ssh_options()
          assert isinstance(ssh_options, list)
  
          command = command + ssh_options + ["-MNnT", self._ssh_guest()]
          self.debug(f"Create the master ssh connection: {' '.join(command)}")
          self._ssh_master_process = subprocess.Popen(
              command,
              stdin=subprocess.DEVNULL,
              stdout=subprocess.DEVNULL,
              stderr=subprocess.DEVNULL)
  
      def _ssh_command(self, join: bool = False) -> Union[str, List[str]]:
          """ Prepare an ssh command line for execution (list or joined) """
          command = []
          if self.password:
              password = shlex.quote(self.password) if join else self.password
              command.extend(["sshpass", "-p", password])
          command.append("ssh")
  
          # Check the master connection
          self._ssh_master_connection(command)
  
          if join:
              # TODO: _ssh_options() returns two incompatible types, we need to chose the right one.
              joined_ssh_options = self._ssh_options(join=True)
              assert isinstance(joined_ssh_options, str)
  
              return " ".join(command) + " " + joined_ssh_options
          else:
              # TODO: _ssh_options() returns two incompatible types, we need to chose the right one.
              ssh_options = self._ssh_options()
              assert isinstance(ssh_options, list)
  
              return command + ssh_options
  
      def ansible(self, playbook: str, extra_args: Optional[str] = None) -> None:
          """ Prepare guest using ansible playbook """
          playbook = self._ansible_playbook_path(playbook)
  
          # TODO: _ssh_options() returns two incompatible types, we need to chose the right one.
          joined_ssh_options = self._ssh_options(join=True)
          assert isinstance(joined_ssh_options, str)
  
          ansible_command = [
              'ansible-playbook'
              ] + self._ansible_verbosity()
  
          if extra_args:
              ansible_command += self._ansible_extra_args(extra_args)
  
          ansible_command += [
              f'--ssh-common-args={joined_ssh_options}',
              '-i', f'{self._ssh_guest()},',
              playbook
              ]
  
          # TODO: with generic Step and Common, this should not be needed
          parent = cast(Provision, self.parent)
  
          stdout, stderr = self.run(
              ansible_command,
              cwd=parent.plan.worktree,
              env=self._prepare_environment())
          self._ansible_summary(stdout)
  
      def execute(self, command: Union[str, List[str]], **kwargs: Any) -> tmt.utils.CommandOutput:
          """
          Execute command on the guest
  
          command ... string or list of command arguments (required)
          env ....... dictionary with environment variables
          cwd ....... working directory to be entered before execution
  
          If the command is provided as a list, it will be space-joined.
          If necessary, quote escaping has to be handled by the caller.
          """
          # Abort if guest is unavailable
          if self.guest is None:
              if not self.opt('dry'):
                  raise tmt.utils.GeneralError('The guest is not available.')
  
          # Prepare the export of environment variables
          environment = self._export_environment(
              self._prepare_environment(kwargs.get('env', dict())))
  
          # Change to given directory on guest if cwd provided
          directory = kwargs.get('cwd') or ''
          if directory:
              directory = f"cd {quote(directory)}; "
  
          # Run in interactive mode if requested
          interactive = ['-t'] if kwargs.get('interactive') else []
  
          # Prepare command and run it
          if isinstance(command, (list, tuple)):
              command = ' '.join(command)
          self.debug(f"Execute command '{command}' on guest '{self.guest}'.")
  
          # TODO: _ssh_command() returns two incompatible types, we need to chose the right one.
          ssh_command = self._ssh_command()
          assert isinstance(ssh_command, list)
  
          command = (
              ssh_command + interactive + [self._ssh_guest()] +
              [f'{environment}{directory}{command}'])
          return self.run(command, **kwargs)
  
      def push(self,
               source: Optional[str] = None,
               destination: Optional[str] = None,
               options: Optional[List[str]] = None) -> None:
          """
          Push files to the guest
  
          By default the whole plan workdir is synced to the same location
          on the guest. Use the 'source' and 'destination' to sync custom
          location and the 'options' parametr to modify default options
          which are '-Rrz --links --safe-links --delete'.
          """
          # Abort if guest is unavailable
          if self.guest is None:
              if not self.opt('dry'):
                  raise tmt.utils.GeneralError('The guest is not available.')
  
          # Prepare options and the push command
          options = options or DEFAULT_RSYNC_PUSH_OPTIONS
          if destination is None:
              destination = "/"
          if source is None:
              # TODO: with generic Step and Common, this should not be needed
              parent = cast(Provision, self.parent)
  
              assert parent.plan.workdir is not None
  
              source = parent.plan.workdir
              self.debug(f"Push workdir to guest '{self.guest}'.")
          else:
              self.debug(f"Copy '{source}' to '{destination}' on the guest.")
  
          def rsync() -> None:
              """ Run the rsync command """
              # In closure, mypy has hard times to reason about the state of used variables.
              assert options
              assert source
              assert destination
  
              # TODO: _ssh_command() returns two incompatible types, we need to chose the right one.
              joined_ssh_command = self._ssh_command(join=True)
              assert isinstance(joined_ssh_command, str)
  
              self.run(
                  ["rsync"] + options
                  + ["-e", joined_ssh_command]
                  + [source, f"{self._ssh_guest()}:{destination}"])
  
          # Try to push twice, check for rsync after the first failure
          try:
              rsync()
          except tmt.utils.RunError:
              try:
                  if self._check_rsync() == CheckRsyncOutcome.ALREADY_INSTALLED:
                      raise
                  rsync()
              except tmt.utils.RunError:
                  # Provide a reasonable error to the user
                  self.fail(
                      f"Failed to push workdir to the guest. This usually means "
                      f"that login as '{self.user}' to the guest does not work.")
                  raise
  
      def pull(self,
               source: Optional[str] = None,
               destination: Optional[str] = None,
               options: Optional[List[str]] = None,
               extend_options: Optional[List[str]] = None) -> None:
          f"""
          Pull files from the guest
  
          By default the whole plan workdir is synced from the same
          location on the guest. Use the 'source' and 'destination' to
          sync custom location, the 'options' parameter to modify
          default options '{" ".join(DEFAULT_RSYNC_PULL_OPTIONS)}'
          and 'extend_options' to extend them (e.g. by exclude).
          """
          # Abort if guest is unavailable
          if self.guest is None:
              if not self.opt('dry'):
                  raise tmt.utils.GeneralError('The guest is not available.')
  
          # Prepare options and the pull command
          options = options or DEFAULT_RSYNC_PULL_OPTIONS
          if extend_options is not None:
              options.extend(extend_options)
          if destination is None:
              destination = "/"
          if source is None:
              # TODO: with generic Step and Common, this should not be needed
              parent = cast(Provision, self.parent)
  
              assert parent.plan.workdir is not None
  
              source = parent.plan.workdir
              self.debug(f"Pull workdir from guest '{self.guest}'.")
          else:
              self.debug(f"Copy '{source}' from the guest to '{destination}'.")
  
          def rsync() -> None:
              """ Run the rsync command """
              # In closure, mypy has hard times to reason about the state of used variables.
              assert options
              assert source
              assert destination
  
              # TODO: _ssh_command() returns two incompatible types, we need to chose the right one.
              joined_ssh_command = self._ssh_command(join=True)
              assert isinstance(joined_ssh_command, str)
  
              self.run(
                  ["rsync"] + options
                  + ["-e", joined_ssh_command]
                  + [f"{self._ssh_guest()}:{source}", destination])
  
          # Try to pull twice, check for rsync after the first failure
          try:
              rsync()
          except tmt.utils.RunError:
              try:
                  if self._check_rsync() == CheckRsyncOutcome.ALREADY_INSTALLED:
                      raise
                  rsync()
              except tmt.utils.RunError:
                  # Provide a reasonable error to the user
                  self.fail(
                      f"Failed to pull workdir from the guest. "
                      f"This usually means that login as '{self.user}' "
                      f"to the guest does not work.")
                  raise
  
      def stop(self) -> None:
          """
          Stop the guest
  
          Shut down a running guest instance so that it does not consume
          any memory or cpu resources. If needed, perform any actions
          necessary to store the instance status to disk.
          """
  
          # Close the master ssh connection
          if self._ssh_master_process:
              self.debug("Close the master ssh connection.", level=3)
              try:
                  self._ssh_master_process.terminate()
                  self._ssh_master_process.wait(timeout=3)
              except subprocess.TimeoutExpired:
                  pass
  
          # Remove the ssh socket
          if self._ssh_socket_path and os.path.exists(self._ssh_socket_path):
              self.debug(
                  f"Remove ssh socket '{self._ssh_socket_path}'.", level=3)
              try:
                  os.unlink(self._ssh_socket_path)
              except OSError as error:
                  self.debug(f"Failed to remove the socket: {error}", level=3)
  
      def reboot(
              self,
              hard: bool = False,
              command: Optional[str] = None,
              timeout: Optional[int] = None,
              tick: float = tmt.utils.DEFAULT_WAIT_TICK,
              tick_increase: float = tmt.utils.DEFAULT_WAIT_TICK_INCREASE) -> bool:
          """
          Reboot the guest, return True if reconnect was successful
  
          Parameter 'hard' set to True means that guest should be
          rebooted by way which is not clean in sense that data can be
          lost. When set to False reboot should be done gracefully.
  
          Use the 'command' parameter to specify a custom reboot command
          instead of the default 'reboot'.
          """
  
          if hard:
              raise tmt.utils.ProvisionError(
                  "Method does not support hard reboot.")
  
          command = command or "reboot"
          self.debug(f"Reboot using the command '{command}'.")
  
          re_boot_time = re.compile(r'btime\s+(\d+)')
  
          def get_boot_time() -> int:
              """ Reads btime from /proc/stat """
              stdout = self.execute(["cat", "/proc/stat"]).stdout
              assert stdout
  
              match = re_boot_time.search(stdout)
  
              if match is None:
                  raise tmt.utils.ProvisionError('Failed to retrieve boot time from guest')
  
              return int(match.group(1))
  
          current_boot_time = get_boot_time()
  
          try:
              self.execute(command)
          except tmt.utils.RunError as error:
              # Connection can be closed by the remote host even before the
              # reboot command is completed. Let's ignore such errors.
              if error.returncode == 255:
                  self.debug(
                      "Seems the connection was closed too fast, ignoring.")
              else:
                  raise
  
          # Wait until we get new boot time, connection will drop and will be
          # unreachable for some time
          def check_boot_time() -> None:
              try:
                  new_boot_time = get_boot_time()
  
                  if new_boot_time != current_boot_time:
                      # Different boot time and we are reconnected
                      return
  
                  # Same boot time, reboot didn't happen yet, retrying
                  raise tmt.utils.WaitingIncomplete()
  
              except tmt.utils.RunError:
                  self.debug('Failed to connect to the guest.')
                  raise tmt.utils.WaitingIncomplete()
  
          timeout = timeout or CONNECTION_TIMEOUT
  
          try:
              tmt.utils.wait(
                  self,
                  check_boot_time,
                  datetime.timedelta(seconds=timeout),
                  tick=tick,
                  tick_increase=tick_increase)
  
          except tmt.utils.WaitingTimedOutError:
              self.debug("Connection to guest failed after reboot.")
              return False
  
          self.debug("Connection to guest succeeded after reboot.")
          return True
  
      def remove(self) -> None:
          """
          Remove the guest
  
          Completely remove all guest instance data so that it does not
          consume any disk resources.
          """
          self.debug(f"Doing nothing to remove guest '{self.guest}'.")
  
      def _check_rsync(self) -> CheckRsyncOutcome:
          """
          Make sure that rsync is installed on the guest
  
          On read-only distros install it under the '/root/pkg' directory.
          Returns 'already installed' when rsync is already present.
          """
  
          # Check for rsync (nothing to do if already installed)
          self.debug("Ensure that rsync is installed on the guest.")
          try:
              self.execute("rsync --version")
              return CheckRsyncOutcome.ALREADY_INSTALLED
          except tmt.utils.RunError:
              pass
  
          # Check the package manager
          self.debug("Check the package manager.")
          try:
              self.execute("dnf --version")
              package_manager = "dnf"
          except tmt.utils.RunError:
              package_manager = "yum"
  
          # Install under '/root/pkg' for read-only distros
          # (for now the check is based on 'rpm-ostree' presence)
          # FIXME: Find a better way how to detect read-only distros
          self.debug("Check for a read-only distro.")
          try:
              self.execute("rpm-ostree --version")
              readonly = (
                  " --installroot=/root/pkg --releasever / "
                  "&& ln -sf /root/pkg/bin/rsync /usr/local/bin/rsync")
          except tmt.utils.RunError:
              readonly = ""
  
          # Install the rsync
          self.execute(f"{package_manager} install -y rsync" + readonly)
  
          return CheckRsyncOutcome.INSTALLED
  
      @classmethod
      def requires(cls) -> List[str]:
          """ No extra requires needed """
          return []
  
files: tmt/steps/provision/__init__.py
files: import dataclasses
  import datetime
  import sys
  from typing import Any, Dict, List, Optional, cast
  
  import click
  import requests
  
  import tmt
  import tmt.options
  import tmt.steps
  import tmt.steps.provision
  import tmt.utils
  from tmt.utils import ProvisionError, retry_session, updatable_message
  
  if sys.version_info >= (3, 8):
      from typing import TypedDict
  else:
      from typing_extensions import TypedDict
  
  
  # List of Artemis API versions supported and understood by this plugin.
  # Since API gains support for new features over time, it is important to
  # know when particular feature became available, and avoid using it with
  # older APIs.
  SUPPORTED_API_VERSIONS = (
      # NEW: added hostname HW constraint
      '0.0.38',
      # NEW: virtualization HW constraint
      '0.0.37',
      # NEW: boot.method HW constraint
      '0.0.32',
      # NEW: network HW constraint
      '0.0.28'
      )
  
  
  # The default Artemis API version - the most recent supported versions
  # should be perfectly fine.
  DEFAULT_API_VERSION = SUPPORTED_API_VERSIONS[0]
  
  DEFAULT_API_URL = 'http://127.0.0.1:8001'
  DEFAULT_USER = 'root'
  DEFAULT_ARCH = 'x86_64'
  DEFAULT_PRIORITY_GROUP = 'default-priority'
  DEFAULT_KEYNAME = 'default'
  DEFAULT_PROVISION_TIMEOUT = 600
  DEFAULT_PROVISION_TICK = 60
  DEFAULT_API_TIMEOUT = 10
  DEFAULT_API_RETRIES = 10
  # Should lead to delays of 0.5, 1, 2, 4, 8, 16, 32, 64, 128, 256 seconds
  DEFAULT_RETRY_BACKOFF_FACTOR = 1
  
  # Type annotation for "data" package describing a guest instance. Passed
  # between load() and save() calls
  # TODO: get rid of `ignore` once superclass is no longer `Any`
  
  
  @dataclasses.dataclass
  class ArtemisGuestData(tmt.steps.provision.GuestSshData):
      # Override parent class with our defaults
      user: str = DEFAULT_USER
  
      # API
      api_url: str = DEFAULT_API_URL
      api_version: str = DEFAULT_API_VERSION
  
      # Guest request properties
      arch: str = DEFAULT_ARCH
      image: Optional[str] = None
      hardware: Optional[Any] = None
      pool: Optional[str] = None
      priority_group: str = DEFAULT_PRIORITY_GROUP
      keyname: str = DEFAULT_KEYNAME
      user_data: Dict[str, str] = dataclasses.field(default_factory=dict)
  
      # Provided by Artemis response
      guestname: Optional[str] = None
  
      # Timeouts and deadlines
      provision_timeout: int = DEFAULT_PROVISION_TIMEOUT
      provision_tick: int = DEFAULT_PROVISION_TICK
      api_timeout: int = DEFAULT_API_TIMEOUT
      api_retries: int = DEFAULT_API_RETRIES
      api_retry_backoff_factor: int = DEFAULT_RETRY_BACKOFF_FACTOR
  
  
  GUEST_STATE_COLOR_DEFAULT = 'green'
  
  GUEST_STATE_COLORS = {
      'routing': 'yellow',
      'provisioning': 'magenta',
      'promised': 'blue',
      'preparing': 'cyan',
      'cancelled': 'red',
      'error': 'red'
  }
  
  
  # Type annotation for Artemis API `GET /guests/$guestname` response.
  # Partial, not all fields necessary since plugin ignores most of them.
  GuestInspectType = TypedDict(
      'GuestInspectType', {
          'state': str,
          'address': Optional[str]
          }
      )
  
  
  class ArtemisAPI:
      def __init__(self, guest: 'GuestArtemis') -> None:
          self._guest = guest
  
          self.http_session = retry_session.create(
              retries=guest.api_retries,
              backoff_factor=guest.api_retry_backoff_factor,
              allowed_methods=('HEAD', 'GET', 'POST', 'DELETE', 'PUT'),
              status_forcelist=(
                  429,  # Too Many Requests
                  500,  # Internal Server Error
                  502,  # Bad Gateway
                  503,  # Service Unavailable
                  504   # Gateway Timeout
                  ),
              timeout=guest.api_timeout
              )
  
      def query(
              self,
              path: str,
              method: str = 'get',
              request_kwargs: Optional[Dict[str, Any]] = None
              ) -> requests.Response:
          """
          Base helper for Artemis API queries.
  
          Trivial dispatcher per method, returning retrieved response.
  
          :param path: API path to contact.
          :param method: HTTP method to use.
          :param request_kwargs: optional request options, as supported by
              :py:mod:`requests` library.
          """
  
          request_kwargs = request_kwargs or {}
  
          url = f'{self._guest.api_url}{path}'
  
          if method == 'get':
              return self.http_session.get(url, **request_kwargs)
  
          if method == 'post':
              return self.http_session.post(url, **request_kwargs)
  
          if method == 'delete':
              return self.http_session.delete(url, **request_kwargs)
  
          if method == 'put':
              return self.http_session.put(url, **request_kwargs)
  
          raise tmt.utils.GeneralError(
              f'Unsupported Artemis API method.\n{method}')
  
      def create(
              self,
              path: str,
              data: Dict[str, Any],
              request_kwargs: Optional[Dict[str, Any]] = None
              ) -> requests.Response:
          """
          Create - or request creation of - a resource.
  
          :param path: API path to contact.
          :param data: optional key/value data to send with the request.
          :param request_kwargs: optional request options, as supported by
              :py:mod:`requests` library.
          """
  
          request_kwargs = request_kwargs or {}
          request_kwargs['json'] = data
  
          return self.query(path, method='post', request_kwargs=request_kwargs)
  
      def inspect(
              self,
              path: str,
              params: Optional[Dict[str, Any]] = None,
              request_kwargs: Optional[Dict[str, Any]] = None
              ) -> requests.Response:
          """
          Inspect a resource.
  
          :param path: API path to contact.
          :param params: optional key/value query parameters.
          :param request_kwargs: optional request options, as supported by
              :py:mod:`requests` library.
          """
  
          request_kwargs = request_kwargs or {}
  
          if params:
              request_kwargs['params'] = params
  
          return self.query(path, request_kwargs=request_kwargs)
  
      def delete(
              self,
              path: str,
              request_kwargs: Optional[Dict[str, Any]] = None
              ) -> requests.Response:
          """
          Delete - or request removal of - a resource.
  
          :param path: API path to contact.
          :param request_kwargs: optional request options, as supported by
              :py:mod:`requests` library.
          """
  
          return self.query(path, method='delete', request_kwargs=request_kwargs)
  
  
  @tmt.steps.provides_method('artemis')
  class ProvisionArtemis(tmt.steps.provision.ProvisionPlugin):
      """
      Provision guest using Artemis backend
  
      Minimal configuration could look like this:
  
          provision:
              how: artemis
              image: Fedora
              api-url: https://your-artemis.com/
  
      Note that the actual value of "image" depends on what images - or
      "composes" as Artemis calls them - supports and can deliver.
  
      Note that "api-url" can be also given via ARTEMIS_API_URL
      environment variable.
  
      Full configuration example:
  
          provision:
              how: artemis
  
              # Artemis API
              api-url: https://your-artemis.com/
              api-version: 0.0.32
  
              # Mandatory environment properties
              image: Fedora
  
              # Optional environment properties
              arch: aarch64
              pool: optional-pool-name
  
              # Provisioning process control (optional)
              priority-group: custom-priority-group
              keyname: custom-SSH-key-name
  
              # Labels to be attached to guest request (optional)
              user-data:
                  foo: bar
  
              # Timeouts and deadlines (optional)
              provision-timeout: 3600
              provision-tick: 10
              api-timeout: 600
              api-retries: 5
              api-retry-backoff-factor: 1
      """
  
      # Guest instance
      _guest = None
  
      _keys = [
          'api-url',
          'api-version',
          'arch',
          'image',
          'hardware',
          'pool',
          'priority-group',
          'keyname',
          'user-data',
          'provision-timeout',
          'provision-tick',
          'api-timeout',
          'api-retries',
          'api-retry-backoff-factor']
  
      # TODO: fix types once superclass gains its annotations
      @classmethod
      def options(cls, how: Any = None) -> List[tmt.options.ClickOptionDecoratorType]:
          """ Prepare command line options for Artemis """
          return cast(List[tmt.options.ClickOptionDecoratorType], [
              click.option(
                  '--api-url', metavar='URL',
                  help="Artemis API URL.",
                  envvar='ARTEMIS_API_URL'
                  ),
              click.option(
                  '--api-version', metavar='x.y.z',
                  help="Artemis API version to use.",
                  type=click.Choice(SUPPORTED_API_VERSIONS),
                  envvar='ARTEMIS_API_VERSION'
                  ),
              click.option(
                  '--arch', metavar='ARCH',
                  help='Architecture to provision.'
                  ),
              click.option(
                  '--image', metavar='COMPOSE',
                  help='Image (or "compose" in Artemis terminology) '
                       'to provision.'
                  ),
              click.option(
                  '--pool', metavar='NAME',
                  help='Pool to enforce.'
                  ),
              click.option(
                  '--priority-group', metavar='NAME',
                  help='Provisioning priority group.'
                  ),
              click.option(
                  '--keyname', metavar='NAME',
                  help='SSH key name.'
                  ),
              click.option(
                  '--user-data', metavar='KEY=VALUE',
                  help='Optional data to attach to guest.',
                  multiple=True,
                  default=[]
                  ),
              click.option(
                  '--provision-timeout', metavar='SECONDS',
                  help=f'How long to wait for provisioning to complete, '
                       f'{DEFAULT_PROVISION_TIMEOUT} seconds by default.'
                  ),
              click.option(
                  '--provision-tick', metavar='SECONDS',
                  help=f'How often check Artemis API for provisioning status, '
                       f'{DEFAULT_PROVISION_TICK} seconds by default.',
                  ),
              click.option(
                  '--api-timeout', metavar='SECONDS',
                  help=f'How long to wait for API operations to complete, '
                       f'{DEFAULT_API_TIMEOUT} seconds by default.',
                  ),
              click.option(
                  '--api-retries', metavar='COUNT',
                  help=f'How many attempts to use when talking to API, '
                       f'{DEFAULT_API_RETRIES} by default.',
                  ),
              click.option(
                  '--api-retry-backoff-factor', metavar='COUNT',
                  help=f'A factor for exponential API retry backoff, '
                       f'{DEFAULT_RETRY_BACKOFF_FACTOR} by default.',
                  ),
              ]) + super().options(how)
  
      def default(self, option: str, default: Optional[Any] = None) -> Any:
          """ Return default data for given option """
  
          return getattr(ArtemisGuestData(), option.replace('-', '_'), default)
  
      # More specific type is a violation of Liskov substitution principle, and mypy
      # complains about it - rightfully so. Ignoring the issue which should be resolved
      # with https://github.com/teemtee/tmt/pull/1439.
      def wake(self, data: Optional[ArtemisGuestData] = None) -> None:  # type: ignore[override]
          """ Wake up the plugin, process data, apply options """
  
          super().wake(data=data)
  
          if data:
              self._guest = GuestArtemis(data, name=self.name, parent=self.step)
  
      def go(self) -> None:
          """ Provision the guest """
          super().go()
  
          api_version = self.get('api-version')
  
          if api_version not in SUPPORTED_API_VERSIONS:
              raise ProvisionError(f"API version '{api_version}' not supported.")
  
          try:
              user_data = {
                  key.strip(): value.strip()
                  for key, value in (
                      pair.split('=', 1)
                      for pair in self.get('user-data')
                      )
                  }
  
          except ValueError:
              raise ProvisionError('Cannot parse user-data.')
  
          data = ArtemisGuestData(
              api_url=self.get('api-url'),
              api_version=api_version,
              arch=self.get('arch'),
              image=self.get('image'),
              hardware=self.get('hardware'),
              pool=self.get('pool'),
              priority_group=self.get('priority-group'),
              keyname=self.get('keyname'),
              user_data=user_data,
              user=self.get('user'),
              provision_timeout=self.get('provision-timeout'),
              provision_tick=self.get('provision-tick'),
              api_timeout=self.get('api-timeout'),
              api_retries=self.get('api-retries'),
              api_retry_backoff_factor=self.get('api-retry-backoff-factor')
              )
  
          self._guest = GuestArtemis(data, name=self.name, parent=self.step)
          self._guest.start()
  
      def guest(self) -> Optional['GuestArtemis']:
          """ Return the provisioned guest """
          return self._guest
  
  
  class GuestArtemis(tmt.GuestSsh):
      """
      Artemis guest instance
  
      The following keys are expected in the 'data' dictionary:
      """
  
      # API
      api_url: str
      api_version: str
  
      # Guest request properties
      arch: str
      image: str
      hardware: Optional[Any]
      pool: Optional[str]
      priority_group: str
      keyname: str
      user_data: Dict[str, str]
  
      # Provided by Artemis response
      guestname: Optional[str]
  
      # Timeouts and deadlines
      provision_timeout: int
      provision_tick: int
      api_timeout: int
      api_retries: int
      api_retry_backoff_factor: int
  
      _api: Optional[ArtemisAPI] = None
  
      @property
      def api(self) -> ArtemisAPI:
          if self._api is None:
              self._api = ArtemisAPI(self)
  
          return self._api
  
      def _create(self) -> None:
          environment: Dict[str, Any] = {
              'hw': {
                  'arch': self.arch
                  },
              'os': {
                  'compose': self.image
                  }
              }
  
          data: Dict[str, Any] = {
              'environment': environment,
              'keyname': self.keyname,
              'priority_group': self.priority_group,
              'user_data': self.user_data
              }
  
          if self.pool:
              environment['pool'] = self.pool
  
          if self.hardware is not None:
              assert isinstance(self.hardware, dict)
  
              environment['hw']['constraints'] = self.hardware
  
          # TODO: snapshots
          # TODO: spot instance
          # TODO: post-install script
          # TODO: log types
  
          response = self.api.create('/guests/', data)
  
          if response.status_code == 201:
              self.info('guest', 'has been requested', 'green')
  
          else:
              raise ProvisionError(
                  f"Failed to create, "
                  f"unhandled API response '{response.status_code}'.")
  
          self.guestname = response.json()['guestname']
          self.info('guestname', self.guestname, 'green')
  
          with updatable_message(
                  'state', indent_level=self._level()) as progress_message:
  
              def get_new_state() -> GuestInspectType:
                  response = self.api.inspect(f'/guests/{self.guestname}')
  
                  if response.status_code != 200:
                      raise ProvisionError(
                          f"Failed to create, "
                          f"unhandled API response '{response.status_code}'.")
  
                  current = cast(GuestInspectType, response.json())
                  state = current['state']
                  state_color = GUEST_STATE_COLORS.get(
                      state, GUEST_STATE_COLOR_DEFAULT)
  
                  progress_message.update(state, color=state_color)
  
                  if state == 'error':
                      raise ProvisionError(
                          'Failed to create, provisioning failed.')
  
                  if state == 'ready':
                      return current
  
                  raise tmt.utils.WaitingIncomplete()
  
              try:
                  guest_info = tmt.utils.wait(
                      self, get_new_state, datetime.timedelta(
                          seconds=self.provision_timeout), tick=self.provision_tick)
  
              except tmt.utils.WaitingTimedOutError:
                  raise ProvisionError(
                      f'Failed to provision in the given amount '
                      f'of time (--provision-timeout={self.provision_timeout}).')
  
          self.guest = guest_info['address']
          self.info('address', self.guest, 'green')
  
      def start(self) -> None:
          """
          Start the guest
  
          Get a new guest instance running. This should include preparing
          any configuration necessary to get it started. Called after
          load() is completed so all guest data should be available.
          """
  
          if self.guestname is None or self.guest is None:
              self._create()
  
      def remove(self) -> None:
          """ Remove the guest """
  
          if self.guestname is None:
              return
  
          response = self.api.delete(f'/guests/{self.guestname}')
  
          if response.status_code == 404:
              self.info('guest', 'no longer exists', 'red')
  
          elif response.status_code == 409:
              self.info('guest', 'has existing snapshots', 'red')
  
          elif response.ok:
              self.info('guest', 'has been removed', 'green')
  
          else:
              self.info(
                  'guest',
                  f"Failed to remove, "
                  f"unhandled API response '{response.status_code}'.")
  
files: tmt/steps/provision/artemis.py
files: import dataclasses
  import datetime
  import sys
  from typing import Any, Dict, List, Optional, cast
  
  import click
  import requests
  
  import tmt
  import tmt.options
  import tmt.steps
  import tmt.steps.provision
  import tmt.utils
  from tmt.utils import ProvisionError, retry_session, updatable_message
  
  if sys.version_info >= (3, 8):
      from typing import TypedDict
  else:
      from typing_extensions import TypedDict
  
  
  # List of Artemis API versions supported and understood by this plugin.
  # Since API gains support for new features over time, it is important to
  # know when particular feature became available, and avoid using it with
  # older APIs.
  SUPPORTED_API_VERSIONS = (
      # NEW: added hostname HW constraint
      '0.0.38',
      # NEW: virtualization HW constraint
      '0.0.37',
      # NEW: boot.method HW constraint
      '0.0.32',
      # NEW: network HW constraint
      '0.0.28'
      )
  
  
  # The default Artemis API version - the most recent supported versions
  # should be perfectly fine.
  DEFAULT_API_VERSION = SUPPORTED_API_VERSIONS[0]
  
  DEFAULT_API_URL = 'http://127.0.0.1:8001'
  DEFAULT_USER = 'root'
  DEFAULT_ARCH = 'x86_64'
  DEFAULT_PRIORITY_GROUP = 'default-priority'
  DEFAULT_KEYNAME = 'default'
  DEFAULT_PROVISION_TIMEOUT = 600
  DEFAULT_PROVISION_TICK = 60
  DEFAULT_API_TIMEOUT = 10
  DEFAULT_API_RETRIES = 10
  # Should lead to delays of 0.5, 1, 2, 4, 8, 16, 32, 64, 128, 256 seconds
  DEFAULT_RETRY_BACKOFF_FACTOR = 1
  
  # Type annotation for "data" package describing a guest instance. Passed
  # between load() and save() calls
  # TODO: get rid of `ignore` once superclass is no longer `Any`
  
  
  @dataclasses.dataclass
  class ArtemisGuestData(tmt.steps.provision.GuestSshData):
      # Override parent class with our defaults
      user: str = DEFAULT_USER
  
      # API
      api_url: str = DEFAULT_API_URL
      api_version: str = DEFAULT_API_VERSION
  
      # Guest request properties
      arch: str = DEFAULT_ARCH
      image: Optional[str] = None
      hardware: Optional[Any] = None
      pool: Optional[str] = None
      priority_group: str = DEFAULT_PRIORITY_GROUP
      keyname: str = DEFAULT_KEYNAME
      user_data: Dict[str, str] = dataclasses.field(default_factory=dict)
  
      # Provided by Artemis response
      guestname: Optional[str] = None
  
      # Timeouts and deadlines
      provision_timeout: int = DEFAULT_PROVISION_TIMEOUT
      provision_tick: int = DEFAULT_PROVISION_TICK
      api_timeout: int = DEFAULT_API_TIMEOUT
      api_retries: int = DEFAULT_API_RETRIES
      api_retry_backoff_factor: int = DEFAULT_RETRY_BACKOFF_FACTOR
  
  
  GUEST_STATE_COLOR_DEFAULT = 'green'
  
  GUEST_STATE_COLORS = {
      'routing': 'yellow',
      'provisioning': 'magenta',
      'promised': 'blue',
      'preparing': 'cyan',
      'cancelled': 'red',
      'error': 'red'
  }
  
  
  # Type annotation for Artemis API `GET /guests/$guestname` response.
  # Partial, not all fields necessary since plugin ignores most of them.
  GuestInspectType = TypedDict(
      'GuestInspectType', {
          'state': str,
          'address': Optional[str]
          }
      )
  
  
  class ArtemisAPI:
      def __init__(self, guest: 'GuestArtemis') -> None:
          self._guest = guest
  
          self.http_session = retry_session.create(
              retries=guest.api_retries,
              backoff_factor=guest.api_retry_backoff_factor,
              allowed_methods=('HEAD', 'GET', 'POST', 'DELETE', 'PUT'),
              status_forcelist=(
                  429,  # Too Many Requests
                  500,  # Internal Server Error
                  502,  # Bad Gateway
                  503,  # Service Unavailable
                  504   # Gateway Timeout
                  ),
              timeout=guest.api_timeout
              )
  
      def query(
              self,
              path: str,
              method: str = 'get',
              request_kwargs: Optional[Dict[str, Any]] = None
              ) -> requests.Response:
          """
          Base helper for Artemis API queries.
  
          Trivial dispatcher per method, returning retrieved response.
  
          :param path: API path to contact.
          :param method: HTTP method to use.
          :param request_kwargs: optional request options, as supported by
              :py:mod:`requests` library.
          """
  
          request_kwargs = request_kwargs or {}
  
          url = f'{self._guest.api_url}{path}'
  
          if method == 'get':
              return self.http_session.get(url, **request_kwargs)
  
          if method == 'post':
              return self.http_session.post(url, **request_kwargs)
  
          if method == 'delete':
              return self.http_session.delete(url, **request_kwargs)
  
          if method == 'put':
              return self.http_session.put(url, **request_kwargs)
  
          raise tmt.utils.GeneralError(
              f'Unsupported Artemis API method.\n{method}')
  
      def create(
              self,
              path: str,
              data: Dict[str, Any],
              request_kwargs: Optional[Dict[str, Any]] = None
              ) -> requests.Response:
          """
          Create - or request creation of - a resource.
  
          :param path: API path to contact.
          :param data: optional key/value data to send with the request.
          :param request_kwargs: optional request options, as supported by
              :py:mod:`requests` library.
          """
  
          request_kwargs = request_kwargs or {}
          request_kwargs['json'] = data
  
          return self.query(path, method='post', request_kwargs=request_kwargs)
  
      def inspect(
              self,
              path: str,
              params: Optional[Dict[str, Any]] = None,
              request_kwargs: Optional[Dict[str, Any]] = None
              ) -> requests.Response:
          """
          Inspect a resource.
  
          :param path: API path to contact.
          :param params: optional key/value query parameters.
          :param request_kwargs: optional request options, as supported by
              :py:mod:`requests` library.
          """
  
          request_kwargs = request_kwargs or {}
  
          if params:
              request_kwargs['params'] = params
  
          return self.query(path, request_kwargs=request_kwargs)
  
      def delete(
              self,
              path: str,
              request_kwargs: Optional[Dict[str, Any]] = None
              ) -> requests.Response:
          """
          Delete - or request removal of - a resource.
  
          :param path: API path to contact.
          :param request_kwargs: optional request options, as supported by
              :py:mod:`requests` library.
          """
  
          return self.query(path, method='delete', request_kwargs=request_kwargs)
  
  
  @tmt.steps.provides_method('artemis')
  class ProvisionArtemis(tmt.steps.provision.ProvisionPlugin):
      """
      Provision guest using Artemis backend
  
      Minimal configuration could look like this:
  
          provision:
              how: artemis
              image: Fedora
              api-url: https://your-artemis.com/
  
      Note that the actual value of "image" depends on what images - or
      "composes" as Artemis calls them - supports and can deliver.
  
      Note that "api-url" can be also given via ARTEMIS_API_URL
      environment variable.
  
      Full configuration example:
  
          provision:
              how: artemis
  
              # Artemis API
              api-url: https://your-artemis.com/
              api-version: 0.0.32
  
              # Mandatory environment properties
              image: Fedora
  
              # Optional environment properties
              arch: aarch64
              pool: optional-pool-name
  
              # Provisioning process control (optional)
              priority-group: custom-priority-group
              keyname: custom-SSH-key-name
  
              # Labels to be attached to guest request (optional)
              user-data:
                  foo: bar
  
              # Timeouts and deadlines (optional)
              provision-timeout: 3600
              provision-tick: 10
              api-timeout: 600
              api-retries: 5
              api-retry-backoff-factor: 1
      """
  
      # Guest instance
      _guest = None
  
      _keys = [
          'api-url',
          'api-version',
          'arch',
          'image',
          'hardware',
          'pool',
          'priority-group',
          'keyname',
          'user-data',
          'provision-timeout',
          'provision-tick',
          'api-timeout',
          'api-retries',
          'api-retry-backoff-factor']
  
      # TODO: fix types once superclass gains its annotations
      @classmethod
      def options(cls, how: Any = None) -> List[tmt.options.ClickOptionDecoratorType]:
          """ Prepare command line options for Artemis """
          return cast(List[tmt.options.ClickOptionDecoratorType], [
              click.option(
                  '--api-url', metavar='URL',
                  help="Artemis API URL.",
                  envvar='ARTEMIS_API_URL'
                  ),
              click.option(
                  '--api-version', metavar='x.y.z',
                  help="Artemis API version to use.",
                  type=click.Choice(SUPPORTED_API_VERSIONS),
                  envvar='ARTEMIS_API_VERSION'
                  ),
              click.option(
                  '--arch', metavar='ARCH',
                  help='Architecture to provision.'
                  ),
              click.option(
                  '--image', metavar='COMPOSE',
                  help='Image (or "compose" in Artemis terminology) '
                       'to provision.'
                  ),
              click.option(
                  '--pool', metavar='NAME',
                  help='Pool to enforce.'
                  ),
              click.option(
                  '--priority-group', metavar='NAME',
                  help='Provisioning priority group.'
                  ),
              click.option(
                  '--keyname', metavar='NAME',
                  help='SSH key name.'
                  ),
              click.option(
                  '--user-data', metavar='KEY=VALUE',
                  help='Optional data to attach to guest.',
                  multiple=True,
                  default=[]
                  ),
              click.option(
                  '--provision-timeout', metavar='SECONDS',
                  help=f'How long to wait for provisioning to complete, '
                       f'{DEFAULT_PROVISION_TIMEOUT} seconds by default.'
                  ),
              click.option(
                  '--provision-tick', metavar='SECONDS',
                  help=f'How often check Artemis API for provisioning status, '
                       f'{DEFAULT_PROVISION_TICK} seconds by default.',
                  ),
              click.option(
                  '--api-timeout', metavar='SECONDS',
                  help=f'How long to wait for API operations to complete, '
                       f'{DEFAULT_API_TIMEOUT} seconds by default.',
                  ),
              click.option(
                  '--api-retries', metavar='COUNT',
                  help=f'How many attempts to use when talking to API, '
                       f'{DEFAULT_API_RETRIES} by default.',
                  ),
              click.option(
                  '--api-retry-backoff-factor', metavar='COUNT',
                  help=f'A factor for exponential API retry backoff, '
                       f'{DEFAULT_RETRY_BACKOFF_FACTOR} by default.',
                  ),
              ]) + super().options(how)
  
      def default(self, option: str, default: Optional[Any] = None) -> Any:
          """ Return default data for given option """
  
          return getattr(ArtemisGuestData(), option.replace('-', '_'), default)
  
      # More specific type is a violation of Liskov substitution principle, and mypy
      # complains about it - rightfully so. Ignoring the issue which should be resolved
      # with https://github.com/teemtee/tmt/pull/1439.
      def wake(self, keys: Optional[List[str]] = None,  # type: ignore[override]
               data: Optional[ArtemisGuestData] = None) -> None:
          """ Wake up the plugin, process data, apply options """
  
          super().wake(keys=keys, data=data)
  
          if data:
              self._guest = GuestArtemis(data, name=self.name, parent=self.step)
  
      def go(self) -> None:
          """ Provision the guest """
          super().go()
  
          api_version = self.get('api-version')
  
          if api_version not in SUPPORTED_API_VERSIONS:
              raise ProvisionError(f"API version '{api_version}' not supported.")
  
          try:
              user_data = {
                  key.strip(): value.strip()
                  for key, value in (
                      pair.split('=', 1)
                      for pair in self.get('user-data')
                      )
                  }
  
          except ValueError:
              raise ProvisionError('Cannot parse user-data.')
  
          data = ArtemisGuestData(
              api_url=self.get('api-url'),
              api_version=api_version,
              arch=self.get('arch'),
              image=self.get('image'),
              hardware=self.get('hardware'),
              pool=self.get('pool'),
              priority_group=self.get('priority-group'),
              keyname=self.get('keyname'),
              user_data=user_data,
              user=self.get('user'),
              provision_timeout=self.get('provision-timeout'),
              provision_tick=self.get('provision-tick'),
              api_timeout=self.get('api-timeout'),
              api_retries=self.get('api-retries'),
              api_retry_backoff_factor=self.get('api-retry-backoff-factor')
              )
  
          self._guest = GuestArtemis(data, name=self.name, parent=self.step)
          self._guest.start()
  
      def guest(self) -> Optional['GuestArtemis']:
          """ Return the provisioned guest """
          return self._guest
  
  
  class GuestArtemis(tmt.GuestSsh):
      """
      Artemis guest instance
  
      The following keys are expected in the 'data' dictionary:
      """
  
      # API
      api_url: str
      api_version: str
  
      # Guest request properties
      arch: str
      image: str
      hardware: Optional[Any]
      pool: Optional[str]
      priority_group: str
      keyname: str
      user_data: Dict[str, str]
  
      # Provided by Artemis response
      guestname: Optional[str]
  
      # Timeouts and deadlines
      provision_timeout: int
      provision_tick: int
      api_timeout: int
      api_retries: int
      api_retry_backoff_factor: int
  
      _api: Optional[ArtemisAPI] = None
  
      @property
      def api(self) -> ArtemisAPI:
          if self._api is None:
              self._api = ArtemisAPI(self)
  
          return self._api
  
      def _create(self) -> None:
          environment: Dict[str, Any] = {
              'hw': {
                  'arch': self.arch
                  },
              'os': {
                  'compose': self.image
                  }
              }
  
          data: Dict[str, Any] = {
              'environment': environment,
              'keyname': self.keyname,
              'priority_group': self.priority_group,
              'user_data': self.user_data
              }
  
          if self.pool:
              environment['pool'] = self.pool
  
          if self.hardware is not None:
              assert isinstance(self.hardware, dict)
  
              environment['hw']['constraints'] = self.hardware
  
          # TODO: snapshots
          # TODO: spot instance
          # TODO: post-install script
          # TODO: log types
  
          response = self.api.create('/guests/', data)
  
          if response.status_code == 201:
              self.info('guest', 'has been requested', 'green')
  
          else:
              raise ProvisionError(
                  f"Failed to create, "
                  f"unhandled API response '{response.status_code}'.")
  
          self.guestname = response.json()['guestname']
          self.info('guestname', self.guestname, 'green')
  
          with updatable_message(
                  'state', indent_level=self._level()) as progress_message:
  
              def get_new_state() -> GuestInspectType:
                  response = self.api.inspect(f'/guests/{self.guestname}')
  
                  if response.status_code != 200:
                      raise ProvisionError(
                          f"Failed to create, "
                          f"unhandled API response '{response.status_code}'.")
  
                  current = cast(GuestInspectType, response.json())
                  state = current['state']
                  state_color = GUEST_STATE_COLORS.get(
                      state, GUEST_STATE_COLOR_DEFAULT)
  
                  progress_message.update(state, color=state_color)
  
                  if state == 'error':
                      raise ProvisionError(
                          'Failed to create, provisioning failed.')
  
                  if state == 'ready':
                      return current
  
                  raise tmt.utils.WaitingIncomplete()
  
              try:
                  guest_info = tmt.utils.wait(
                      self, get_new_state, datetime.timedelta(
                          seconds=self.provision_timeout), tick=self.provision_tick)
  
              except tmt.utils.WaitingTimedOutError:
                  raise ProvisionError(
                      f'Failed to provision in the given amount '
                      f'of time (--provision-timeout={self.provision_timeout}).')
  
          self.guest = guest_info['address']
          self.info('address', self.guest, 'green')
  
      def start(self) -> None:
          """
          Start the guest
  
          Get a new guest instance running. This should include preparing
          any configuration necessary to get it started. Called after
          load() is completed so all guest data should be available.
          """
  
          if self.guestname is None or self.guest is None:
              self._create()
  
      def remove(self) -> None:
          """ Remove the guest """
  
          if self.guestname is None:
              return
  
          response = self.api.delete(f'/guests/{self.guestname}')
  
          if response.status_code == 404:
              self.info('guest', 'no longer exists', 'red')
  
          elif response.status_code == 409:
              self.info('guest', 'has existing snapshots', 'red')
  
          elif response.ok:
              self.info('guest', 'has been removed', 'green')
  
          else:
              self.info(
                  'guest',
                  f"Failed to remove, "
                  f"unhandled API response '{response.status_code}'.")
  
files: tmt/steps/provision/artemis.py
files: from typing import Any, List, Optional
  
  import click
  
  import tmt
  import tmt.steps
  import tmt.steps.provision
  import tmt.utils
  from tmt.steps.provision import GuestSshData
  
  
  @tmt.steps.provides_method('connect')
  class ProvisionConnect(tmt.steps.provision.ProvisionPlugin):
      """
      Connect to a provisioned guest using ssh
  
      Private key authentication:
  
          provision:
              how: connect
              guest: host.example.org
              user: root
              key: /home/psss/.ssh/example_rsa
  
      Password authentication:
  
          provision:
              how: connect
              guest: host.example.org
              user: root
              password: secret
  
      User defaults to 'root', so if you have private key correctly set
      the minimal configuration can look like this:
  
          provision:
              how: connect
              guest: host.example.org
      """
  
      # Guest instance
      _guest = None
  
      # Supported keys
      _keys = ["guest", "key", "user", "password", "port"]
  
      @classmethod
      def options(cls, how: Optional[str] = None) -> List[tmt.options.ClickOptionDecoratorType]:
          """ Prepare command line options for connect """
          options = super().options(how)
          options[:0] = [
              click.option(
                  '-g', '--guest', metavar='GUEST',
                  help='Select remote host to connect to (hostname or ip).'),
              click.option(
                  '-P', '--port', metavar='PORT',
                  help='Use specific port to connect to.'),
              click.option(
                  '-k', '--key', metavar='PRIVATE_KEY',
                  help='Private key for login into the guest system.'),
              click.option(
                  '-u', '--user', metavar='USER',
                  help='Username to use for all guest operations.'),
              click.option(
                  '-p', '--password', metavar='PASSWORD',
                  help='Password for login into the guest system.'),
              ]
          return options
  
      def default(self, option: str, default: Optional[Any] = None) -> Any:
          """ Return default data for given option """
          # User root as the default user
          if option == 'user':
              return 'root'
          # No other defaults available
          return default
  
      def wake(self, data: Optional[GuestSshData] = None) -> None:  # type: ignore[override]
          """ Wake up the plugin, process data, apply options """
          super().wake(data=data)
          if data:
              self._guest = tmt.GuestSsh(data, name=self.name, parent=self.step)
  
      def go(self) -> None:
          """ Prepare the connection """
          super().go()
  
          # Check connection details
          guest = self.get('guest')
          user = self.get('user')
          key = self.get('key')
          password = self.get('password')
          port = self.get('port')
  
          # Check guest and auth info
          if not guest:
              raise tmt.utils.SpecificationError(
                  'Provide a host name or an ip address to connect.')
          data = GuestSshData(
              role=self.get('role'),
              guest=guest,
              user=user
              )
          self.info('guest', guest, 'green')
          self.info('user', user, 'green')
          if port:
              self.info('port', port, 'green')
              data.port = port
  
          # Use provided password for authentication
          if password:
              self.info('password', password, 'green')
              self.debug('Using password authentication.')
              data.password = password
          # Default to using a private key (user can have configured one)
          else:
              self.info('key', key or 'not provided', 'green')
              self.debug('Using private key authentication.')
              data.key = [key]
  
          # And finally create the guest
          self._guest = tmt.GuestSsh(data, name=self.name, parent=self.step)
  
      def guest(self) -> Optional[tmt.GuestSsh]:
          """ Return the provisioned guest """
          return self._guest
  
files: tmt/steps/provision/connect.py
files: from typing import Any, List, Optional
  
  import click
  
  import tmt
  import tmt.steps
  import tmt.steps.provision
  import tmt.utils
  from tmt.steps.provision import GuestSshData
  
  
  @tmt.steps.provides_method('connect')
  class ProvisionConnect(tmt.steps.provision.ProvisionPlugin):
      """
      Connect to a provisioned guest using ssh
  
      Private key authentication:
  
          provision:
              how: connect
              guest: host.example.org
              user: root
              key: /home/psss/.ssh/example_rsa
  
      Password authentication:
  
          provision:
              how: connect
              guest: host.example.org
              user: root
              password: secret
  
      User defaults to 'root', so if you have private key correctly set
      the minimal configuration can look like this:
  
          provision:
              how: connect
              guest: host.example.org
      """
  
      # Guest instance
      _guest = None
  
      # Supported keys
      _keys = ["guest", "key", "user", "password", "port"]
  
      @classmethod
      def options(cls, how: Optional[str] = None) -> List[tmt.options.ClickOptionDecoratorType]:
          """ Prepare command line options for connect """
          options = super().options(how)
          options[:0] = [
              click.option(
                  '-g', '--guest', metavar='GUEST',
                  help='Select remote host to connect to (hostname or ip).'),
              click.option(
                  '-P', '--port', metavar='PORT',
                  help='Use specific port to connect to.'),
              click.option(
                  '-k', '--key', metavar='PRIVATE_KEY',
                  help='Private key for login into the guest system.'),
              click.option(
                  '-u', '--user', metavar='USER',
                  help='Username to use for all guest operations.'),
              click.option(
                  '-p', '--password', metavar='PASSWORD',
                  help='Password for login into the guest system.'),
              ]
          return options
  
      def default(self, option: str, default: Optional[Any] = None) -> Any:
          """ Return default data for given option """
          # User root as the default user
          if option == 'user':
              return 'root'
          # No other defaults available
          return default
  
      # More specific type is a violation of Liskov substitution principle, and mypy
      # complains about it - rightfully so. Ignoring the issue which should be resolved
      # with https://github.com/teemtee/tmt/pull/1439.
      def wake(self, keys: Optional[List[str]] = None,  # type: ignore[override]
               data: Optional[GuestSshData] = None) -> None:
          """ Wake up the plugin, process data, apply options """
          super().wake(keys=keys, data=data)
          if data:
              self._guest = tmt.GuestSsh(data, name=self.name, parent=self.step)
  
      def go(self) -> None:
          """ Prepare the connection """
          super().go()
  
          # Check connection details
          guest = self.get('guest')
          user = self.get('user')
          key = self.get('key')
          password = self.get('password')
          port = self.get('port')
  
          # Check guest and auth info
          if not guest:
              raise tmt.utils.SpecificationError(
                  'Provide a host name or an ip address to connect.')
          data = GuestSshData(
              role=self.get('role'),
              guest=guest,
              user=user
              )
          self.info('guest', guest, 'green')
          self.info('user', user, 'green')
          if port:
              self.info('port', port, 'green')
              data.port = port
  
          # Use provided password for authentication
          if password:
              self.info('password', password, 'green')
              self.debug('Using password authentication.')
              data.password = password
          # Default to using a private key (user can have configured one)
          else:
              self.info('key', key or 'not provided', 'green')
              self.debug('Using private key authentication.')
              data.key = [key]
  
          # And finally create the guest
          self._guest = tmt.GuestSsh(data, name=self.name, parent=self.step)
  
      def guest(self) -> Optional[tmt.GuestSsh]:
          """ Return the provisioned guest """
          return self._guest
  
files: tmt/steps/provision/connect.py
files: import tmt
  import tmt.steps
  import tmt.steps.provision
  
  
  @tmt.steps.provides_method('local')
  class ProvisionLocal(tmt.steps.provision.ProvisionPlugin):
      """
      Use local host for test execution
  
      In general it is not recommended to run tests on your local machine
      as there might be security risks. Run only those tests which you
      know are safe so that you don't destroy your laptop ;-)
  
      Example config:
  
          provision:
              how: local
  
      Note that 'tmt run' is expected to be executed under a regular user.
      If there are admin rights required (for example in the prepare step)
      you might be asked for a sudo password.
      """
  
      # Guest instance
      _guest = None
  
      def wake(self, data=None):
          """ Wake up the plugin, process data, apply options """
          super().wake(data=data)
          if data:
              self._guest = GuestLocal(data, name=self.name, parent=self.step)
  
      def go(self):
          """ Provision the container """
          super().go()
  
          # Create a GuestLocal instance
          data = tmt.steps.provision.GuestSshData(
              guest='localhost',
              role=self.get('role')
              )
          self._guest = GuestLocal(data, name=self.name, parent=self.step)
  
      def guest(self):
          """ Return the provisioned guest """
          return self._guest
  
      def requires(self):
          """ List of required packages needed for workdir sync """
          return GuestLocal.requires()
  
  
  class GuestLocal(tmt.Guest):
      """ Local Host """
  
      localhost = True
  
      def ansible(self, playbook, extra_args=None):
          """ Prepare localhost using ansible playbook """
          playbook = self._ansible_playbook_path(playbook)
          stdout, stderr = self.run(
              ['sudo', '-E', 'ansible-playbook'] +
              self._ansible_verbosity() +
              self._ansible_extra_args(extra_args) +
              ['-c', 'local', '-i', 'localhost,', playbook],
              env=self._prepare_environment())
          self._ansible_summary(stdout)
  
      def execute(self, command, **kwargs):
          """ Execute command on localhost """
          # Prepare the environment (plan/cli variables override)
          environment = dict()
          environment.update(kwargs.pop('env', dict()))
          environment.update(self.parent.plan.environment)
          # Run the command under the prepared environment
          return self.run(command, env=environment, shell=True, **kwargs)
  
      def stop(self):
          """ Stop the guest """
  
          self.debug(f"Doing nothing to stop guest '{self.guest}'.")
  
      def reboot(self, hard=False, command=None, timeout=None):
          """ Reboot the guest, return True if successful """
  
          self.debug(f"Doing nothing to reboot guest '{self.guest}'.")
  
          return False
  
      def push(self, source=None, destination=None, options=None):
          """ Nothing to be done to push workdir """
  
      def pull(
              self,
              source=None,
              destination=None,
              options=None,
              extend_options=None):
          """ Nothing to be done to pull workdir """
  
files: tmt/steps/provision/local.py
files: import tmt
  import tmt.steps
  import tmt.steps.provision
  
  
  @tmt.steps.provides_method('local')
  class ProvisionLocal(tmt.steps.provision.ProvisionPlugin):
      """
      Use local host for test execution
  
      In general it is not recommended to run tests on your local machine
      as there might be security risks. Run only those tests which you
      know are safe so that you don't destroy your laptop ;-)
  
      Example config:
  
          provision:
              how: local
  
      Note that 'tmt run' is expected to be executed under a regular user.
      If there are admin rights required (for example in the prepare step)
      you might be asked for a sudo password.
      """
  
      # Guest instance
      _guest = None
  
      def wake(self, keys=None, data=None):
          """ Wake up the plugin, process data, apply options """
          super().wake(keys=keys, data=data)
          if data:
              self._guest = GuestLocal(data, name=self.name, parent=self.step)
  
      def go(self):
          """ Provision the container """
          super().go()
  
          # Create a GuestLocal instance
          data = tmt.steps.provision.GuestSshData(
              guest='localhost',
              role=self.get('role')
              )
          self._guest = GuestLocal(data, name=self.name, parent=self.step)
  
      def guest(self):
          """ Return the provisioned guest """
          return self._guest
  
      def requires(self):
          """ List of required packages needed for workdir sync """
          return GuestLocal.requires()
  
  
  class GuestLocal(tmt.Guest):
      """ Local Host """
  
      localhost = True
  
      def ansible(self, playbook, extra_args=None):
          """ Prepare localhost using ansible playbook """
          playbook = self._ansible_playbook_path(playbook)
          stdout, stderr = self.run(
              ['sudo', '-E', 'ansible-playbook'] +
              self._ansible_verbosity() +
              self._ansible_extra_args(extra_args) +
              ['-c', 'local', '-i', 'localhost,', playbook],
              env=self._prepare_environment())
          self._ansible_summary(stdout)
  
      def execute(self, command, **kwargs):
          """ Execute command on localhost """
          # Prepare the environment (plan/cli variables override)
          environment = dict()
          environment.update(kwargs.pop('env', dict()))
          environment.update(self.parent.plan.environment)
          # Run the command under the prepared environment
          return self.run(command, env=environment, shell=True, **kwargs)
  
      def stop(self):
          """ Stop the guest """
  
          self.debug(f"Doing nothing to stop guest '{self.guest}'.")
  
      def reboot(self, hard=False, command=None, timeout=None):
          """ Reboot the guest, return True if successful """
  
          self.debug(f"Doing nothing to reboot guest '{self.guest}'.")
  
          return False
  
      def push(self, source=None, destination=None, options=None):
          """ Nothing to be done to push workdir """
  
      def pull(
              self,
              source=None,
              destination=None,
              options=None,
              extend_options=None):
          """ Nothing to be done to pull workdir """
  
files: tmt/steps/provision/local.py
files: import dataclasses
  import os
  from shlex import quote
  from typing import Optional
  
  import click
  
  import tmt
  import tmt.steps
  import tmt.steps.provision
  
  # Timeout in seconds of waiting for a connection
  CONNECTION_TIMEOUT = 60
  
  # Defaults
  DEFAULT_IMAGE = "fedora"
  DEFAULT_USER = "root"
  
  # TODO: get rid of `ignore` once superclass is no longer `Any`
  
  
  @dataclasses.dataclass
  class PodmanGuestData(tmt.steps.provision.GuestData):  # type: ignore[misc]
      image: str = DEFAULT_IMAGE
      user: str = DEFAULT_USER
      force_pull: bool = False
  
      container: Optional[str] = None
  
  
  @tmt.steps.provides_method('container')
  class ProvisionPodman(tmt.steps.provision.ProvisionPlugin):
      """
      Create a new container using podman
  
      Example config:
  
          provision:
              how: container
              image: fedora:latest
  
      In order to always pull the fresh container image use 'pull: true'.
  
      In order to run the container with different user as the default 'root',
      use 'user: USER'.
      """
  
      # Guest instance
      _guest = None
  
      # Supported keys
      _keys = ["image", "container", "pull", "user"]
  
      @classmethod
      def options(cls, how=None):
          """ Prepare command line options for connect """
          return [
              click.option(
                  '-i', '--image', metavar='IMAGE',
                  help='Select image to use. Short name or complete url.'),
              click.option(
                  '-c', '--container', metavar='NAME',
                  help='Name or id of an existing container to be used.'),
              click.option(
                  '-p', '--pull', is_flag=True,
                  help='Force pulling a fresh container image.'),
              click.option(
                  '-u', '--user', metavar='USER',
                  help='User to use for all container operations.')
              ] + super().options(how)
  
      def default(self, option, default=None):
          """ Return default data for given option """
          if option == 'pull':
              return PodmanGuestData().force_pull
  
          return getattr(PodmanGuestData(), option.replace('-', '_'), default)
  
      def wake(self, data=None):
          """ Wake up the plugin, process data, apply options """
          super().wake(data=data)
          # Wake up podman instance
          if data:
              guest = GuestContainer(data, name=self.name, parent=self.step)
              guest.wake()
              self._guest = guest
  
      def go(self):
          """ Provision the container """
          super().go()
  
          # Show which image we are using
          pull = ' (force pull)' if self.get('pull') else ''
          self.info('image', f"{self.get('image')}{pull}", 'green')
  
          # Prepare data for the guest instance
          data_from_options = {
              key: self.get(key)
              for key in PodmanGuestData.keys()
              if key != 'force_pull'
              }
  
          data_from_options['force_pull'] = self.get('pull')
  
          data = PodmanGuestData(**data_from_options)
  
          # Create a new GuestTestcloud instance and start it
          self._guest = GuestContainer(data, name=self.name, parent=self.step)
          self._guest.start()
  
      def guest(self):
          """ Return the provisioned guest """
          return self._guest
  
      def requires(self):
          """ List of required packages needed for workdir sync """
          return GuestContainer.requires()
  
  
  class GuestContainer(tmt.Guest):
      """ Container Instance """
  
      _data_class = PodmanGuestData
  
      image: Optional[str]
      container: Optional[str]
      user: str
      force_pull: bool
  
      def wake(self):
          """ Wake up the guest """
          self.debug(
              f"Waking up container '{self.container}'.", level=2, shift=0)
  
      def start(self):
          """ Start provisioned guest """
          if self.opt('dry'):
              return
          # Check if the image is available
          image_id = self.podman(
              ['images', '-q', self.image],
              message=f"Check for container image '{self.image}'.")[0].strip()
  
          # Pull image if not available or pull forced
          if not image_id or self.force_pull:
              self.podman(
                  ['pull', '-q', self.image],
                  message=f"Pull image '{self.image}'.")
  
          # Mount the whole plan directory in the container
          workdir = self.parent.plan.workdir
  
          self.container = self.guest = self._tmt_name()
          self.verbose('name', self.container, 'green')
  
          # FIXME: Workaround for BZ#1900021 (f34 container on centos-8)
          workaround = ['--security-opt', 'seccomp=unconfined']
  
          # Run the container
          self.debug(f"Start container '{self.image}'.")
          self.podman(
              ['run'] + workaround +
              ['--name', self.container, '-v', f'{workdir}:{workdir}:z',
               '-itd', '--user', self.user, self.image])
  
      def reboot(self, hard=False, command=None, timeout=None):
          """ Restart the container, return True if successful  """
          if command:
              raise tmt.utils.ProvisionError(
                  "Custom reboot command not supported in podman provision.")
          if not hard:
              raise tmt.utils.ProvisionError(
                  "Containers do not support soft reboot, they can only be "
                  "stopped and started again (hard reboot).")
          self.podman(['container', 'restart', self.container])
          return self.reconnect(timeout=timeout or CONNECTION_TIMEOUT)
  
      def ansible(self, playbook, extra_args=None):
          """ Prepare container using ansible playbook """
          playbook = self._ansible_playbook_path(playbook)
          # As non-root we must run with podman unshare
          podman_unshare = ['podman', 'unshare'] if os.geteuid() != 0 else []
          stdout, stderr = self.run(
              podman_unshare + ['ansible-playbook'] +
              self._ansible_verbosity() +
              self._ansible_extra_args(extra_args) +
              ['-c', 'podman', '-i', f'{self.container},', playbook],
              cwd=self.parent.plan.worktree,
              env=self._prepare_environment())
          self._ansible_summary(stdout)
  
      def podman(self, command, **kwargs):
          """ Run given command via podman """
          return self.run(['podman'] + command, **kwargs)
  
      def execute(self, command, **kwargs):
          """ Execute given commands in podman via shell """
          if not self.container and not self.opt('dry'):
              raise tmt.utils.ProvisionError(
                  'Could not execute without provisioned container.')
  
          # Change to given directory on guest if cwd provided
          directory = kwargs.get('cwd', '')
          if directory:
              directory = f"cd {quote(directory)}; "
  
          # Prepare the environment variables export
          environment = self._export_environment(
              self._prepare_environment(kwargs.get('env', dict())))
  
          # Run in interactive mode if requested
          interactive = ['-it'] if kwargs.get('interactive') else []
  
          # Note that we MUST run commands via bash, so variables
          # work as expected
          if isinstance(command, list):
              command = ' '.join(command)
          command = directory + environment + command
          return self.podman(
              ['exec'] + interactive +
              [self.container or 'dry', 'bash', '-c', command], **kwargs)
  
      def push(self, source=None, destination=None, options=None):
          """ Make sure that the workdir has a correct selinux context """
          self.debug("Update selinux context of the run workdir.", level=3)
          self.run(
              ["chcon", "--recursive", "--type=container_file_t",
               self.parent.plan.workdir], shell=False)
          # In case explicit destination is given, use `podman cp` to copy data
          # to the container
          if destination:
              self.podman(["cp", source, f"{self.container}:{destination}"])
  
      def pull(
              self,
              source=None,
              destination=None,
              options=None,
              extend_options=None):
          """ Nothing to be done to pull workdir """
  
      def stop(self):
          """ Stop provisioned guest """
          if self.container:
              self.podman(['container', 'stop', self.container])
              self.info('container', 'stopped', 'green')
  
      def remove(self):
          """ Remove the container """
          if self.container:
              self.podman(['container', 'rm', '-f', self.container])
              self.info('container', 'removed', 'green')
  
files: tmt/steps/provision/podman.py
files: import dataclasses
  import os
  from shlex import quote
  from typing import Optional
  
  import click
  
  import tmt
  import tmt.steps
  import tmt.steps.provision
  
  # Timeout in seconds of waiting for a connection
  CONNECTION_TIMEOUT = 60
  
  # Defaults
  DEFAULT_IMAGE = "fedora"
  DEFAULT_USER = "root"
  
  # TODO: get rid of `ignore` once superclass is no longer `Any`
  
  
  @dataclasses.dataclass
  class PodmanGuestData(tmt.steps.provision.GuestData):  # type: ignore[misc]
      image: str = DEFAULT_IMAGE
      user: str = DEFAULT_USER
      force_pull: bool = False
  
      container: Optional[str] = None
  
  
  @tmt.steps.provides_method('container')
  class ProvisionPodman(tmt.steps.provision.ProvisionPlugin):
      """
      Create a new container using podman
  
      Example config:
  
          provision:
              how: container
              image: fedora:latest
  
      In order to always pull the fresh container image use 'pull: true'.
  
      In order to run the container with different user as the default 'root',
      use 'user: USER'.
      """
  
      # Guest instance
      _guest = None
  
      # Supported keys
      _keys = ["image", "container", "pull", "user"]
  
      @classmethod
      def options(cls, how=None):
          """ Prepare command line options for connect """
          return [
              click.option(
                  '-i', '--image', metavar='IMAGE',
                  help='Select image to use. Short name or complete url.'),
              click.option(
                  '-c', '--container', metavar='NAME',
                  help='Name or id of an existing container to be used.'),
              click.option(
                  '-p', '--pull', is_flag=True,
                  help='Force pulling a fresh container image.'),
              click.option(
                  '-u', '--user', metavar='USER',
                  help='User to use for all container operations.')
              ] + super().options(how)
  
      def default(self, option, default=None):
          """ Return default data for given option """
          if option == 'pull':
              return PodmanGuestData().force_pull
  
          return getattr(PodmanGuestData(), option.replace('-', '_'), default)
  
      def wake(self, keys=None, data=None):
          """ Wake up the plugin, process data, apply options """
          super().wake(keys=keys, data=data)
          # Wake up podman instance
          if data:
              guest = GuestContainer(data, name=self.name, parent=self.step)
              guest.wake()
              self._guest = guest
  
      def go(self):
          """ Provision the container """
          super().go()
  
          # Show which image we are using
          pull = ' (force pull)' if self.get('pull') else ''
          self.info('image', f"{self.get('image')}{pull}", 'green')
  
          # Prepare data for the guest instance
          data_from_options = {
              key: self.get(key)
              for key in PodmanGuestData.keys()
              if key != 'force_pull'
              }
  
          data_from_options['force_pull'] = self.get('pull')
  
          data = PodmanGuestData(**data_from_options)
  
          # Create a new GuestTestcloud instance and start it
          self._guest = GuestContainer(data, name=self.name, parent=self.step)
          self._guest.start()
  
      def guest(self):
          """ Return the provisioned guest """
          return self._guest
  
      def requires(self):
          """ List of required packages needed for workdir sync """
          return GuestContainer.requires()
  
  
  class GuestContainer(tmt.Guest):
      """ Container Instance """
  
      _data_class = PodmanGuestData
  
      image: Optional[str]
      container: Optional[str]
      user: str
      force_pull: bool
  
      def wake(self):
          """ Wake up the guest """
          self.debug(
              f"Waking up container '{self.container}'.", level=2, shift=0)
  
      def start(self):
          """ Start provisioned guest """
          if self.opt('dry'):
              return
          # Check if the image is available
          image_id = self.podman(
              ['images', '-q', self.image],
              message=f"Check for container image '{self.image}'.")[0].strip()
  
          # Pull image if not available or pull forced
          if not image_id or self.force_pull:
              self.podman(
                  ['pull', '-q', self.image],
                  message=f"Pull image '{self.image}'.")
  
          # Mount the whole plan directory in the container
          workdir = self.parent.plan.workdir
  
          self.container = self.guest = self._tmt_name()
          self.verbose('name', self.container, 'green')
  
          # FIXME: Workaround for BZ#1900021 (f34 container on centos-8)
          workaround = ['--security-opt', 'seccomp=unconfined']
  
          # Run the container
          self.debug(f"Start container '{self.image}'.")
          self.podman(
              ['run'] + workaround +
              ['--name', self.container, '-v', f'{workdir}:{workdir}:z',
               '-itd', '--user', self.user, self.image])
  
      def reboot(self, hard=False, command=None, timeout=None):
          """ Restart the container, return True if successful  """
          if command:
              raise tmt.utils.ProvisionError(
                  "Custom reboot command not supported in podman provision.")
          if not hard:
              raise tmt.utils.ProvisionError(
                  "Containers do not support soft reboot, they can only be "
                  "stopped and started again (hard reboot).")
          self.podman(['container', 'restart', self.container])
          return self.reconnect(timeout=timeout or CONNECTION_TIMEOUT)
  
      def ansible(self, playbook, extra_args=None):
          """ Prepare container using ansible playbook """
          playbook = self._ansible_playbook_path(playbook)
          # As non-root we must run with podman unshare
          podman_unshare = ['podman', 'unshare'] if os.geteuid() != 0 else []
          stdout, stderr = self.run(
              podman_unshare + ['ansible-playbook'] +
              self._ansible_verbosity() +
              self._ansible_extra_args(extra_args) +
              ['-c', 'podman', '-i', f'{self.container},', playbook],
              cwd=self.parent.plan.worktree,
              env=self._prepare_environment())
          self._ansible_summary(stdout)
  
      def podman(self, command, **kwargs):
          """ Run given command via podman """
          return self.run(['podman'] + command, **kwargs)
  
      def execute(self, command, **kwargs):
          """ Execute given commands in podman via shell """
          if not self.container and not self.opt('dry'):
              raise tmt.utils.ProvisionError(
                  'Could not execute without provisioned container.')
  
          # Change to given directory on guest if cwd provided
          directory = kwargs.get('cwd', '')
          if directory:
              directory = f"cd {quote(directory)}; "
  
          # Prepare the environment variables export
          environment = self._export_environment(
              self._prepare_environment(kwargs.get('env', dict())))
  
          # Run in interactive mode if requested
          interactive = ['-it'] if kwargs.get('interactive') else []
  
          # Note that we MUST run commands via bash, so variables
          # work as expected
          if isinstance(command, list):
              command = ' '.join(command)
          command = directory + environment + command
          return self.podman(
              ['exec'] + interactive +
              [self.container or 'dry', 'bash', '-c', command], **kwargs)
  
      def push(self, source=None, destination=None, options=None):
          """ Make sure that the workdir has a correct selinux context """
          self.debug("Update selinux context of the run workdir.", level=3)
          self.run(
              ["chcon", "--recursive", "--type=container_file_t",
               self.parent.plan.workdir], shell=False)
          # In case explicit destination is given, use `podman cp` to copy data
          # to the container
          if destination:
              self.podman(["cp", source, f"{self.container}:{destination}"])
  
      def pull(
              self,
              source=None,
              destination=None,
              options=None,
              extend_options=None):
          """ Nothing to be done to pull workdir """
  
      def stop(self):
          """ Stop provisioned guest """
          if self.container:
              self.podman(['container', 'stop', self.container])
              self.info('container', 'stopped', 'green')
  
      def remove(self):
          """ Remove the container """
          if self.container:
              self.podman(['container', 'rm', '-f', self.container])
              self.info('container', 'removed', 'green')
  
files: tmt/steps/provision/podman.py
files: # coding: utf-8
  
  import dataclasses
  import datetime
  import os
  import platform
  import re
  import time
  from typing import Optional
  
  import click
  import requests
  
  import tmt
  import tmt.steps
  import tmt.steps.provision
  import tmt.utils
  from tmt.utils import WORKDIR_ROOT, ProvisionError, retry_session
  
  
  def import_testcloud():
      """
      Import testcloud module only when needed
  
      Until we have a separate package for each plugin.
      """
      global testcloud
      global libvirt
      try:
          import libvirt
          import testcloud.image
          import testcloud.instance
          import testcloud.util
      except ImportError:
          raise ProvisionError(
              "Install 'testcloud' to provision using this method.")
  
  
  # Testcloud cache to our tmt's workdir root
  TESTCLOUD_DATA = os.path.join(WORKDIR_ROOT, 'testcloud')
  TESTCLOUD_IMAGES = os.path.join(TESTCLOUD_DATA, 'images')
  
  # Userdata for cloud-init
  USER_DATA = """#cloud-config
  chpasswd:
    list: |
      {user_name}:%s
    expire: false
  users:
    - default
    - name: {user_name}
  ssh_authorized_keys:
    - {public_key}
  ssh_pwauth: true
  disable_root: false
  runcmd:
    - sed -i -e '/^.*PermitRootLogin/s/^.*$/PermitRootLogin yes/'
      -e '/^.*UseDNS/s/^.*$/UseDNS no/'
      -e '/^.*GSSAPIAuthentication/s/^.*$/GSSAPIAuthentication no/'
      /etc/ssh/sshd_config
    - systemctl reload sshd
    - [sh, -c, 'if [ ! -f /etc/systemd/network/20-tc-usernet.network ] &&
    systemctl status systemd-networkd | grep -q "enabled;\\svendor\\spreset:\\senabled";
    then mkdir -p /etc/systemd/network/ &&
    echo "[Match]" >> /etc/systemd/network/20-tc-usernet.network &&
    echo "Name=en*" >> /etc/systemd/network/20-tc-usernet.network &&
    echo "[Network]" >> /etc/systemd/network/20-tc-usernet.network &&
    echo "DHCP=yes" >> /etc/systemd/network/20-tc-usernet.network; fi']
    - [sh, -c, 'if systemctl status systemd-networkd |
    grep -q "enabled;\\svendor\\spreset:\\senabled"; then
    systemctl restart systemd-networkd; fi']
    - [sh, -c, 'if cat /etc/os-release |
    grep -q platform:el8; then systemctl restart sshd; fi']
  """
  
  COREOS_DATA = """variant: fcos
  version: 1.4.0
  passwd:
    users:
      - name: {user_name}
        ssh_authorized_keys:
          - {public_key}
  """
  
  # Libvirt domain XML template related variables
  DOMAIN_TEMPLATE_NAME = 'domain-template.jinja'
  DOMAIN_TEMPLATE_FILE = os.path.join(TESTCLOUD_DATA, DOMAIN_TEMPLATE_NAME)
  DOMAIN_TEMPLATE = """<domain type='{{ virt_type }}' xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'>
    <name>{{ domain_name }}</name>
    <uuid>{{ uuid }}</uuid>
    <memory unit='KiB'>{{ memory }}</memory>
    <currentMemory unit='KiB'>{{ memory }}</currentMemory>
    <vcpu placement='static'>2</vcpu>
    <os>
      <type arch='{{ arch }}' machine='{{ model }}'>hvm</type>
      {{ uefi_loader }}
      <boot dev='hd'/>
    </os>
    {{ cpu }}
    {{ extra_specs }}
    <clock offset='utc'>
      <timer name='rtc' tickpolicy='catchup'/>
      <timer name='pit' tickpolicy='delay'/>
      <timer name='hpet' present='no'/>
    </clock>
    <on_poweroff>destroy</on_poweroff>
    <on_reboot>restart</on_reboot>
    <on_crash>restart</on_crash>
    <devices>
      <emulator>{{ emulator_path }}</emulator>
      <disk type='file' device='disk'>
        <driver name='qemu' type='qcow2' cache='unsafe'/>
        <source file="{{ disk }}"/>
        <target dev='vda' bus='virtio'/>
        {{ boot_drive_address }}
      </disk>
      <disk type='file' device='disk'>
        <driver name='qemu' type='raw'/>
        <source file="{{ seed }}"/>
        <target dev='vdb' bus='virtio'/>
        <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
      </disk>
      <interface type='{{ network_type }}'>
        <mac address="{{ mac_address }}"/>
        {{ network_source }}
        {{ ip_setup }}
        <model type='virtio'/>
        <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
      </interface>
      <serial type='pty'>
        <target port='0'/>
      </serial>
      <console type='pty'>
        <target type='serial' port='0'/>
      </console>
      <input type="keyboard" bus="virtio"/>
      <rng model='virtio'>
        <backend model='random'>/dev/urandom</backend>
      </rng>
    </devices>
    {{ qemu_args }}
  </domain>
  """  # noqa: E501
  
  # VM defaults
  DEFAULT_BOOT_TIMEOUT = 60      # seconds
  DEFAULT_CONNECT_TIMEOUT = 60   # seconds
  NON_KVM_ADDITIONAL_WAIT = 10       # seconds
  NON_KVM_TIMEOUT_COEF = 10          # times
  
  # SSH key type, set None for ssh-keygen default one
  SSH_KEYGEN_TYPE = "ecdsa"
  
  DEFAULT_USER = 'root'
  DEFAULT_MEMORY = 2048
  DEFAULT_DISK = 10
  DEFAULT_IMAGE = 'fedora'
  DEFAULT_CONNECTION = 'session'
  DEFAULT_ARCH = platform.machine()
  
  # TODO: get rid of `ignore` once superclass is no longer `Any`
  
  
  @dataclasses.dataclass
  class TestcloudGuestData(
          tmt.steps.provision.GuestSshData):  # type: ignore[misc]
      # Override parent class with our defaults
      user: str = DEFAULT_USER
  
      image: str = DEFAULT_IMAGE
      memory: int = DEFAULT_MEMORY
      disk: int = DEFAULT_DISK
      connection: str = DEFAULT_CONNECTION
      arch: str = DEFAULT_ARCH
  
      image_url: Optional[str] = None
      instance_name: Optional[str] = None
  
  
  @tmt.steps.provides_method('virtual.testcloud')
  class ProvisionTestcloud(tmt.steps.provision.ProvisionPlugin):
      """
      Local virtual machine using testcloud
  
      Minimal config which uses the latest fedora image:
  
          provision:
              how: virtual
  
      Here's a full config example:
  
          provision:
              how: virtual
              image: fedora
              user: root
              memory: 2048
  
      As the image use 'fedora' for the latest released Fedora compose,
      'rawhide' for the latest Rawhide compose, short aliases such as
      'fedora-32', 'f-32' or 'f32' for specific release or a full url to
      the qcow2 image for example from:
  
          https://kojipkgs.fedoraproject.org/compose/
  
      Short names are also provided for 'centos', 'centos-stream',
      'debian' and 'ubuntu' (e.g. 'centos-8' or 'c8').
  
      Supported Fedora CoreOS images are:
  
          fedora-coreos
          fedora-coreos-stable
          fedora-coreos-testing
          fedora-coreos-next
  
      Use the full path for images stored on local disk, for example:
  
          /var/tmp/images/Fedora-Cloud-Base-31-1.9.x86_64.qcow2
  
      In addition to the qcow2 format, vagrant boxes can be used as well,
      testcloud will take care of unpacking the image for you.
      """
  
      # Guest instance
      _guest = None
  
      # Supported keys
      _keys = ["image", "user", "memory", "disk", "connection", "arch"]
  
      @classmethod
      def options(cls, how=None):
          """ Prepare command line options for testcloud """
          return [
              click.option(
                  '-i', '--image', metavar='IMAGE',
                  help='Select image to be used. Provide a short name, '
                       'full path to a local file or a complete url.'),
              click.option(
                  '-m', '--memory', metavar='MEMORY', type=int,
                  help='Set available memory in MB, 2048 MB by default.'),
              click.option(
                  '-D', '--disk', metavar='MEMORY', type=int,
                  help='Specify disk size in GB, 10 GB by default.'),
              click.option(
                  '-u', '--user', metavar='USER',
                  help='Username to use for all guest operations.'),
              click.option(
                  '-c', '--connection',
                  type=click.Choice(['session', 'system']),
                  help="What session type to use, 'session' by default."),
              click.option(
                  '-a', '--arch',
                  type=click.Choice(['x86_64', 'aarch64', 's390x', 'ppc64le']),
                  help="What architecture to virtualize, host arch by default."),
              ] + super().options(how)
  
      def default(self, option, default=None):
          """ Return default data for given option """
          return getattr(TestcloudGuestData(), option, default)
  
      def wake(self, data=None):
          """ Wake up the plugin, process data, apply options """
          super().wake(data=data)
  
          # Wake up testcloud instance
          if data:
              guest = GuestTestcloud(data, name=self.name, parent=self.step)
              guest.wake()
              self._guest = guest
  
      def go(self):
          """ Provision the testcloud instance """
          super().go()
  
          # Give info about provided data
          data = TestcloudGuestData(**{
              key: self.get(key)
              for key in TestcloudGuestData.keys()
              })
  
          # Once plan schema is enforced this won't be necessary
          # click enforces int for cmdline and schema validation
          # will make sure 'int' gets from plan data.
          # Another key is 'port' however that is not exposed to the cli
          for int_key in ["memory", "disk", "port"]:
              value = getattr(data, int_key)
              if value is not None:
                  try:
                      setattr(data, int_key, int(value))
                  except ValueError:
                      raise tmt.utils.SpecificationError(
                          f"Value '{value}' cannot be converted to int for '{int_key}' attribute.")
  
          for key, value in data.to_dict().items():
              if key == 'memory':
                  self.info('memory', f"{value} MB", 'green')
              elif key == 'disk':
                  self.info('disk', f"{value} GB", 'green')
              elif key == 'connection':
                  self.verbose('connection', value, 'green')
              elif value is not None:
                  self.info(key, value, 'green')
  
          # Create a new GuestTestcloud instance and start it
          self._guest = GuestTestcloud(data, name=self.name, parent=self.step)
          self._guest.start()
  
      def guest(self):
          """ Return the provisioned guest """
          return self._guest
  
      @classmethod
      def clean_images(cls, clean, dry):
          """ Remove the testcloud images """
          clean.info('testcloud', shift=1, color='green')
          if not os.path.exists(TESTCLOUD_IMAGES):
              clean.warn(
                  f"Directory '{TESTCLOUD_IMAGES}' does not exist.", shift=2)
              return
          for image in os.listdir(TESTCLOUD_IMAGES):
              image = os.path.join(TESTCLOUD_IMAGES, image)
              if dry:
                  clean.verbose(f"Would remove '{image}'.", shift=2)
              else:
                  clean.verbose(f"Removing '{image}'.", shift=2)
                  os.remove(image)
  
  
  class GuestTestcloud(tmt.GuestSsh):
      """
      Testcloud Instance
  
      The following keys are expected in the 'data' dictionary::
  
          image ...... qcov image name or url
          user ....... user name to log in
          memory ..... memory size for vm
          disk ....... disk size for vm
          connection . either session (default) or system, to be passed to qemu
          arch ....... architecture for the VM, host arch is the default
      """
  
      _data_class = TestcloudGuestData
  
      image: str
      image_url: Optional[str]
      instance_name: Optional[str]
      memory: int
      disk: str
      connection: str
      arch: str
  
      # Not to be saved, recreated from image_url/instance_name/... every
      # time guest is instantiated.
      _image: Optional['testcloud.image.Image'] = None
      _instance: Optional['testcloud.instance.Instance'] = None
  
      def _get_url(self, url, message):
          """ Get url, retry when fails, return response """
  
          def try_get_url() -> requests.Response:
              try:
                  with retry_session() as session:
                      response = session.get(url)
  
                  if response.ok:
                      return response
  
              except requests.RequestException:
                  raise tmt.utils.WaitingIncomplete()
  
          try:
              return tmt.utils.wait(
                  self, try_get_url, datetime.timedelta(
                      seconds=DEFAULT_CONNECT_TIMEOUT), tick=1)
  
          except tmt.utils.WaitingTimedOutError:
              raise ProvisionError(
                  f'Failed to {message} in {DEFAULT_CONNECT_TIMEOUT}s.')
  
      def _guess_image_url(self, name):
          """ Guess image url for given name """
  
          # Try to check if given url is a local file
          if os.path.isabs(name) and os.path.isfile(name):
              return f'file://{name}'
  
          name = name.lower().strip()
          url = None
  
          # Map fedora aliases (e.g. rawhide, fedora, fedora-32, f-32, f32)
          matched_fedora = re.match(r'^f(edora)?-?(\d+)$', name)
          # Map fedora coreos aliases (e.g. stable, next or testing)
          matched_fedora_coreos = re.match(r'^f(edora-coreos)?-?(stable|testing|next)$', name)
          # Map centos aliases (e.g. centos:X, centos, centos-stream:X)
          matched_centos = [re.match(r'^c(entos)?-?(\d+)$', name),
                            re.match(r'^c(entos-stream)?-?(\d+)$', name)]
          matched_ubuntu = re.match(r'^u(buntu)?-?(\w+)$', name)
          matched_debian = re.match(r'^d(ebian)?-?(\w+)$', name)
  
          # Plain name match means we want the latest release
          if name == 'fedora':
              url = testcloud.util.get_fedora_image_url("latest", self.arch)
          elif name == 'fedora-coreos':
              url = testcloud.util.get_fedora_image_url("stable", self.arch)
          elif name == 'centos':
              url = testcloud.util.get_centos_image_url("latest", self.arch)
          elif name == 'centos-stream':
              url = testcloud.util.get_centos_image_url(
                  "latest", stream=True, arch=self.arch)
          elif name == 'ubuntu':
              url = testcloud.util.get_ubuntu_image_url("latest", self.arch)
          elif name == 'debian':
              url = testcloud.util.get_debian_image_url("latest", self.arch)
  
          elif matched_fedora:
              url = testcloud.util.get_fedora_image_url(
                  matched_fedora.group(2), self.arch)
          elif matched_fedora_coreos:
              url = testcloud.util.get_fedora_image_url(
                  matched_fedora_coreos.group(2), self.arch)
          elif matched_centos[0]:
              url = testcloud.util.get_centos_image_url(
                  matched_centos[0].group(2), self.arch)
          elif matched_centos[1]:
              url = testcloud.util.get_centos_image_url(
                  matched_centos[1].group(2), stream=True, arch=self.arch)
          elif matched_ubuntu:
              url = testcloud.util.get_ubuntu_image_url(
                  matched_ubuntu.group(2), self.arch)
          elif matched_debian:
              url = testcloud.util.get_debian_image_url(
                  matched_debian.group(2), self.arch)
          elif 'rawhide' in name:
              url = testcloud.util.get_fedora_image_url("rawhide", self.arch)
  
          if not url:
              raise ProvisionError(f"Could not map '{name}' to compose.")
          return url
  
      @staticmethod
      def _create_template():
          """ Create libvirt domain template """
          # Write always to ovewrite possible outdated version
          with open(DOMAIN_TEMPLATE_FILE, 'w') as template:
              template.write(DOMAIN_TEMPLATE)
  
      def wake(self):
          """ Wake up the guest """
          self.debug(
              f"Waking up testcloud instance '{self.instance_name}'.",
              level=2, shift=0)
          self.prepare_config()
          self._image = testcloud.image.Image(self.image_url)
          self._instance = testcloud.instance.Instance(
              self.instance_name, image=self._image,
              connection=f"qemu:///{self.connection}", desired_arch=self.arch)
  
      def prepare_ssh_key(self, key_type=None):
          """ Prepare ssh key for authentication """
          # Create ssh key paths
          key_name = "id_{}".format(key_type if key_type is not None else 'rsa')
          self.key = [os.path.join(self.workdir, key_name)]
          self.pubkey = os.path.join(self.workdir, f'{key_name}.pub')
  
          # Generate ssh key
          self.debug('Generating an ssh key.')
          command = ["ssh-keygen", "-f", self.key[0], "-N", ""]
          if key_type is not None:
              command.extend(["-t", key_type])
          self.run(command)
          with open(self.pubkey, 'r') as pubkey:
              self.config.USER_DATA = USER_DATA.format(
                  user_name=self.user, public_key=pubkey.read())
          with open(self.pubkey, 'r') as pubkey:
              self.config.COREOS_DATA = COREOS_DATA.format(
                  user_name=self.user, public_key=pubkey.read())
  
      def prepare_config(self):
          """ Prepare common configuration """
          import_testcloud()
  
          # Get configuration
          self.config = testcloud.config.get_config()
  
          # Make sure download progress is disabled unless in debug mode,
          # so it does not spoil our logging
          self.config.DOWNLOAD_PROGRESS = self.opt('debug') > 2
          self.config.DOWNLOAD_PROGRESS_VERBOSE = False
  
          # Configure to tmt's storage directories
          self.config.DATA_DIR = TESTCLOUD_DATA
          self.config.STORE_DIR = TESTCLOUD_IMAGES
  
      def start(self):
          """ Start provisioned guest """
          if self.opt('dry'):
              return
          # Make sure required directories exist
          os.makedirs(TESTCLOUD_DATA, exist_ok=True)
          os.makedirs(TESTCLOUD_IMAGES, exist_ok=True)
  
          # Make sure libvirt domain template exists
          GuestTestcloud._create_template()
  
          # Prepare config
          self.prepare_config()
  
          # Kick off image URL with the given image
          self.image_url = self.image
  
          # If image does not start with http/https/file, consider it a
          # mapping value and try to guess the URL
          if not re.match(r'^(?:https?|file)://.*', self.image_url):
              self.image_url = self._guess_image_url(self.image_url)
              self.debug(f"Guessed image url: '{self.image_url}'", level=3)
  
          # Initialize and prepare testcloud image
          self._image = testcloud.image.Image(self.image_url)
          self.verbose('qcow', self._image.name, 'green')
          if not os.path.exists(self._image.local_path):
              self.info('progress', 'downloading...', 'cyan')
          try:
              self._image.prepare()
          except FileNotFoundError as error:
              raise ProvisionError(
                  f"Image '{self._image.local_path}' not found.", original=error)
          except (testcloud.exceptions.TestcloudPermissionsError,
                  PermissionError) as error:
              raise ProvisionError(
                  f"Failed to prepare the image. Check the '{TESTCLOUD_IMAGES}' "
                  f"directory permissions.", original=error)
  
          # Create instance
          self.instance_name = self._tmt_name()
          self._instance = testcloud.instance.Instance(
              name=self.instance_name, image=self._image,
              connection=f"qemu:///{self.connection}", desired_arch=self.arch)
          self.verbose('name', self.instance_name, 'green')
  
          # Decide if we want to multiply timeouts when emulating an architecture
          time_coeff = NON_KVM_TIMEOUT_COEF if not self._instance.kvm else 1
  
          # Decide which networking setup to use
          # Autodetect works with libguestfs python bindings
          # We fall back to basic heuristics based on file name
          # without that installed (eg. from pypi).
          # https://bugzilla.redhat.com/show_bug.cgi?id=1075594
          try:
              import guestfs  # noqa: F401
          except ImportError:
              match_legacy = re.search(
                  r'(rhel|centos).*-7', self.image_url.lower())
              if match_legacy:
                  self._instance.pci_net = "e1000"
              else:
                  self._instance.pci_net = "virtio-net-pci"
  
          # Prepare ssh key
          # TODO: Maybe... some better way to do this?
          if "coreos" in self.image.lower():
              self._instance.coreos = True
              self._instance.ssh_path = self.key[0]
          self.prepare_ssh_key(SSH_KEYGEN_TYPE)
  
          # Boot the virtual machine
          self.info('progress', 'booting...', 'cyan')
          self._instance.ram = self.memory
          self._instance.disk_size = self.disk
          try:
              self._instance.prepare()
              self._instance.spawn_vm()
              self._instance.start(DEFAULT_BOOT_TIMEOUT * time_coeff)
          except (testcloud.exceptions.TestcloudInstanceError,
                  libvirt.libvirtError) as error:
              raise ProvisionError(
                  f'Failed to boot testcloud instance ({error}).')
          self.guest = self._instance.get_ip()
          self.port = self._instance.get_instance_port()
          self.verbose('ip', self.guest, 'green')
          self.verbose('port', self.port, 'green')
          self._instance.create_ip_file(self.guest)
  
          # Wait a bit until the box is up
          if not self.reconnect(
                  timeout=DEFAULT_CONNECT_TIMEOUT *
                  time_coeff,
                  tick=1):
              raise ProvisionError(
                  f"Failed to connect in {DEFAULT_CONNECT_TIMEOUT * time_coeff}s.")
  
          if not self._instance.kvm:
              self.debug(
                  f"Waiting {NON_KVM_ADDITIONAL_WAIT} seconds "
                  f"for non-kvm instance...")
              time.sleep(NON_KVM_ADDITIONAL_WAIT)
  
      def stop(self):
          """ Stop provisioned guest """
          super().stop()
          # Stop only if the instance successfully booted
          if self._instance and self.guest:
              self.debug(f"Stopping testcloud instance '{self.instance_name}'.")
              try:
                  self._instance.stop()
              except testcloud.exceptions.TestcloudInstanceError as error:
                  raise tmt.utils.ProvisionError(
                      f"Failed to stop testcloud instance: {error}")
              self.info('guest', 'stopped', 'green')
  
      def remove(self):
          """ Remove the guest (disk cleanup) """
          if self._instance:
              self.debug(f"Removing testcloud instance '{self.instance_name}'.")
              try:
                  self._instance.remove(autostop=True)
              except FileNotFoundError as error:
                  raise tmt.utils.ProvisionError(
                      f"Failed to remove testcloud instance: {error}")
              self.info('guest', 'removed', 'green')
  
      def reboot(self, hard=False, command=None, timeout=None):
          """ Reboot the guest, return True if successful """
          # Use custom reboot command if provided
          if command:
              return super().reboot(hard=hard, command=command)
          if not self._instance:
              raise tmt.utils.ProvisionError("No instance initialized.")
          self._instance.reboot(soft=not hard)
          return self.reconnect(timeout=timeout)
  
files: tmt/steps/provision/testcloud.py
files: # coding: utf-8
  
  import dataclasses
  import datetime
  import os
  import platform
  import re
  import time
  from typing import Optional
  
  import click
  import requests
  
  import tmt
  import tmt.steps
  import tmt.steps.provision
  import tmt.utils
  from tmt.utils import WORKDIR_ROOT, ProvisionError, retry_session
  
  
  def import_testcloud():
      """
      Import testcloud module only when needed
  
      Until we have a separate package for each plugin.
      """
      global testcloud
      global libvirt
      try:
          import libvirt
          import testcloud.image
          import testcloud.instance
          import testcloud.util
      except ImportError:
          raise ProvisionError(
              "Install 'testcloud' to provision using this method.")
  
  
  # Testcloud cache to our tmt's workdir root
  TESTCLOUD_DATA = os.path.join(WORKDIR_ROOT, 'testcloud')
  TESTCLOUD_IMAGES = os.path.join(TESTCLOUD_DATA, 'images')
  
  # Userdata for cloud-init
  USER_DATA = """#cloud-config
  chpasswd:
    list: |
      {user_name}:%s
    expire: false
  users:
    - default
    - name: {user_name}
  ssh_authorized_keys:
    - {public_key}
  ssh_pwauth: true
  disable_root: false
  runcmd:
    - sed -i -e '/^.*PermitRootLogin/s/^.*$/PermitRootLogin yes/'
      -e '/^.*UseDNS/s/^.*$/UseDNS no/'
      -e '/^.*GSSAPIAuthentication/s/^.*$/GSSAPIAuthentication no/'
      /etc/ssh/sshd_config
    - systemctl reload sshd
    - [sh, -c, 'if [ ! -f /etc/systemd/network/20-tc-usernet.network ] &&
    systemctl status systemd-networkd | grep -q "enabled;\\svendor\\spreset:\\senabled";
    then mkdir -p /etc/systemd/network/ &&
    echo "[Match]" >> /etc/systemd/network/20-tc-usernet.network &&
    echo "Name=en*" >> /etc/systemd/network/20-tc-usernet.network &&
    echo "[Network]" >> /etc/systemd/network/20-tc-usernet.network &&
    echo "DHCP=yes" >> /etc/systemd/network/20-tc-usernet.network; fi']
    - [sh, -c, 'if systemctl status systemd-networkd |
    grep -q "enabled;\\svendor\\spreset:\\senabled"; then
    systemctl restart systemd-networkd; fi']
    - [sh, -c, 'if cat /etc/os-release |
    grep -q platform:el8; then systemctl restart sshd; fi']
  """
  
  COREOS_DATA = """variant: fcos
  version: 1.4.0
  passwd:
    users:
      - name: {user_name}
        ssh_authorized_keys:
          - {public_key}
  """
  
  # Libvirt domain XML template related variables
  DOMAIN_TEMPLATE_NAME = 'domain-template.jinja'
  DOMAIN_TEMPLATE_FILE = os.path.join(TESTCLOUD_DATA, DOMAIN_TEMPLATE_NAME)
  DOMAIN_TEMPLATE = """<domain type='{{ virt_type }}' xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'>
    <name>{{ domain_name }}</name>
    <uuid>{{ uuid }}</uuid>
    <memory unit='KiB'>{{ memory }}</memory>
    <currentMemory unit='KiB'>{{ memory }}</currentMemory>
    <vcpu placement='static'>2</vcpu>
    <os>
      <type arch='{{ arch }}' machine='{{ model }}'>hvm</type>
      {{ uefi_loader }}
      <boot dev='hd'/>
    </os>
    {{ cpu }}
    {{ extra_specs }}
    <clock offset='utc'>
      <timer name='rtc' tickpolicy='catchup'/>
      <timer name='pit' tickpolicy='delay'/>
      <timer name='hpet' present='no'/>
    </clock>
    <on_poweroff>destroy</on_poweroff>
    <on_reboot>restart</on_reboot>
    <on_crash>restart</on_crash>
    <devices>
      <emulator>{{ emulator_path }}</emulator>
      <disk type='file' device='disk'>
        <driver name='qemu' type='qcow2' cache='unsafe'/>
        <source file="{{ disk }}"/>
        <target dev='vda' bus='virtio'/>
        {{ boot_drive_address }}
      </disk>
      <disk type='file' device='disk'>
        <driver name='qemu' type='raw'/>
        <source file="{{ seed }}"/>
        <target dev='vdb' bus='virtio'/>
        <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
      </disk>
      <interface type='{{ network_type }}'>
        <mac address="{{ mac_address }}"/>
        {{ network_source }}
        {{ ip_setup }}
        <model type='virtio'/>
        <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
      </interface>
      <serial type='pty'>
        <target port='0'/>
      </serial>
      <console type='pty'>
        <target type='serial' port='0'/>
      </console>
      <input type="keyboard" bus="virtio"/>
      <rng model='virtio'>
        <backend model='random'>/dev/urandom</backend>
      </rng>
    </devices>
    {{ qemu_args }}
  </domain>
  """  # noqa: E501
  
  # VM defaults
  DEFAULT_BOOT_TIMEOUT = 60      # seconds
  DEFAULT_CONNECT_TIMEOUT = 60   # seconds
  NON_KVM_ADDITIONAL_WAIT = 10       # seconds
  NON_KVM_TIMEOUT_COEF = 10          # times
  
  # SSH key type, set None for ssh-keygen default one
  SSH_KEYGEN_TYPE = "ecdsa"
  
  DEFAULT_USER = 'root'
  DEFAULT_MEMORY = 2048
  DEFAULT_DISK = 10
  DEFAULT_IMAGE = 'fedora'
  DEFAULT_CONNECTION = 'session'
  DEFAULT_ARCH = platform.machine()
  
  # TODO: get rid of `ignore` once superclass is no longer `Any`
  
  
  @dataclasses.dataclass
  class TestcloudGuestData(
          tmt.steps.provision.GuestSshData):  # type: ignore[misc]
      # Override parent class with our defaults
      user: str = DEFAULT_USER
  
      image: str = DEFAULT_IMAGE
      memory: int = DEFAULT_MEMORY
      disk: int = DEFAULT_DISK
      connection: str = DEFAULT_CONNECTION
      arch: str = DEFAULT_ARCH
  
      image_url: Optional[str] = None
      instance_name: Optional[str] = None
  
  
  @tmt.steps.provides_method('virtual.testcloud')
  class ProvisionTestcloud(tmt.steps.provision.ProvisionPlugin):
      """
      Local virtual machine using testcloud
  
      Minimal config which uses the latest fedora image:
  
          provision:
              how: virtual
  
      Here's a full config example:
  
          provision:
              how: virtual
              image: fedora
              user: root
              memory: 2048
  
      As the image use 'fedora' for the latest released Fedora compose,
      'rawhide' for the latest Rawhide compose, short aliases such as
      'fedora-32', 'f-32' or 'f32' for specific release or a full url to
      the qcow2 image for example from:
  
          https://kojipkgs.fedoraproject.org/compose/
  
      Short names are also provided for 'centos', 'centos-stream',
      'debian' and 'ubuntu' (e.g. 'centos-8' or 'c8').
  
      Supported Fedora CoreOS images are:
  
          fedora-coreos
          fedora-coreos-stable
          fedora-coreos-testing
          fedora-coreos-next
  
      Use the full path for images stored on local disk, for example:
  
          /var/tmp/images/Fedora-Cloud-Base-31-1.9.x86_64.qcow2
  
      In addition to the qcow2 format, vagrant boxes can be used as well,
      testcloud will take care of unpacking the image for you.
      """
  
      # Guest instance
      _guest = None
  
      # Supported keys
      _keys = ["image", "user", "memory", "disk", "connection", "arch"]
  
      @classmethod
      def options(cls, how=None):
          """ Prepare command line options for testcloud """
          return [
              click.option(
                  '-i', '--image', metavar='IMAGE',
                  help='Select image to be used. Provide a short name, '
                       'full path to a local file or a complete url.'),
              click.option(
                  '-m', '--memory', metavar='MEMORY', type=int,
                  help='Set available memory in MB, 2048 MB by default.'),
              click.option(
                  '-D', '--disk', metavar='MEMORY', type=int,
                  help='Specify disk size in GB, 10 GB by default.'),
              click.option(
                  '-u', '--user', metavar='USER',
                  help='Username to use for all guest operations.'),
              click.option(
                  '-c', '--connection',
                  type=click.Choice(['session', 'system']),
                  help="What session type to use, 'session' by default."),
              click.option(
                  '-a', '--arch',
                  type=click.Choice(['x86_64', 'aarch64', 's390x', 'ppc64le']),
                  help="What architecture to virtualize, host arch by default."),
              ] + super().options(how)
  
      def default(self, option, default=None):
          """ Return default data for given option """
          return getattr(TestcloudGuestData(), option, default)
  
      def wake(self, keys=None, data=None):
          """ Wake up the plugin, process data, apply options """
          super().wake(keys=keys, data=data)
  
          # Wake up testcloud instance
          if data:
              guest = GuestTestcloud(data, name=self.name, parent=self.step)
              guest.wake()
              self._guest = guest
  
      def go(self):
          """ Provision the testcloud instance """
          super().go()
  
          # Give info about provided data
          data = TestcloudGuestData(**{
              key: self.get(key)
              for key in TestcloudGuestData.keys()
              })
  
          # Once plan schema is enforced this won't be necessary
          # click enforces int for cmdline and schema validation
          # will make sure 'int' gets from plan data.
          # Another key is 'port' however that is not exposed to the cli
          for int_key in ["memory", "disk", "port"]:
              value = getattr(data, int_key)
              if value is not None:
                  try:
                      setattr(data, int_key, int(value))
                  except ValueError:
                      raise tmt.utils.SpecificationError(
                          f"Value '{value}' cannot be converted to int for '{int_key}' attribute.")
  
          for key, value in data.to_dict().items():
              if key == 'memory':
                  self.info('memory', f"{value} MB", 'green')
              elif key == 'disk':
                  self.info('disk', f"{value} GB", 'green')
              elif key == 'connection':
                  self.verbose('connection', value, 'green')
              elif value is not None:
                  self.info(key, value, 'green')
  
          # Create a new GuestTestcloud instance and start it
          self._guest = GuestTestcloud(data, name=self.name, parent=self.step)
          self._guest.start()
  
      def guest(self):
          """ Return the provisioned guest """
          return self._guest
  
      @classmethod
      def clean_images(cls, clean, dry):
          """ Remove the testcloud images """
          clean.info('testcloud', shift=1, color='green')
          if not os.path.exists(TESTCLOUD_IMAGES):
              clean.warn(
                  f"Directory '{TESTCLOUD_IMAGES}' does not exist.", shift=2)
              return
          for image in os.listdir(TESTCLOUD_IMAGES):
              image = os.path.join(TESTCLOUD_IMAGES, image)
              if dry:
                  clean.verbose(f"Would remove '{image}'.", shift=2)
              else:
                  clean.verbose(f"Removing '{image}'.", shift=2)
                  os.remove(image)
  
  
  class GuestTestcloud(tmt.GuestSsh):
      """
      Testcloud Instance
  
      The following keys are expected in the 'data' dictionary::
  
          image ...... qcov image name or url
          user ....... user name to log in
          memory ..... memory size for vm
          disk ....... disk size for vm
          connection . either session (default) or system, to be passed to qemu
          arch ....... architecture for the VM, host arch is the default
      """
  
      _data_class = TestcloudGuestData
  
      image: str
      image_url: Optional[str]
      instance_name: Optional[str]
      memory: int
      disk: str
      connection: str
      arch: str
  
      # Not to be saved, recreated from image_url/instance_name/... every
      # time guest is instantiated.
      _image: Optional['testcloud.image.Image'] = None
      _instance: Optional['testcloud.instance.Instance'] = None
  
      def _get_url(self, url, message):
          """ Get url, retry when fails, return response """
  
          def try_get_url() -> requests.Response:
              try:
                  with retry_session() as session:
                      response = session.get(url)
  
                  if response.ok:
                      return response
  
              except requests.RequestException:
                  raise tmt.utils.WaitingIncomplete()
  
          try:
              return tmt.utils.wait(
                  self, try_get_url, datetime.timedelta(
                      seconds=DEFAULT_CONNECT_TIMEOUT), tick=1)
  
          except tmt.utils.WaitingTimedOutError:
              raise ProvisionError(
                  f'Failed to {message} in {DEFAULT_CONNECT_TIMEOUT}s.')
  
      def _guess_image_url(self, name):
          """ Guess image url for given name """
  
          # Try to check if given url is a local file
          if os.path.isabs(name) and os.path.isfile(name):
              return f'file://{name}'
  
          name = name.lower().strip()
          url = None
  
          # Map fedora aliases (e.g. rawhide, fedora, fedora-32, f-32, f32)
          matched_fedora = re.match(r'^f(edora)?-?(\d+)$', name)
          # Map fedora coreos aliases (e.g. stable, next or testing)
          matched_fedora_coreos = re.match(r'^f(edora-coreos)?-?(stable|testing|next)$', name)
          # Map centos aliases (e.g. centos:X, centos, centos-stream:X)
          matched_centos = [re.match(r'^c(entos)?-?(\d+)$', name),
                            re.match(r'^c(entos-stream)?-?(\d+)$', name)]
          matched_ubuntu = re.match(r'^u(buntu)?-?(\w+)$', name)
          matched_debian = re.match(r'^d(ebian)?-?(\w+)$', name)
  
          # Plain name match means we want the latest release
          if name == 'fedora':
              url = testcloud.util.get_fedora_image_url("latest", self.arch)
          elif name == 'fedora-coreos':
              url = testcloud.util.get_fedora_image_url("stable", self.arch)
          elif name == 'centos':
              url = testcloud.util.get_centos_image_url("latest", self.arch)
          elif name == 'centos-stream':
              url = testcloud.util.get_centos_image_url(
                  "latest", stream=True, arch=self.arch)
          elif name == 'ubuntu':
              url = testcloud.util.get_ubuntu_image_url("latest", self.arch)
          elif name == 'debian':
              url = testcloud.util.get_debian_image_url("latest", self.arch)
  
          elif matched_fedora:
              url = testcloud.util.get_fedora_image_url(
                  matched_fedora.group(2), self.arch)
          elif matched_fedora_coreos:
              url = testcloud.util.get_fedora_image_url(
                  matched_fedora_coreos.group(2), self.arch)
          elif matched_centos[0]:
              url = testcloud.util.get_centos_image_url(
                  matched_centos[0].group(2), self.arch)
          elif matched_centos[1]:
              url = testcloud.util.get_centos_image_url(
                  matched_centos[1].group(2), stream=True, arch=self.arch)
          elif matched_ubuntu:
              url = testcloud.util.get_ubuntu_image_url(
                  matched_ubuntu.group(2), self.arch)
          elif matched_debian:
              url = testcloud.util.get_debian_image_url(
                  matched_debian.group(2), self.arch)
          elif 'rawhide' in name:
              url = testcloud.util.get_fedora_image_url("rawhide", self.arch)
  
          if not url:
              raise ProvisionError(f"Could not map '{name}' to compose.")
          return url
  
      @staticmethod
      def _create_template():
          """ Create libvirt domain template """
          # Write always to ovewrite possible outdated version
          with open(DOMAIN_TEMPLATE_FILE, 'w') as template:
              template.write(DOMAIN_TEMPLATE)
  
      def wake(self):
          """ Wake up the guest """
          self.debug(
              f"Waking up testcloud instance '{self.instance_name}'.",
              level=2, shift=0)
          self.prepare_config()
          self._image = testcloud.image.Image(self.image_url)
          self._instance = testcloud.instance.Instance(
              self.instance_name, image=self._image,
              connection=f"qemu:///{self.connection}", desired_arch=self.arch)
  
      def prepare_ssh_key(self, key_type=None):
          """ Prepare ssh key for authentication """
          # Create ssh key paths
          key_name = "id_{}".format(key_type if key_type is not None else 'rsa')
          self.key = [os.path.join(self.workdir, key_name)]
          self.pubkey = os.path.join(self.workdir, f'{key_name}.pub')
  
          # Generate ssh key
          self.debug('Generating an ssh key.')
          command = ["ssh-keygen", "-f", self.key[0], "-N", ""]
          if key_type is not None:
              command.extend(["-t", key_type])
          self.run(command)
          with open(self.pubkey, 'r') as pubkey:
              self.config.USER_DATA = USER_DATA.format(
                  user_name=self.user, public_key=pubkey.read())
          with open(self.pubkey, 'r') as pubkey:
              self.config.COREOS_DATA = COREOS_DATA.format(
                  user_name=self.user, public_key=pubkey.read())
  
      def prepare_config(self):
          """ Prepare common configuration """
          import_testcloud()
  
          # Get configuration
          self.config = testcloud.config.get_config()
  
          # Make sure download progress is disabled unless in debug mode,
          # so it does not spoil our logging
          self.config.DOWNLOAD_PROGRESS = self.opt('debug') > 2
          self.config.DOWNLOAD_PROGRESS_VERBOSE = False
  
          # Configure to tmt's storage directories
          self.config.DATA_DIR = TESTCLOUD_DATA
          self.config.STORE_DIR = TESTCLOUD_IMAGES
  
      def start(self):
          """ Start provisioned guest """
          if self.opt('dry'):
              return
          # Make sure required directories exist
          os.makedirs(TESTCLOUD_DATA, exist_ok=True)
          os.makedirs(TESTCLOUD_IMAGES, exist_ok=True)
  
          # Make sure libvirt domain template exists
          GuestTestcloud._create_template()
  
          # Prepare config
          self.prepare_config()
  
          # Kick off image URL with the given image
          self.image_url = self.image
  
          # If image does not start with http/https/file, consider it a
          # mapping value and try to guess the URL
          if not re.match(r'^(?:https?|file)://.*', self.image_url):
              self.image_url = self._guess_image_url(self.image_url)
              self.debug(f"Guessed image url: '{self.image_url}'", level=3)
  
          # Initialize and prepare testcloud image
          self._image = testcloud.image.Image(self.image_url)
          self.verbose('qcow', self._image.name, 'green')
          if not os.path.exists(self._image.local_path):
              self.info('progress', 'downloading...', 'cyan')
          try:
              self._image.prepare()
          except FileNotFoundError as error:
              raise ProvisionError(
                  f"Image '{self._image.local_path}' not found.", original=error)
          except (testcloud.exceptions.TestcloudPermissionsError,
                  PermissionError) as error:
              raise ProvisionError(
                  f"Failed to prepare the image. Check the '{TESTCLOUD_IMAGES}' "
                  f"directory permissions.", original=error)
  
          # Create instance
          self.instance_name = self._tmt_name()
          self._instance = testcloud.instance.Instance(
              name=self.instance_name, image=self._image,
              connection=f"qemu:///{self.connection}", desired_arch=self.arch)
          self.verbose('name', self.instance_name, 'green')
  
          # Decide if we want to multiply timeouts when emulating an architecture
          time_coeff = NON_KVM_TIMEOUT_COEF if not self._instance.kvm else 1
  
          # Decide which networking setup to use
          # Autodetect works with libguestfs python bindings
          # We fall back to basic heuristics based on file name
          # without that installed (eg. from pypi).
          # https://bugzilla.redhat.com/show_bug.cgi?id=1075594
          try:
              import guestfs  # noqa: F401
          except ImportError:
              match_legacy = re.search(
                  r'(rhel|centos).*-7', self.image_url.lower())
              if match_legacy:
                  self._instance.pci_net = "e1000"
              else:
                  self._instance.pci_net = "virtio-net-pci"
  
          # Prepare ssh key
          # TODO: Maybe... some better way to do this?
          if "coreos" in self.image.lower():
              self._instance.coreos = True
              self._instance.ssh_path = self.key[0]
          self.prepare_ssh_key(SSH_KEYGEN_TYPE)
  
          # Boot the virtual machine
          self.info('progress', 'booting...', 'cyan')
          self._instance.ram = self.memory
          self._instance.disk_size = self.disk
          try:
              self._instance.prepare()
              self._instance.spawn_vm()
              self._instance.start(DEFAULT_BOOT_TIMEOUT * time_coeff)
          except (testcloud.exceptions.TestcloudInstanceError,
                  libvirt.libvirtError) as error:
              raise ProvisionError(
                  f'Failed to boot testcloud instance ({error}).')
          self.guest = self._instance.get_ip()
          self.port = self._instance.get_instance_port()
          self.verbose('ip', self.guest, 'green')
          self.verbose('port', self.port, 'green')
          self._instance.create_ip_file(self.guest)
  
          # Wait a bit until the box is up
          if not self.reconnect(
                  timeout=DEFAULT_CONNECT_TIMEOUT *
                  time_coeff,
                  tick=1):
              raise ProvisionError(
                  f"Failed to connect in {DEFAULT_CONNECT_TIMEOUT * time_coeff}s.")
  
          if not self._instance.kvm:
              self.debug(
                  f"Waiting {NON_KVM_ADDITIONAL_WAIT} seconds "
                  f"for non-kvm instance...")
              time.sleep(NON_KVM_ADDITIONAL_WAIT)
  
      def stop(self):
          """ Stop provisioned guest """
          super().stop()
          # Stop only if the instance successfully booted
          if self._instance and self.guest:
              self.debug(f"Stopping testcloud instance '{self.instance_name}'.")
              try:
                  self._instance.stop()
              except testcloud.exceptions.TestcloudInstanceError as error:
                  raise tmt.utils.ProvisionError(
                      f"Failed to stop testcloud instance: {error}")
              self.info('guest', 'stopped', 'green')
  
      def remove(self):
          """ Remove the guest (disk cleanup) """
          if self._instance:
              self.debug(f"Removing testcloud instance '{self.instance_name}'.")
              try:
                  self._instance.remove(autostop=True)
              except FileNotFoundError as error:
                  raise tmt.utils.ProvisionError(
                      f"Failed to remove testcloud instance: {error}")
              self.info('guest', 'removed', 'green')
  
      def reboot(self, hard=False, command=None, timeout=None):
          """ Reboot the guest, return True if successful """
          # Use custom reboot command if provided
          if command:
              return super().reboot(hard=hard, command=command)
          if not self._instance:
              raise tmt.utils.ProvisionError("No instance initialized.")
          self._instance.reboot(soft=not hard)
          return self.reconnect(timeout=timeout)
  
files: tmt/steps/provision/testcloud.py