text: #! /usr/bin/python2
  #
  # Copyright (c) 2017 Intel Corporation
  #
  # SPDX-License-Identifier: Apache-2.0
  #
  
  """TCF's backbone test case finder and runner
  ------------------------------------------
  
  This implements a high level test case finder and runner to find and
  execute test cases.
  
  The execution of each test case consists of the following phases:
  
   - configure
   - build
   - target acquisition
   - deploy
   - one or more evaluation sequences
     (each consisting of setup, start, evaluation per se, teardown)
   - clean [not executed by default]
  
  the configuration, build and deployment phases happen in the local
  host; the evaluation phases can happen in the local host (for static
  tests) or in remote targets (for dynamic tests).
  
  The backbone is designed so multiple drivers (test case drivers) can
  be implemented that find test cases and specify to the backbone how to
  build, run and test for success or failure. This is done by
  subclassing :class:`tcfl.tc.tc_c` and extending or redefining
  :meth:`tc_c.is_testcase`.
  
  Testcases are defined also by subclassing :class:`tcfl.tc.tc_c` and
  implementing the different methods the meta runner will call to
  evaluate. Testcases can manipulate targets (if they need any) by using
  the APIs defined by :class:`tcfl.tc.target_c`.
  
  The runner will collect the list of available targets and determine
  whih testcases have to be run on which targets, and then create an
  instance of each testcase for those groups of targets where it has to
  run. All the instances will then be run in parallel throgh a
  multiprocessing pool.
  
  To testcases report results via the report API, which are handled by
  drivers defined following :class:`tcfl.report.report_c`, which can be
  subclassed and extended to report to different destinations according
  to need. Default drivers report to console and logfiles.
  
  .. _tc_id:
  
  Testcase run identification
  ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  
  A message identification mechanism for which all the messages are
  prefixed with a code:
  
  [RUNID:]HASH{CBDEL}[XX][.N]
  
  - RUNID is a constant string to identify a run of a set of test cases
    (defaults to nothing, can be autogenerated with `-i` or an specific
    string given with `-i RUNID`).
  
  - HASH is a base32 encoded hash of the testcase name, targets where it
    is to be run and their :term:`BSP model`.
  
  - CBDEL is one capital letter representing the phase being ran
    (Configure, Build, Deploy, Evaluation, cLean)
  
  - [XX]: base32 encoded hash of the BSP name, applies only to dynamic
    test case builds per BSP.
  
  This helps locating anything specific to a testcase by grepping in the
  logfile for a given string and adding more components restricts the
  output.
  
  This means that the message IDs are stable across runs, save RUNIDs
  being specified.
  
  We also use the RUNID:TC combination for a ticket when requesting a
  target lock--note this does not conflict with other users, as the
  tickets are namespaced per-user. This allows the server log to be used
  to crossreference what was being ran, to sort out issues.
  
  """
  import ast
  import atexit
  import collections
  import contextlib
  import copy
  import datetime
  import errno
  import imp
  import inspect
  import logging
  import numbers
  import os
  import platform
  import pprint
  import random
  import re
  import shutil
  import string
  import subprocess
  import sys
  import tempfile
  import threading
  import time
  import traceback
  import types
  
  # We multithread to run testcases in parallel
  #
  # When massively running threads in production environments, we end up
  # with hundreds/thousands of threads based on the setup which are just
  # launching a build and waiting. However, sometimes something dies
  # inside Python and leaves the thing hanging with the GIL taken and
  # everythin deadlocks.
  #
  # For those situations, using the PathOS/pools library works better,
  # as it can multithread as processes (because of better pickling
  # abilities) and doesn't die.
  #
  # So there, the PATHOS.multiprocess library if available and in said
  # case, use process pools for the testcases.
  
  _multiprocessing_method_pool_c = None
  _multiprocessing_tc_pool_c = None
  _multiprocessing_sync_manager = None
  _multiprocessing = None
  
  def import_mp_pathos():
      import pathos.pools
      import pathos.multiprocessing
      import multiprocess.managers
      global _multiprocessing_method_pool_c
      global _multiprocessing_tc_pool_c
      global _multiprocessing_sync_manager
      global _multiprocessing
      _multiprocessing_method_pool_c = pathos.pools._ThreadPool
      _multiprocessing_tc_pool_c = pathos.pools._ProcessPool
      _multiprocessing_sync_manager = multiprocess.managers.SyncManager
      _multiprocessing = pathos.multiprocessing
  
  def import_mp_std():
      import multiprocessing.pool
      import multiprocessing
      import multiprocessing.managers
      global _multiprocessing_method_pool_c
      global _multiprocessing_tc_pool_c
      global _multiprocessing_sync_manager
      global _multiprocessing
      _multiprocessing_method_pool_c = multiprocessing.pool.ThreadPool
      _multiprocessing_tc_pool_c = multiprocessing.pool.ThreadPool
      _multiprocessing_sync_manager = multiprocessing.managers.SyncManager
      _multiprocessing = multiprocessing
  
  mp = os.environ.get('TCF_USE_MP', None)
  if mp == None:
      try:
          import_mp_pathos()
      except ImportError as e:
          import_mp_std()
  elif mp.lower() == 'std':
      import_mp_std()
  elif mp.lower() == 'pathos':
      import_mp_pathos()
  else:
      raise RuntimeError('Invalid value to TCF_USE_MP (%s)' % mp)
  
  
  import requests.exceptions
  
  import app
  import commonl
  import commonl.expr_parser
  import expecter
  import report
  import tcfl
  import ttb_client
  
  from . import msgid_c
  
  # For debugging, levels are D2: 9, D3: 8, D4:7 ...
  logging.addLevelName(50, "C")
  logging.addLevelName(40, "E")
  logging.addLevelName(30, "W")
  logging.addLevelName(20, "I")
  logging.addLevelName(10, "D")
  logging.addLevelName(9, "D2")
  logging.addLevelName(8, "D3")
  logging.addLevelName(7, "D4")
  logging.addLevelName(6, "D5")
  
  logger = logging.getLogger("tcfl.tc")
  
  class exception(Exception):
      """\
      General base exception for reporting results of any phase of test
      cases
  
      :param str msg: a message to report
  
      :param dict attachments: a dictionary of items to report, with a
        few special fields:
  
        - `target`: this is a :py:class:`tcfl.tc.target_c` which shall be used
          for reporting
  
        - `dlevel`: this is an integer that indicates the relative
          level of verbosity (FIXME: link to detailed explanation)
  
        - `alevel`: this is an integer that indicates the relative
          level of verbosity for attachments (FIXME: link to detailed
          explanation)
  
        - any other fields will be passed verbatim and reported
  
      Have to use a dictionary (vs using kwargs) so the name of the
      keys can contain spaces, for better reporting.
      """
      def __init__(self, description, attachments = None):
          if attachments:
              assert isinstance(attachments, dict)
          Exception.__init__(self, description, attachments)
  
      def attachments_get(self):
          if len(self.args) < 1 or not self.args[1]:
              return {}
          assert isinstance(self.args[1], dict), \
              "expected dict, got type %s" % type(self.args[1]).__name__
          return self.args[1]
  
  class pass_e(exception):
      """
      The test case passed
      """
      pass
  
  class blocked_e(exception):
      """
      The test case could not be completed because something failed and
      disallowed testing if it woud pass or fail
      """
      pass
  
  class error_e(exception):
      """
      Executing the test case found an error
      """
      pass
  
  class failed_e(exception):
      """
      The test case failed
      """
      pass
  
  class skip_e(exception):
      """
      A decission was made to skip executing the test case
      """
      pass
  
  #: list of valid results and translations in present and past tense
  valid_results = dict(
      PASS = ( 'pass', 'passed' ),
      ERRR = ( 'error', 'errored' ),
      FAIL = ( 'fail', 'failed' ),
      BLCK = ( 'block', 'blocked' ),
      SKIP = ( 'skip', 'skipped' ),
  )
  
  #
  # I dislike globals, but sometimes they are needed
  #
  
  log_dir = None
  report_console = None
  report_failures = None
  # FIXME: no like, this is a hack and shall use tc_c.ticket
  ticket = None
  
  def _globals_init():
      # This function is used by test routines to cleanup the global state
  
      global log_dir
      global report_console
      global report_failures
      # Should be possible to move most of these to tc_c
      log_dir = None
      tc_c._dry_run = False
      tc_c.runid = None
      if report_console:
          report.report_c.driver_rm(report_console)
          report_console = None
      if report_failures:
          report.report_c.driver_rm(report_failures)
          report_failures = None
  
  class target_extension_c(object):
      r"""Implement API extensions to targets
  
      An API extension allows you to extend the API for the
      :py:class:`tcfl.tc.target_c` class so that more functionality can
      be added to the `target` objects passed to testcase methods (like
      build\*(), eval\*(), etc) and used as::
  
      >>> class extension_a(target_extension_c):
      >>>   def function(self):
      >>>     self.target.report_info("Hello world from extension_a")
      >>>   variable = 34
      >>> ...
      >>> target_c.extension_register(etension_a)
      >>> ...
  
      Now, in an (e.g) evaluation function in a testcase:
  
      >>> @tcfl.tc.target()
      >>> @tcfl.tc.target()
      >>> class _test(tcfl.tc.tc_c):
      >>>
      >>>   def eval_something(self, target, target1):
      >>>      target1.extension_a.function()
      >>>      if target1.extension_a.variable > 3:
      >>>        do_something()
      >>> ...
  
      Extensions have to be registered with
      :py:func:`tcfl.tc.target_c.extension_register`. Unregister with
      :py:func:`tcfl.tc.target_c.extension_unregister`
  
      The extension can be anything, but are commonly used to provide
      the code to access APIs that are not distributed as part of the
      core TCF distribution (like for example, an API to access an
      special sensor).
  
      A package might add support on the server side for an interface to
      access the target and on the client side to access said
      interfaces.
  
      The `__init__()` method will typically first check if the target
      meets the criteria needed for the extension to work or be used. If
      not it can raise :py:class:`target_extension_c.unneeded` to avoid
      the extension to be created.
  
      Then it proceeds to create an instance that will be attached to
      the target for later use.
  
      """
  
      class unneeded(Exception):
          """
          Raise this from __init__() if this extension is not needed for
          this target.
          """
          pass
  
      def __init__(self, _target):
          assert isinstance(_target, target_c)
          #: Target this extension applies to
          self.target = _target
  
      @classmethod
      def __check_name(cls):
          # verify this is a valid extension name
          pass
  
  
  class target_c(object):
      """A remote target that can be manipulated
  
      :param dict rt: remote target descriptor (dictionary) as returned
          by :py:func:`tcfl.ttb_client.rest_target_find_all` and others.
      :param tc_c tescase: testcase descriptor to which this target
          instance will be uniquely assigned.
  
      A target always operate in a given :term:`BSP model`, as decided
      by the testcase runner. If a remote target A has two BSP models (1
      and 2) and a testcase T shall be run on both, it will create two
      testcase instances, T1 and T2. Each will be assigned an instance
      of :class:`target_c`, A1 and A2 respectively, representing the
      same target A, but each set to a different :term:`BSP model`.
  
      Note these object expose the basic target API, but then different
      extensions provide APIs to access other interfaces, depending on
      if the target exposes it or not; these is the current list of
      implemented interfaces:
  
      - :py:class:`console <tcfl.target_ext_console.console>`
      - :py:class:`debug <tcfl.target_ext_debug.debug>`
      - :py:class:`fastboot <tcfl.target_ext_fastboot.fastboot>`
      - :py:class:`images <tcfl.target_ext_images.images>`
      - :py:class:`ioc_flash_server_app <tcfl.target_ext_ioc_flash_server_app.extension>`
      - :py:class:`power <tcfl.target_ext_power.power>`
      - :py:class:`shell <tcfl.target_ext_shell.shell>`
      - :py:class:`ssh <tcfl.target_ext_ssh.ssh>`
      - :py:class:`tcob <tcfl.target_ext_tcob.tcob>`
      - :py:class:`tunnel <tcfl.target_ext_tunnel.tunnel>`
      - :py:class:`zephyr <tcfl.app_zephyr.zephyr>`
  
      """
  
      #
      # Public API
      #
      def __init__(self, rt, testcase, bsp_model, target_want_name):
          assert isinstance(rt, dict)
          assert isinstance(testcase, tc_c)
          #: Name this target is known to by the testcase (as it was
          #: claimed with the :func:`tcfl.tc.target` decorator)
          self.want_name = target_want_name
          #: Remote tags of this target
          self.rt = rt
          # Mind those static TCs, local target has no rtb
          self.rtb = rt.get('rtb', None)
          #: (short) id of this target
          self.id = rt['id']
          #: Full id of this target
          self.fullid = rt['fullid']
          #: Type name of this target
          self.type = rt['type']
          #: ticket used to acquire this target
          self.ticket = None
          #: Testcase that this target is currently executing
          self.testcase = testcase
          #: All of BSPs supported by this target
          self._bsps_all = rt.get('bsps', {}).keys()
          #: :term:`BSP model` this target is currently configured for
          self.bsp_model = None
          #: List of BSPs supported by this target's currently selected
          #: :term:`BSP model`
          self.bsps = []
          #: Make sure the testcase indicates the daemon that this
          #: target is to be marked as active during the testcase
          #: execution of expectation loops.
          self.keep_active = True
          #: Dict of BSPs that have to be stubbed for the board to work
          #: correctly in the current :term:`BSP model` (if the board
          #: has two BSPs but BSP1 needs to have an image of something
          #: so BSP2 can start). The App builder can manipulate this to
          #: remove BSPs that can be ignored. The value is a tuple (app,
          #: srcinfo) that indicates the App builder that will build the
          #: stub and with which source information (path to the
          #: source).
          self.bsps_stub = {}
          #: BSP (inside the :term:`BSP model`) this target is currently
          #: configured for
          self.bsp = None
          self._prefix = ""
          # KW handling for targets is a little bit messed up, as we
          # want to include the keywords root testcase kws, the ones for
          # the testcase instance, the ones for the target (incuding
          # anything the user might want to set), the ones for the
          # target's :term:`BSP model` and the ones for the target's
          # BSP.  So we keep the "originals" for modification with
          # kw_set in self._kw and then the update KWS modifies the
          # self.kws set of keywords merging everthing together.
          #: Target specific keywords
          self._kws = {}
          #: Target specific keywords for each BSP
          #:
          #: This is where BSP specific information can be set
          self._kws_bsp = {}
          #: Origin of keys defined in self._kws
          self._kws_origin = {}
  
          # Combination of runid/ident string
          self._ident = ""
  
          #: Temporary directory where to store files -- this is the same
          #: as the testcase's -- it is needed for the report driver to
          #: find where to put stuff.
          self.tmpdir = testcase.tmpdir
  
          #: Keywords for ``%(KEY)[sd]`` substitution specific to this
          #: target and current active :term:`BSP model` and BSP as set
          #: with :meth:`bsp_set`.
          #:
          #: These are obtained from the remote target descriptor
          #: (`self.rt`) as obtained from the remote *ttbd* server.
          #:
          #: These can be used to generate strings based on information,
          #: as:
          #:
          #:   >>>  print "Something %(id)s" % target.kws
          #:   >>>  target.shcmd_local("cp %(id)s.config final.config")
          #:
          #: To find which fields are available for a target::
          #:
          #:   $ tcf list -vv TARGETNAME
          #:
          #: The testcase will provide also other fields, in
          #: :data:`tcfl.tc.tc_c.kws`, which are rolled in in this variable
          #: too. See how to more available keywords :ref:`here
          #: <finding_testcase_metadata>`
          #:
          #: Note that testcases might be setting more keywords in the
          #: target or the testcase with:
          #:
          #: >>> target.kw_set("keywordname", "somevalue")
          #: >>> self.kw_set("keywordname", "somevalue")
          #:
          #: as well, any of the target's properties set with
          #: :meth:`TARGET.property_set<tcfl.tc.target_c.property_set>`
          #: (or it's command line equivalent ``tcf property-set TARGET
          #: PROPERTY VALUE``) will show up as keywords.
          self.kws = {}
          #: Origin of keys defined in self.kws
          self.kws_origin = {}
          # Generate KWS in case the extensions have to use it
          if bsp_model:
              self._bsp_model_set(bsp_model)
          else:
              self._report_mk_prefix()
          # Fire up the extensions
          for name, extension in self.__extensions.iteritems():
              try:
                  self.report_info("%s: %s: initializing target extension"
                                   % (self.want_name, name),
                                   dlevel = 6)
                  e = extension(self)
                  setattr(self, name, e)
                  e.target = self
                  self.report_info("%s: %s: initialized target extension"
                                   % (self.want_name, name),
                                   dlevel = 5)
              except target_extension_c.unneeded:
                  self.report_info("%s: %s: unneeded target extension"
                                   % (self.want_name, name),
                                   dlevel = 5)
                  continue
              except Exception as e:
                  self.report_blck("%s: %s: exception initializing extension: %s"
                                   % (self.want_name, name, e))
                  raise
          # Re-generate KWS in case the extensions have changed anything
          if bsp_model:
              self._bsp_model_set(bsp_model)
          else:
              self._report_mk_prefix()
          # And finally, set the default BSP to the default (which is
          # the first one declared by the target, if any)
          if self.bsps:
              self.bsp_set()
          #: Shall we acquire this target? By default the testcases get
          #: the targets they request acquired for exclusive use, but in
          #: some cases, it might not be needed (default: True)
          self.acquire = True
  
      @classmethod
      def extension_register(cls, ext_cls, name = None):
          """
          Register an extension to the :py:class:`tcfl.tc.target_c` class.
  
          This is usually called from a config file to register an
          extension provided by a package.
  
          See :py:class:`target_extension_c` for details
  
          :param target_extension_c ext_cls: a class that provides an extension
          :param str name: (optional) name of the extension (defaults to
            the class name)
          """
          assert issubclass(ext_cls, target_extension_c)
          assert name == None or isinstance(name, basestring)
          if name == None:
              name = ext_cls.__name__
          if name in target_c.__dict__.keys():
              raise RuntimeError("%s: can't register extension, there is "
                                 "already a field in class target_c with "
                                 "said name that would be overriden"
                                 % name)
          if name in cls.__extensions:
              raise RuntimeError("%s: can't register extension, there is "
                                 "already one registered with said name"
                                 % name)
          cls.__extensions[name] = ext_cls
  
      @classmethod
      def extension_unregister(cls, ext_cls, name = None):
          """
          Unregister an extension to the :py:class:`tcfl.tc.target_c` class.
  
          This is usually used by unit tests. There usually is no need
          to unregister extensions.
  
          See :py:class:`target_extension_c` for details
  
          :param target_extension_c ext_cls: a class that provides an extension
          :param str name: (optional) name of the extension (defaults to
            the class name)
          """
          assert name == None or isinstance(name, basestring)
          if name == None:
              name = ext_cls.__name__
          del cls.__extensions[name]
  
      #
      # Actions that don't affect the target
      #
      @property
      def bsps_all(self):
          """
          Return a list of all BSPs in the target (note this might be
          more than the ones available in the currentlt selected BSP
          model).
          """
          return self._bsps_all
  
      def _bsp_model_set(self, bsp_model):
          """\
          Set which :term:`BSP model` this target shall operate on.
  
          Note this does not physically alter anything on the hardware,
          but it will affect how the build configuration is done so that
          only the right BSPs are used.
  
          :param str bsp_model: name of the :term:`BSP model` to use, which must
            be one of the ones listed in target.bsp_models.
  
          """
          if 'bsp_models' not in self.rt.keys() \
             or not bsp_model in self.rt['bsp_models']:
              raise blocked_e("target %s: BSP model '%s' not supported"
                              % (self.fullid, bsp_model))
          bsps = self.rt['bsp_models'][bsp_model]
          if bsps == None:
              bsps = [bsp_model]
          self.bsps = bsps
          self.bsps_stub = {
              bsp: (None, None, None) for bsp in set(self.bsps_all) - set(bsps)
          }
          self.bsp_model = bsp_model
          self._kws_update()
          self._report_mk_prefix()
  
      def bsp_set(self, bsp = None):
          """Set the active BSP
  
          If the BSP is omitted, this will select the first BSP in the
          current BSP model. This means that if there is a preference in
          multiple BSPs, they have to be listed as such in the target's
          configuration.
  
          If there are no BSPs, this will raise an exception
  
          :param str bsp: (optional) The name of any BSP supported by
            the board (not necessarily in the BSP-Model list of active
            BSPs. These are always in :attr:`bsps_all`.
            If this argument is False, then the active BSP is reset to none.
  
          """
          # This function assumes there is always an active BSP-Model
          # that has been set, somehow...or that there are no BSPs, so
          # then it makes no sense to call it.
          if bsp == None:
              if not self.bsps:
                  raise blocked_e(
                      "%s: can't set any BSP; target's BSP model "
                      "%s has no BSPs" % (self.id, self.bsp_model))
              bsp = self.bsps[0]
          if bsp == False:
              self.bsp = None
          elif isinstance(bsp, basestring):
              if not bsp in self.bsps_all:
                  raise blocked_e(
                      "%s: can't set BSP '%s'; target's BSP model "
                      "%s has only BSPs %s"
                      % (self.id, bsp, self.bsp_model, ",".join(self.bsps_all)))
              self.bsp = bsp
          else:
              raise AssertionError("Bad arguments to target_c.bsp_set()")
          self._kws_update(bsp)
          self._report_mk_prefix()
  
      def _kws_update_interconnect_addrs(self, interconnects):
          # Some interconnect specific work
          # If we are doing a testcase with interconnects and the target
          # belongs to any of the interconnects, pull out the addresses
          # of the targets in such interconnect to top level, so it can
          # easily do linux.kws['ipv4_addr'] (for example)
          #
          # So we are going to go over all the interconnects this
          # testcase is using and if the target has addresses in any,
          # we'll put those addresses at the top level
          for ic_name in sorted(interconnects):
              for tg_ic_name, tg_ic_dict \
                  in self.rt.get('interconnects', {}).iteritems():
                  if tg_ic_name != ic_name:
                      continue
                  for key, value in tg_ic_dict.iteritems():
                      if not key.endswith("_addr"):
                          continue
                      self._kws[key] = value
  
      def kws_set(self, d, bsp = None):
          """
          Set a bunch of target's keywords and values
  
          :param dict d: A dictionary of keywords and values
          :param str bsp: (optional) BSP for which we want to set the
            keywords; if omitted, we are setting the keywords for the whole
            target
          """
          assert isinstance(d, dict)
          for kw in d.keys():
              assert isinstance(kw, basestring)
          if bsp == None:
              self._kws.update(d)
          else:
              self._kws_bsp.setdefault(bsp, {}).update(d)
          self._kws_update(self.bsp)
          self._report_mk_prefix()
  
      def kw_set(self, kw, val, bsp = None):
          """
          Set a target's keyword and value
  
          :param str kw: A string describing naming the keyword/value pair
          :param val: the value (can be anything)
          :param str bsp: (optional) BSP for which we want to set the
            keyword; if omitted, we are setting the keyword for the whole
            target
          """
          assert isinstance(kw, basestring)
          if bsp == None:
              self._kws[kw] = val
          else:
              self._kws_bsp.setdefault(bsp, {})[kw] = val
          self._kws_update(self.bsp)
          self._report_mk_prefix()
  
      def kw_unset(self, kw, bsp = None):
          """
          Unset a target's string keyword
  
          :param str kw: keyword name
          :param str bsp: (optional) BSP for which we want to unset the
            keyword; if omitted, we are setting the keyword for the whole
            target
          """
          assert isinstance(kw, basestring)
          if bsp == None:
              if kw in self._kws:
                  del self._kws[kw]
          else:
              if kw in self._kws_bsp[bsp]:
                  del self._kws_bsp[bsp][kw]
          self._kws_update(self.bsp)
          self._report_mk_prefix()
  
      def kws_required_verify(self, kws):
          """
          Verify if a target exports required keywords, raise blocked
          exception if any is missing.
          """
          assert isinstance(kws, list)
          missing = []
          for kw in kws:
              if not kw in self.kws.keys():
                  missing.append(kw)
          if missing:
              raise tcfl.tc.blocked_e(
                  'target does not export needed keywords: %s'
                  % ', '.join(missing))
  
      def ic_field_get(self, ic, field, field_description = ""):
          """Obtain the value of a field for a target in an interconnect
  
          A target might be a member of one or more interconnects, as
          described by its tags (*interconnects* section).
  
          :param tcfl.tc.target_c ic: target describing the interconnect
            of which this target is a member (as defined in a
            @ :func:`tcfl.tc.interconnect` decorator to the testcase
            class)
  
          :param str field: name of the filed whose value we want.
  
          >>> def eval_somestep(self, ic, target1, target2):
          >>>    target1.shell.run("ifconfig eth0 %s/%s"
          >>>                      % (target2.addr_get(ic, 'ipv4'),
          >>>                         target2.ic_field_get(ic, 'ipv4_addr_len'))
          """
          assert isinstance(ic, tcfl.tc.target_c)
          assert isinstance(field, basestring)
  
          if not 'interconnects' in self.rt:
              raise RuntimeError('target provides no interconnect information')
          if not ic.id in self.rt['interconnects']:
              raise RuntimeError("target is not in interconnect '%s'" % ic.id)
          if not field in self.rt['interconnects'][ic.id]:
              raise RuntimeError(
                  "target does not provide field '%s' "
                  "%s in interconnect '%s'"
                  % (field, field_description, ic.id))
          return self.rt['interconnects'][ic.id][field]
  
      def addr_get(self, ic, tech, instance = None):
          """Obtain the address for a target in an interconnect
  
          A target might be a member of one or more interconnects, as
          described by its tags (*interconnects* section).
  
          :param tcfl.tc.target_c ic: target describing the interconnect
            of which this target is a member (as defined in a
            @ :func:`tcfl.tc.interconnect` decorator to the testcase
            class)
  
          :param str tech: name of the technology on which address we
            are interested.
  
            As part of said membership, one or more key/value pairs can be
            specified. Assigned addresses are always called
            *TECHNOLOGY_addr*, were *TECHNOLOGY* can be things like
            *ipv4*, *ipv6*, *bt*, *mac*, etc...
  
            If *tech* fits a whole key name, it will be used instead.
  
          :param str instance: (optional) when this target has multiple
            connections to the same interconnect (via multiple physical
            or virtual network interfaces), you can select which
            instance of those it is wanted. 
  
            By default this will return the default instance (eg, the
            one corresponding to the interconnect ``ICNAME``), but if an
            instance is added, it will return the IP address for
            ``ICNAME#INSTANCE`` as declared in the target's
            configuration with functions such as
            :func:`ttbl.test_target.add_to_interconnect`.
  
          When the target, for the current testcase is member of a
          single interconnect, any *TECHNOLOGY_addr* for the
          interconnect key/value will be available in the :attr:`kws`
          member as for example.
  
          >>> target.kws['ipv4_addr']
  
          However, when member of multiple interconnects, which members
          are promoted to top level is undertermined if both
          interconnects provide address information for the same
          technology. Use this function to obtain the
          interconnect-specific information.
  
          >>> def eval_somestep(self, ic, target1, target2):
          >>>    target1.shell.run("scp /etc/passwd %s:/etc/passwd"
          >>>                      % target2.addr_get(ic, 'ipv4'))
  
          """
          if instance:
              assert isinstance(instance, basestring)
              ic = ic + "#" + instance
          return self.ic_field_get(ic, tech + "_addr", "(address)")
  
      def app_get(self, bsp = None, noraise = True):
          """
          Return the App builder that is assigned to a particular BSP in
          the target.
  
          :param str bsp: Name of the BSP we are querying the App
            builder for. Can be omitted if the current :term:`BSP model`
            for the target contains a single BSP, and thus it'll be
            auto-guessed.
  
          :param bool noraise: (optional) if True, do not raise an
            exception if we cannot find an App Builder and just return
            Nones.
          """
          if bsp == None:
              assert len(self.bsps) == 1, "Auto-guessing of BSP can only " \
                  "be done if the current BSP model carries only one BSP"
              bsp = self.bsps[0]
          return self._app_get_for_bsp(None, bsp = bsp, noraise = noraise)[0]
  
  
      def report_pass(self, message, attachments = None,
                      level = None, dlevel = 0, alevel = 2, ulevel = 5):
          if level == None:		# default args are computed upon def'on
              level = msgid_c.depth()
          self._report_argcheck(message, attachments,
                                level, dlevel, alevel, ulevel)
          level += dlevel
          report.report_c.report(level, level + alevel, level + ulevel, self,
                                 "PASS", message, attachments)
  
      def report_fail(self, message, attachments = None,
                      level = None, dlevel = 0, alevel = 2, ulevel = 5):
          if level == None:		# default args are computed upon def'on
              level = msgid_c.depth()
          self._report_argcheck(message, attachments,
                                level, dlevel, alevel, ulevel)
          level += dlevel
          report.report_c.report(level, level + alevel, level + ulevel, self,
                                 "FAIL", message, attachments)
  
      def report_error(self, message, attachments = None,
                       level = None, dlevel = 0, alevel = 2, ulevel = 5):
          if level == None:		# default args are computed upon def'on
              level = msgid_c.depth()
          self._report_argcheck(message, attachments,
                                level, dlevel, alevel, ulevel)
          level += dlevel
          report.report_c.report(level, level + alevel, level + ulevel, self,
                                 "ERRR", message, attachments)
  
      def report_blck(self, message, attachments = None,
                      level = None, dlevel = 0, alevel = 2, ulevel = 5):
          if level == None:		# default args are computed upon def'on
              level = msgid_c.depth()
          self._report_argcheck(message, attachments,
                                level, dlevel, alevel, ulevel)
          level += dlevel
          report.report_c.report(level, level + alevel, level + ulevel, self,
                                 "BLCK", message, attachments)
  
      def report_skip(self,  message, attachments = None,
                      level = None, dlevel = 0, alevel = 2, ulevel = 5):
          if level == None:		# default args are computed upon def'on
              level = msgid_c.depth()
          self._report_argcheck(message, attachments,
                                level, dlevel, alevel, ulevel)
          level += dlevel
          report.report_c.report(level, level + alevel, level + ulevel, self,
                                 "SKIP", message, attachments)
  
      def report_info(self, message, attachments = None,
                      level = None, dlevel = 0, alevel = 2, ulevel = 5):
          if level == None:		# default args are computed upon def'on
              level = msgid_c.depth()
          self._report_argcheck(message, attachments,
                                level, dlevel, alevel, ulevel)
          level += dlevel
          report.report_c.report(level, level + alevel, level + ulevel, self,
                                 "INFO", message, attachments)
  
      def report_data(self, domain, name, value, expand = True):
          """Report measurable data
  
          When running a testcase, if data is collected that has to be
          reported for later analysis, use this function to report
          it. This will be reported by the report driver in a way that
          makes it easy to collect later on.
  
          Measured data is identified by a *domain* and a *name*, plus
          then the actual value.
  
          A way to picture how this data can look once aggregated is as
          a table per domain, on which each invocation is a row and each
          column will be the values for each name.
  
          :param str domain: to which domain this measurement applies
            (eg: "Latency Benchmark %(type)s");
  
          :param str name: name of the value  (eg: "context switch
            (microseconds)"); it is recommended to always add the unit
            the measurement represents.
  
          :param value: value to report for the given domain and name;
             any type can be reported.
  
          :param bool expand: (optional) by default, the *domain* and
            *name* fields will be %(FIELD)s expanded with the keywords
            of the testcase or target. If *False*, it will not be
            expanded.
  
            This enables to, for example, specify a domain of "Latency
            measurements for target %(type)s" which will automatically
            create a different domain for each type of target.
          """
          assert isinstance(domain, basestring)
          assert isinstance(name, basestring)
          if expand:
              domain = domain % self.kws
              name = name % self.kws
          report.report_c.report(
              2, 1000, 1000, self, "DATA", domain + "::" + name + "::%s" % value,
              dict(domain = domain, name = name, value = value))
  
      def report_tweet(self, what, result, extra_report = "",
                       ignore_nothing = False, attachments = None,
                       level = None, dlevel = 0, alevel = 2, ulevel = 5):
          if level == None:		# default args are computed upon def'on
              level = msgid_c.depth()
          self._report_argcheck(what, attachments, level, dlevel, alevel, ulevel)
          level += dlevel
          r = False
          if result.failed > 0:
              tag = "FAIL"
              msg = valid_results[tag][1]
          elif result.errors > 0:
              tag = "ERRR"
              msg = valid_results[tag][1]
          elif result.blocked > 0:
              tag = "BLCK"
              msg = valid_results[tag][1]
          elif result.passed > 0:
              tag = "PASS"
              msg = valid_results[tag][1]
              r = True
          elif result.skipped > 0:
              tag = "SKIP"
              msg = valid_results[tag][1]
              r = True
          else:            # When here, nothing was run, all the counts are zero
              if ignore_nothing == True:
                  return True
              report.report_c.report(level, level + alevel, level + ulevel, self,
                                     "BLCK",
                                     what + " / nothing ran " + extra_report,
                                     attachments)
              return False
          report.report_c.report(level, level + alevel, level + ulevel, self,
                                 tag, what + " " + msg + " " + extra_report,
                                 attachments)
          return r
  
      def shcmd_local(self, cmd, origin = None, reporter = None,
                      logfile = None):
          """
          Run a shell command in the local machine, substituting
          *%(KEYWORD)[sd]* with keywords defined by the target and testcase.
          """
          if origin == None:
              origin = tcfl.origin_get(2)
          return self.testcase._shcmd_local(cmd % self.kws, origin = origin,
                                            reporter = self, logfile = logfile)
      #
      # Actions that operate on the target
      #
  
      def acquire(self):
          """
          Acquire a target
          """
          # This one is annoying, so debuglevel it up
          self.report_info("acquiring", dlevel = 3)
          self.rtb.rest_tb_target_acquire(self.rt, ticket = self.ticket)
          self.report_info("acquired", dlevel = 2)
  
      def release(self):
          """
          Release a target
          """
          # This one is annoying, so debuglevel it up
          self.report_info("releasing", dlevel = 3)
          self.rtb.rest_tb_target_release(self.rt, ticket = self.ticket)
          self.report_info("released", dlevel = 2)
  
      def active(self):
          """
          Mark an owned target as active
  
          For long running tests, indicate to the server that this
          target is still active.
          """
          # This one is annoying, so debuglevel it up
          self.report_info("marking as active", dlevel = 3)
          self.rtb.rest_tb_target_active(self.rt, ticket = self.ticket)
          self.report_info("marked as active", dlevel = 2)
  
      def property_get(self, property_name, default = None):
          """
          Read a property from the target
  
          :param str property_name: Name of the property to read
          :returns str: value of the property (if set) or None
          """
          self.report_info("reading property '%s'" % property_name, dlevel = 1)
          r = self.rtb.rest_tb_property_get(
              self.rt, property_name, ticket = self.ticket)
          self.report_info("read property '%s': '%s'" % (property_name, r))
          if r == None and default != None:
              return default
          return r
  
      def property_set(self, property_name, value = None):
          """
          Set a property on the target
  
          :param str property_name: Name of the property to read
          :param str value: (optional) Value to set; *None* to unset it
          """
          if value:
              assert isinstance(value, basestring)
          self.report_info("setting property '%s' to '%s'"
                           % (property_name, value), dlevel = 1)
          self.rtb.rest_tb_property_set(
              self.rt, property_name, value, ticket = self.ticket)
          self.report_info("set property '%s' to '%s'" % (property_name, value))
  
      def thing_plug(self, thing):
          """
          Connect a thing described in the target's :attr:`tags`
          *things* dictionary to the target.
  
          :param str thing: thing to connect
          """
          if isinstance(thing, target_c):
              thing = thing.id
          assert isinstance(thing, basestring)
          self.report_info("plugging thing '%s'" % thing, dlevel = 3)
          self.rtb.rest_tb_thing_plug(self.rt, thing, ticket = self.ticket)
          self.report_info("plugged thing '%s'" % thing, dlevel = 2)
  
      def thing_unplug(self, thing):
          """
          Disconnect a thing described in the target's :attr:`tags`
          *things* dictionary from the target.
  
          :param str thing: thing to disconnect
          """
          if isinstance(thing, target_c):
              thing = thing.id
          assert isinstance(thing, basestring)
          self.report_info("unplugging thing '%s'" % thing, dlevel = 3)
          self.rtb.rest_tb_thing_unplug(self.rt, thing, ticket = self.ticket)
          self.report_info("unplugged thing '%s'" % thing, dlevel = 2)
  
      def thing_list(self):
          """
          Return a list of connected things
          """
          self.report_info("listing things", dlevel = 3)
          r = self.rtb.rest_tb_thing_list(self.rt, ticket = self.ticket)
          self.report_info("listed things", dlevel = 2)
          return r
  
  
      def console_tx(self, data, console = None):
          """
          Transmits the data over the given console
  
          :param data: data to be sent; data can be anything that can be
             transformed into a sequence of bytes
          :param str console: (optional) name of console over which to
             send the data (otherwise use the default one).
  
          Note this function is equivalent to
          :meth:`tcfl.target_ext_console.console.write`, which is the raw
          version of this function.
  
          However, this function works with the send/expect engine and
          will flush the expect buffer so that next time we call
          :meth:`expect`, it wil look for the expected data **only** in
          data received after calling this function.
  
          """
          # Why does this work? Because next expect runs a new
          # console_rx_eval function that will get it's offset
          # to be zero and thus it'll initialize to the offset from the
          # poller, which this _flush() call is resetting.
          expecter.console_rx_flush(self.testcase.tls.expecter, self, console)
          # the console member is only present if the member extension has
          # been loaded for the target (determined at runtime), hence
          # pylint gets confused
          self.console.write(data, console)	# pylint: disable = no-member
  
      _crlf = '\n'
  
      @property
      def crlf(self):
          """
          What will :meth:`target_c.send` use for CR/LF when sending data
          to the target's consoles. Defaults to ``\\r\\n``, but it can be
          set to any string, even ``""`` for an empty string.
          """
          return self._crlf
  
      @crlf.setter
      def crlf(self, __crlf):
          assert isinstance(__crlf, basestring)
          self._crlf = __crlf
          return self._crlf
  
      def send(self, data, console = None, crlf = None):
          """Like :py:meth:`console_tx`, transmits the string of data
          over the given console.
  
          This function, however, differs in takes only strings and that
          it will append a CRLF sequence at the end of the given
          string. As well, it will *flush* the receive pipe so that next
          time we :meth:`expect` something, it will be only for anything
          received after we called this function.
  
          :param str data: string of data to send
          :param str console: (optional) name of console over which to
             send the data (otherwise use the default one).
          :param str ctlf: (optional) CRLF technique to use, or what to
            append to the string as a CRLF:
  
            - None: use whatever is in :attr:`target_c.crlf`
            - ``\\r``: use carriage return
            - ``\\r\\n``: use carriage return and line feed
            - ``\\n``: use line feed
            - ``ANYSTRING``: append *ANYSTRING*
  
          """
          assert isinstance(data, basestring)
          if crlf == None:
              crlf = self._crlf
          self.console_tx(str(data) + crlf, console)
  
  
      # FIXME: add console_rx_itr() to return an iterator over the read
      # file that closes it on file delete
      # http://stackoverflow.com/a/14798115
  
      def console_rx_read(self, console = None, offset = 0):
          """
          Return the data that has been read for a console until now.
  
          :param str console: (optional) name of console on which the
             the data was received (otherwie use the default one).
          :param int offset: (optional) offset into the recorded data
             from which to read. If negative, it is offset from the last
             data received.
          """
          _, console_code = expecter.console_mk_code(self, console)
          of = self.testcase.tls.expecter.buffers.get(console_code, None)
          if of == None:
              return None
          with open(of.name) as f:
              if offset < 0:
                  f.seek(-offset, 2)
              else:
                  f.seek(offset)
              return f.read()
  
      def console_rx_size(self, console = None):
          """
          Return how many bytes have returned for a console
  
          :param str console: (optional) name of console on which the
             the data was received (otherwie use the default one).
          """
          _, console_code = expecter.console_mk_code(self, console)
          of = self.testcase.tls.expecter.buffers.get(console_code, None)
          if of == None:
              return 0
          with open(of.name) as f:
              stat_info = os.fstat(of.fileno())
              return stat_info.st_size
  
      def on_console_rx(self, regex_or_str, timeout = None, console = None,
                        result = "pass"):
          """Set up an action to perform (pass, fail, block or skip)
          when a string or regular expresison is received on a given
          console in this target.
  
          Note this does not wait for said string; you need to run the
          testcase's *expecter* loop with::
  
          >>>  self.tls.expecter.run()
  
          Als well, those actions will be performed when running
          :meth:`expect` or :meth:`wait` for blocking versions.
  
          This allows you to specify many different things you are
          waiting for from one or *more* targets and wait for all of
          them at the same time and block until all of them are received
          (or timeout).
  
          :param regex_or_str: string or regular expression (compiled
            with :py:func:`re.compile`.
  
          :param int timeout: Seconds to wait for regex_or_str to be
            received, raise :py:exc:`tcfl.tc.failed_e`
            otherwise. If *False*, no timeout check is done; if *None*,
            it is taken from the default timeout set by the testcase.
  
          :param str console: (optional) name of console from which to
             receive the data
  
          :param result: what to do when that regex_or_str is found on
            the given console:
  
            - *pass*, (default) raise :py:exc:`tcfl.tc.pass_e`
            - *block*, raise :py:exc:`tcfl.tc.blocked_e`
            - *error*, raise :py:exc:`tcfl.tc.error_e`,
            - *failed*, raise :py:exc:`tcfl.tc.failed_e`,
            - *blocked*, raise :py:exc:`tcfl.tc.blocked_e`
  
            Note that when running an expecter loop, if seven different
            actions are added indicating they are expected to pass, the
            seven of them must have raised a pass exception (or
            indicated passage somehow) before the loop will consider it
            a full pass. See :py:meth:`tcfl.expecter.expecter_c.run`.
  
          :raises: :py:exc:`tcfl.tc.pass_e`,
            :py:exc:`tcfl.tc.blocked_e`, :py:exc:`tcfl.tc.failed_e`,
            :py:exc:`tcfl.tc.error_e`, :py:exc:`tcfl.tc.skip_e`,
            any other exception from runtimes.
  
          :returns: *True* if a poller for the console was added to the
            testcase's expecter loop, *False* otherwise.
  
          """
          # Won't be re-added if already there
          added = self.testcase.tls.expecter.add(
              False, expecter.console_rx_poller, (self, console),)
          has_to_pass = True if result == "pass" else False
          self.testcase.tls.expecter.add(has_to_pass, expecter.console_rx_eval,
                                         (self, regex_or_str, console, timeout,
                                          result))
          return added
  
      def wait(self, regex_or_str, timeout = None, console = None):
          """
          Wait for a particular regex/string to be received on a given
          console of this target before a given timeout.
  
          See :py:meth:`expect` for a version that just raises
          exceptions when the output is not received.
  
          :param int timeout: Seconds to wait for regex_or_str to be
            received, raise :py:exc:`tcfl.tc.error_e`
            otherwise. If *False*, no timeout check is done; if *None*,
            it is taken from the default timeout set by the testcase.
  
          :param str console: (optional) name of console from which to
             receive the data
  
          :returns: *True* if the output was received before the
            timeout, *False* otherwise.
          """
          if timeout:
              assert isinstance(timeout, int)
          if console:
              assert isinstance(console, basestring)
  
          try:
              with self.on_console_rx_cm(regex_or_str, timeout = timeout,
                                         console = console, result = "pass"):
                  # Run the expect loop
                  self.testcase.tls.expecter.run()
          except pass_e:
              return True
          except error_e:
              return False
  
      def expect(self, regex_or_str, timeout = None, console = None):
          """
          Wait for a particular regex/string to be received on a given
          console of this target before a given timeout.
  
          Similar to :py:func:`wait`, it will raise an exception if
          @regex_or_str is not received before @timeout on @console.
  
          :param int timeout: Seconds to wait for regex_or_str to be
            received, raise :py:exc:`tcfl.tc.error_e`
            otherwise. If *False*, no timeout check is done; if *None*,
            it is taken from the default timeout set by the testcase.
  
          :param str console: (optional) name of console from which to
             receive the data
  
          :returns: Nothing, if the output is received.
  
          :raises: :py:exc:`tcfl.tc.blocked_e` on error,
            :py:exc:`tcfl.tc.error_e` if not received,
            any other runtime exception.
          """
          if timeout:
              assert isinstance(timeout, int)
          if console:
              assert isinstance(console, basestring)
  
          try:
              uid = "just_me" # There is only one target.expect()
                              # running at a single time
              with self.on_console_rx_cm(regex_or_str, timeout = timeout,
                                         console = console, result = "pass",
                                         uid = uid):
                  # Run the expect loop
                  self.testcase.tls.expecter.run()
          except pass_e:
              pass
          finally:
              if uid in self.testcase.tls.expecter.buffers_persistent:
                  del self.testcase.tls.expecter.buffers_persistent[uid]
  
      @contextlib.contextmanager
      def on_console_rx_cm(self, regex_or_str, timeout = None, console = None,
                           result = "pass", uid = None):
          """
          When regex_or_str is received on the given console (default
          console), execute the action given by result.
  
          Context Manager version of :py:meth:`on_console_rx`.
  
          :param regex_or_str: string or regular expression (compiled
            with :py:func:`re.compile`.
  
          :param result: what to do when that regex_or_str is found on
            the given console:
  
            - *pass*, raise :py:exc:`tcfl.tc.pass_e`
            - *block*, raise :py:exc:`tcfl.tc.blocked_e`
            - *failed*, raise :py:exc:`tcfl.tc.error_e`,
            - *failed*, raise :py:exc:`tcfl.tc.failed_e`,
            - *blocked*, raise :py:exc:`tcfl.tc.blocked_e`
  
          :raises: :py:exc:`tcfl.tc.pass_e`,
            :py:exc:`tcfl.tc.blocked_e`,
            :py:exc:`tcfl.tc.error_e`,
            :py:exc:`tcfl.tc.failed_e`,
            :py:exc:`tcfl.tc.skip_e`).
  
          :returns: Nothing
          """
          has_to_pass = True if result == "pass" else False
          _tc = self.testcase
          if isinstance(timeout, numbers.Number) \
             and timeout > _tc.tls.expecter.timeout:
              raise ValueError(
                  "Asked for a timeout of %s that is higher than the "
                  "testcase's timeout of %s; you can adjust the testcases's "
                  "global timeout with `self.tls.expecter.timeout = VALUE`"
                  % (timeout, _tc.tls.expecter.timeout))
          added = False
          try:
              added = _tc.tls.expecter.add(False, expecter.console_rx_poller,
                                           (self, console))
              has_to_pass = True if result == "pass" else False
              _tc.tls.expecter.add(has_to_pass, expecter.console_rx_eval,
                                   (self, regex_or_str, console, timeout,
                                    result, uid))
              yield
          finally:
              _tc.tls.expecter.remove(expecter.console_rx_eval,
                                      (self, regex_or_str, console, timeout,
                                       result, uid))
              if added:
                  _tc.tls.expecter.remove(expecter.console_rx_poller,
                                          (self, console))
  
      def stub_app_add(self, bsp, _app, app_src, app_src_options = ""):
          """\
          Add App builder information for a BSP that has to be stubbed.
  
          When running on a target that has multiple BSPs but some of
          then will not be used by the current :term:`BSP model`, stubs
          might have to be added to those BSPs to make sure their CPUs
          are not going wild. Use this function to specify which app
          builder is going to be used, the path to the stub source and
          build options. The App building mechanism will take it from
          there.
  
          An app builder might determine that a given BSP needs no stub;
          in said case it can remove it from the dict
          :py:meth:`bsps_stub` with:
  
          >>> del target.bsps_stub[BSPNAME]
  
          This is like the app information added by _target_app_add(),
          but it is stored in the target_c instance, not to the testcase
          class.
  
          This is because the stubbing that has to be done is specific
          to each target (as the BSPs to stub each target have might be
          different depending on the target and BSP-model).
  
          Note this information is only added if there is nothing
          existing about said BSP. To override, you need to delete and
          add:
  
          >>> del target.bsps_stub[BSPNAME]
          >>> target.stub_app_add(BSPNAME, etc etc)
          """
          assert isinstance(bsp, basestring)
          assert issubclass(_app, app.app_c)
          assert isinstance(app_src, basestring)
          assert isinstance(app_src_options, basestring)
          if not bsp in self.bsps_stub \
             or self.bsps_stub[bsp] == None \
             or self.bsps_stub[bsp][0] == None:
              self.bsps_stub[bsp] = (_app, app_src, app_src_options)
  
      #
      # Private API
      #
  
      __extensions = {}
  
      def _kws_update(self, bsp = None):
          self.kws.clear()
          # FIXME: all these updates should be checking for overwrites
          self.kws.update(self.testcase.kws)
          self.kws_origin.update(self.testcase.kws_origin)
          self.kws.update(self._kws)
          self.kws_origin.update(self._kws_origin)
          commonl.kws_update_from_rt(self.kws, self.rt)
          kws_bsp = dict()
          if 'bsps' in self.rt.keys() and bsp and bsp in self.rt['bsps']:
              commonl.kws_update_type_string(kws_bsp, self.rt['bsps'][bsp])
              kws_bsp.update(self._kws_bsp.get(bsp, {}))
              kws_bsp['bsp'] = bsp
          self.kws.update(kws_bsp)
  
      def _kw_set(self, kw, value, origin = None):
          """
          Set a string keyword for later substitution in commands
  
          Note these are the target specific keywords (see doc for
          :py:data:`_kws`).
  
          :param str kw: keyword name
          :param str value: value for the keyword
          :param str origin: origin of this setting; if none, it will be
            taken from the stack
  
          """
          # FIXME: override kws __setitem__ to fill it out automagically?
          assert isinstance(kw, basestring)
          assert isinstance(value, basestring)
          if origin == None:
              o = inspect.stack()[1]
              origin = "%s:%s" % (o[1], o[2])
          else:
              assert isinstance(origin, basestring)
          self._kws[kw] = value
          self._kws_origin.setdefault(kw, []).append(origin)
  
      @staticmethod
      def _report_argcheck(message, attachments, level, dlevel, alevel, ulevel):
          assert isinstance(message, basestring)
          if attachments:
              assert isinstance(attachments, dict)
          assert isinstance(level, int)
          assert isinstance(dlevel, int)
          assert isinstance(alevel, int)
          assert isinstance(ulevel, int)
  
      # Semi private API, linkage with tc_c API
      def bsp_model_suffix(self):
          if self.bsp_model:
              return ":%s" % self.bsp_model	# No BSP selected
          else:
              return ""
  
      def bsp_suffix(self):
          if self.bsp_model and self.bsp:
              if self.bsp_model == self.bsp:	# BSP-Model w/ one BSP
                  return ""
              else:				# BSP-Model w/ many BSPs
                  return "/%s" % self.bsp
          else:
              return ""
  
      def _report_mk_prefix(self):
          """
          Update the prefix we use for the logging/reports when some
          parameter changes.
          """
          # Some considerations:
          # - if the target group we are running the testcase on has a
          #   single target, then we don't add anything else, as the target
          #   group will name itself as the lone target--we justneed to add
          #   BSP-model/BSP information as pertinent
          # - if the target group has more than one target, then we add our
          #   name after a | symbol
          tc = self.testcase
          tg = tc.target_group
          if tg:
              if tg.len() == 1:
                  # If there is only one target in the group, the name is
                  # already enough
                  tgname = ""
              else:
                  tgname = "|" + self.fullid
          else:
              tgname = ""
          self._prefix = self.testcase.report_mk_prefix() \
                         + tgname + self.bsp_suffix()
  
      def report_mk_prefix(self):
          return self._prefix
  
  
  
      # Internal target_c API
  
      def _app_get_for_bsp(self, what = None, bsp = None, noraise = False):
          """
          Return which app driver and source has to be used for
          the given what action in the current BSP.
          """
          if what == None:
              what = ""
          else:
              what = what + ": "
          if bsp == None:
              assert self.bsp
              bsp = self.bsp
          target_want = self.testcase._targets[self.want_name]
          twapp = target_want['app']
          if not twapp:
              if noraise:
                  return (None, None)
              raise tcfl.tc.blocked_e("%sNo App builders defined for "
                                      "target '%s', can't figure out "
                                      "what to do" % (what, self.want_name))
          elif bsp in twapp.keys():
              _app = twapp[bsp][0]
              app_src = twapp[bsp][1]
          elif "*" in twapp.keys():
              _app = twapp["*"][0]
              app_src = twapp["*"][1]
          else:
              raise tcfl.tc.blocked_e("%sCan't figure out which "
                                      "app to use" % what)
          return _app, app_src
  
  
  class target_group_c(object):
      """\
      A unique group of targets (each set to an specific :term:`BSP
      model`) assigned to to a testcase for execution.
  
      A testcase can query a :class:`tcfl.tc.target_c` instance of the
      remote target to manipualte it by declaring it as an argument to a
      testcase method, querying the :attr:`targets` dictionary or
      calling :meth:`target`:
  
      >>> @tcfl.tc.target(name = "mytarget")
      >>> class mytest(tcfl.tc.tc_c):
      >>>     ...
      >>>
      >>>     def eval_1(self, mytarget):
      >>>         mytarget.power.cycle()
      >>>
      >>>     def eval_2(self):
      >>>         mytarget = self.target_group.target("mytarget")
      >>>         mytarget.power.cycle()
      >>>
      >>>     def eval_3(self):
      >>>         mytarget = self.targets["mytarget"]
      >>>         mytarget.power.cycle()
      >>>
      """
      def __init__(self, descr):
          assert isinstance(descr, basestring)
          # This is needed so that things can be initialized in the same
          # order (by default) as declared in the testcase
          self._targets = collections.OrderedDict()
          self.descr = descr
          self._name = "FIXME0"
  
      @property
      def name(self):
          return self._name
  
      def name_set(self, tgid):
          self._name = tgid
  
      def len(self):
          """Return number of targets in the group"""
          return len(self._targets)
  
      def target(self, target_name):
          """
          Return the instance of :class:`tcfl.tc.target_c` that represents a
          remote target that met the specification requested
  
          :func:`tcfl.tc.target` decorator with name *target_name*
  
          """
          return self._targets[target_name]
  
      def target_add(self, target_name, _target):
          assert isinstance(target_name, basestring)
          assert isinstance(_target, target_c)
          self._targets[target_name] = _target
  
      @property
      def targets(self):
          """
          Dictionary of :class:`tcfl.tc.target_c` descriptor for a remote target
          keyed by the name they were requested with the
          :func:`tcfl.tc.target` decorator.
          """
          return self._targets
  
      @targets.setter
      def targets(self, targets):
          assert isinstance(targets, dict)
          self._targets = targets
          return targets
  
  assign_period = 2
  
  class result_c():
      def __init__(self, passed = 0, errors = 0, failed = 0,
                   blocked = 0, skipped = 0):
          self.passed = passed
          self.errors = errors
          self.failed = failed
          self.blocked = blocked
          self.skipped = skipped
  
      def __iadd__(self, b):
          self.passed += b.passed
          self.errors += b.errors
          self.failed += b.failed
          self.blocked += b.blocked
          self.skipped += b.skipped
          return self
  
      def __eq__(self, b):
          if b == None:
              return False
          return self.passed == b.passed \
              and self.errors == b.errors \
              and self.failed == b.failed \
              and self.blocked == b.blocked \
              and self.skipped == b.skipped
  
      def __add__(self, b):
          return result_c(self.passed + b.passed,
                          self.errors + b.errors,
                          self.failed + b.failed,
                          self.blocked + b.blocked,
                          self.skipped + b.skipped)
  
      def __repr__(self):
          return "%d (%d %d %d %d %d)" % (self.total(), self.passed,
                                          self.errors, self.failed,
                                          self.blocked, self.skipped)
  
      def total(self):
          return self.passed + self.errors + self.failed \
              + self.blocked + self.skipped
  
      # Return an object on which only one value is set to one and the
      # rest to zero; choose by priority, to report the worse condition
      #
      # If any failed, 1 failure
      # If any errrors, 1 error
      # if any blocked, 1 block
      # if any passed, 1 pass
      # if any skipped, 1 skip
      def summary(self):
          if self.failed > 0:
              return result_c(0, 0, 1, 0, 0)
          elif self.errors > 0:
              return result_c(0, 1, 0, 0, 0)
          elif self.blocked > 0:
              return result_c(0, 0, 0, 1, 0)
          elif self.passed > 0:
              return result_c(1, 0, 0, 0, 0)
          elif self.skipped > 0:
              return result_c(0, 0, 0, 0, 1)
          return result_c(0, 0, 0, 0, 0)
  
      def normalized(self):
          # There might be a more elegant way to do this, but this is
          # quite intelligible
          if self.passed == 0:
              passed = 0
          else:
              passed = 1
          if self.errors == 0:
              errors = 0
          else:
              errors = 1
          if self.failed == 0:
              failed = 0
          else:
              failed = 1
          if self.blocked == 0:
              blocked = 0
          else:
              blocked = 1
          if self.skipped == 0:
              skipped = 0
          else:
              skipped = 1
          return result_c(passed, errors, failed, blocked, skipped)
  
      @staticmethod
      def from_retval(retval):
          if retval == True:
              return result_c(1, 0, 0, 0)
          elif retval == False:
              return result_c(0, 1, 0, 0)
          elif retval == None:
              return result_c(0, 0, 1, 0)
          elif retval == "SKIP":
              return result_c(0, 0, 0, 1)
          else:
              return result_c(0, 0, 0, 0)
  
      @staticmethod
      def _e_maybe_info(e, attachments):
          # Exceptions might raise, in their arguments, a tuple
          # (str, dict), where the string is the message and the
          # dictionary are attachments passed to report_*()
          # functions. In said case, update the attachments and
          # return the message. See :py:class:`exception`.
          if isinstance(e, tcfl.tc.exception):
              attachments.update(e.attachments_get())
              return e.args[0]
          return e
  
      def report(self, tc, message, attachments = None,
                 level = None, dlevel = 0, alevel = 2):
          assert isinstance(tc, tcfl.tc.tc_c)
          assert isinstance(message, basestring)
          if attachments:
              assert isinstance(attachments, dict)
          if level:
              assert level >= 0
          assert dlevel >= 0
          assert alevel >= 0
  
          if 'target' in attachments:
              reporter = attachments['target']
              # create a copy of the dictionary and remove the target
              # spec, use it as reporter.
              attachments = dict(attachments)
              attachments.pop('target')
              assert isinstance(reporter, tcfl.tc.target_c), \
                  "attachment 'target' does not point to a " \
                  "tcfl.tc.target_c but to a type %s" % type(reporter).__name__
          else:
              reporter = tc
  
          if self.blocked:
              report_fn = reporter.report_blck
          elif self.failed:
              report_fn = reporter.report_fail
          elif self.errors:
              report_fn = reporter.report_error
          elif self.passed:
              report_fn = reporter.report_pass
          elif self.skipped:
              report_fn = reporter.report_skip
          else:
              report_fn = reporter.report_blck
              message = '(nothing ran) ' + message
  
          report_fn(message, attachments,
                    level = level, dlevel = dlevel, alevel = alevel)
  
  
      @staticmethod
      def report_from_exception(_tc, e, attachments = None,
                                force_result = None):
          """
          Given an exception, report using the testcase or target report
          infrastructure on the exception, traces to it as well as any
          attachments it came with and return a valid :class:`result_c`
          code.
  
          By default, this is the mapping:
  
          - :meth:`tc_c.report_pass` is used for :exc:`pass_e`
          - :meth:`tc_c.report_error` is used for :exc:`error_e`
          - :meth:`tc_c.report_fail` is used for :exc:`failed_e`
          - :meth:`tc_c.report_blck` is used for :exc:`blocked_e`
            and any other exception
          - :meth:`tc_c.report_skip` is used for :exc:`skip_e`
  
          However, it can be forced by passing as *force_result* or each
          testcase can be told to consider specific exceptions as others
          per reporting using the :attr:`tcfl.tc.tc_c.exception_to_result`.
  
          :param bool force_result: force the exception to be
            interpreted as :exc:`tcfl.tc.pass_e`, :exc:`error_e`,
            :exc:`failed_e`, :exc:`tcfl.tc.blocked_e`, or :exc:`skip_e`; note
            there is also translation that can be done from
            :attr:`tcfl.tc.tc_c.exception_to_result`.
          """
          if attachments == None:
              attachments = {}
          phase = msgid_c.phase()
          if phase == None:
              phase = ""
          else:
              phase = phase + " "
          _e = result_c._e_maybe_info(e, attachments)
          reporter = attachments.pop('target', _tc)
          _alevel = attachments.pop('alevel', 1)
          dlevel = attachments.pop('dlevel', 0)
  
          if isinstance(_tc, target_c):
              tc = _tc.testcase
          else:
              assert isinstance(_tc, tc_c)
              tc = _tc
  
          trace_alevel = 4
          trace_dlevel = 4
          if force_result == None:
              force_result = tc.exception_to_result.get(type(e), None)
          if isinstance(e, pass_e) or force_result == pass_e:
              report_fn = reporter.report_pass
              tag = valid_results['PASS'][1]
              result = result_c(1, 0, 0, 0, 0)
          elif isinstance(e, error_e) or force_result == error_e:
              report_fn = reporter.report_error
              tag = valid_results['ERRR'][1]
              result = result_c(0, 1, 0, 0, 0)
          elif isinstance(e, failed_e) or force_result == failed_e:
              report_fn = reporter.report_fail
              tag = valid_results['FAIL'][1]
              result = result_c(0, 0, 1, 0, 0)
          elif isinstance(e, blocked_e) or force_result == blocked_e:
              report_fn = reporter.report_blck
              tag = valid_results['BLCK'][1]
              result = result_c(0, 0, 0, 1, 0)
          elif isinstance(e, skip_e) or force_result == skip_e:
              report_fn = reporter.report_skip
              tag = valid_results['SKIP'][1]
              result = result_c(0, 0, 0, 0, 1)
          else:
              report_fn = reporter.report_blck
              tag = 'blocked: exception'
              result = result_c(0, 0, 0, 1, 0)
              # This is bad, report as high as we can
              dlevel = 0
              alevel = 0
              trace_alevel = 0
              trace_dlevel = 0
  
          _attachments = { "%s%s trace" % (phase, tag) : traceback.format_exc() }
          _attachments.update(attachments)
          report_fn("%s%s: %s" % (phase, tag, _e), _attachments,
                    dlevel = dlevel, alevel = trace_alevel)
          return result
  
      @staticmethod
      def from_exception(fn):
          """
          Call a phase function to translate exceptions into
          :class:`tcfl.tc.result_c` return codes.
  
  
          Passes through the return code, unless it is None, in which
          case we just return result_c(1, 0, 0, 0, 0)
  
          Note this function prints some more extra detail in case of
          fail/block/skip.
          it.
          """
          def _decorated_fn(*args, **kwargs):
  
              _tc = args[0] # The `self`  argument to the test case
              try:
                  r = fn(*args, **kwargs)
                  if r == None:
                      return result_c(1, 0, 0, 0, 0)
                  else:
                      return r
              # Some exceptions that are common and we know about, so we
              # can print some more info that will be helpful
              except subprocess.CalledProcessError as e:
                  return result_c.report_from_exception(
                      _tc, e, attachments = {
                          "output": e.output,
                          "return": e.returncode,
                          "cmd": e.cmd
                      }, force_result = error_e
                  )
              except OSError as e:
                  attachments = dict(
                      errno = e.errno,
                      strerror = e.strerror
                  )
                  if e.filename:
                      attachments['filename'] = e.filename
                  return result_c.report_from_exception(_tc, e, attachments)
              except Exception as e:
                  return result_c.report_from_exception(_tc, e)
  
          return _decorated_fn
  
  
  class tc_logadapter_c(logging.LoggerAdapter):
      """
      Logging adapter to prefix test case's current :term:`BSP model`,
      bsp and target name.
      """
      id = None
      prefix = None
      def process(self, msg, kwargs):
          return '[%08x] %s: %s ' % (self.id, self.prefix, msg), kwargs
  
  
  
  #
  # Testcase Driver API
  #
  
  
  def tags(*args, **kwargs):
      """
      Add tags to a testcase
  
      :param str args: every `arg` is converted into a boolean tag
        (present, thus True)
      :param dict kwargs: every argument `name = value` is added as tag
        `name` with value `value`.
      """
      origin = tcfl.origin_get(2)
      # It is fine to pass no tags at all; this allows us to
      # calculate them dynamically (for example seeing if some env
      # vars are available and filing a skip if not available) but
      # then leaving it empty as needed.
      def decorate_class(cls):
          """
          Wrap function
          """
          assert isinstance(cls, type)
          assert issubclass(cls, tc_c)
          # Ugly way of doing it; we want to build upon the tags of the
          # base class -- but not modify them; so when we add, we COPY the
          # _tags dictionary from our base class to modify it specific
          # to this class
          origin = tcfl.origin_get(2)
          if id(super(cls, cls)._tags) == id(cls._tags):
              cls._tags = dict(super(cls, cls)._tags)
          for name in args:
              cls._tags[name] = (True, origin)
          if kwargs:
              for key, val in kwargs.iteritems():
                  cls._tags[key] = (val, origin)
          return cls
      return decorate_class
  
  
  def serially():
      """
      Force a testcase method to run serially (vs :func:`concurrently`).
  
      Remember methods that are ran serially are run first and by
      default are those that
  
      - take more than one target as arguments
  
      - are evaluation methods
      """
      def decorate_fn(fn):
          setattr(fn, "execution_mode", 'serial')
          return fn
      return decorate_fn
  
  def concurrently():
      """
      Force a testcase method to run concurrently after all the serial
      methods (vs decorator :func:`serially`).
  
      Remember methods that are ran concurrently are run after the
      serial methods and by default those that:
  
      - are not evaluation methods
  
      - take only one target as argument (if you force two methods that
        share a target to run in parallel, it is your responsiblity to
        ensure proper synchronization
      """
      def decorate_fn(fn):
          setattr(fn, "execution_mode", 'parallel')
          return fn
      return decorate_fn
  
  def _target_app_setup(obj, cls_name, target_want_name):
      """
      Setup the phase hooks so that the App builder is called in the
      different phases of testcase execution.
      """
      if hasattr(obj, "configure_for_%s" % target_want_name):
          # already done
          return
      # Note we don't add eval* or test* hooks here, because we expect
      # the creator of testcases to do that
      for fnname in [
              'configure',
              'build',
              'setup',
              'start',
              'deploy',
              'teardown',
              'clean']:
          method_name = "%s_50_%s" % (fnname, target_want_name)
          dest_method_name = "__%s_%s_50_for_%s" % (cls_name, fnname,
                                                    target_want_name)
          # Do not override existing methods, so the user can override
          # defaults given by app_X.
          if hasattr(obj, method_name):
              method_name = "overriden_" + method_name
          # Monkey Patching for the win; this is basically the dirty way
          # of adding a method to an existing class
          object_code = compile(
              "def __%s_%s_50_for_%s(tc, %s):\n"
              "    return tc._%s_50_for_target(%s)\n"
              % (cls_name, fnname, target_want_name,
                 target_want_name, fnname, target_want_name),
              tcfl.origin_get(2) + "|" + tcfl.origin_get(), 'exec')
          eval(object_code)
          if type(obj) == types.TypeType:
              # Bind to the class, no need to bind to an instance, will
              # self itself in __method_trampoline_call()
              # Note how we cast with the instance set to None, so in
              # __method_trampoline_call() we can tell we need to bind it.
              value = eval(dest_method_name)
          else:
              # Bind to an object, force not binding to an instance,
              # setting it to None, so in __method_trampoline_call() we
              # can tell we need to bind it.
              if isinstance(obj, types.TypeType):
                  # We are binding to a class
                  value = types.MethodType(eval(dest_method_name), None, obj)
              else:
                  # We are binding to a object instance
                  value = types.MethodType(eval(dest_method_name), None,
                                           type(obj))
          setattr(obj, method_name, value)
  
  def _target_app_add(obj, target_want_name, app_name, app_src):
      """
      """
      # note there can be only "*" is it is the ONLY one
      app_src = app.args_app_src_check(app_name, app_src)
      if app.driver_valid(app_name):
          twapp = obj._targets[target_want_name]['app']
          _app = app._driver_get(app_name)
          app_src = app._app_src_validate(app_name, app_src)
          if isinstance(app_src, dict):
              for bsp, src in app_src.iteritems():
                  twapp[bsp] = (_app, src)
          else:
              twapp["*"] = (_app, app_src)
          # Consistency check
          if "*" in twapp and len(twapp) > 1:
              raise blocked_e(
                  "%s: Cannot have a wildcard BSP '*' and other "
                  "BSPs specified in the app_* sections @%s"
                  % (target_want_name, tcfl.origin_get_object(app_name)))
      elif app_name.endswith("_options") and app.driver_valid(app_name[:-8]):
      # Note 8 is the lenght of "_options"
          pass
      else:
          raise blocked_e("%s: unknown App builder '%s' (or option to "
                          "existing) @%s"
                          % (target_want_name, app_name, tcfl.origin_get(3)))
  
  def _target_want_decorate_class(obj, cls_name,
                                  name, type_name, short_type_name,
                                  next_index, **kwargs):
      """
      Wrap function
      """
      # next_index: index number to use if we have to generate a default
      # name; passed so we can use different indexes for ICs or for
      # Targets
  
      # FIXME: the order makes it counterintuitive, because we are # defining:
      #
      # @tcfl.tc.interconnect()
      # @tcfl.tc.interconnect()
      # @tcfl.tc.target()
      # @tcfl.tc.target()
      # @tcfl.tc.target()
      #
      # Would be listed as ic0 ic target2 target1 target. Change so that
      # all the decorator does is write the list of wants and keywords
      # and we'll have init sequence them in the right order, so the
      # (automatic) naming  is more intuitive and results in:
      #
      # ic0 ic1 target target1 target2
      #
      # Will need to remove hack `FIXME: reversed for decorator workaround`_
      if name != None:
          assert isinstance(name, basestring)
          if name in obj._targets:
              raise blocked_e("%s name '%s' @%s already defined, "
                              "choose another"
                              % (type_name, name, tcfl.origin_get(2)))
          target_want_name = name
      else:
          if next_index == 0:
              target_want_name = short_type_name
          else:
              target_want_name = short_type_name + "%d" % next_index
      origin = tcfl.origin_get(3)
      obj._targets[target_want_name] = dict(
          app = {},
          kws = kwargs,
          origin = origin)
      return target_want_name
  
  def _target_want_add_check_key(obj, cls_name, target_want_name,
                                 key, val):
      valid = False
      if key.startswith('app_'):
          _target_app_setup(obj, cls_name, target_want_name)
          _target_app_add(obj, target_want_name, key, val)
          valid = True
      elif key == "mode":
          valid_modes = [ 'one-per-type', 'any', 'all' ]
          if val not in valid_modes:
              raise blocked_e(
                  "%s: unknown value (accepts %s"
                  % (key, ", ".join(valid_modes)))
          valid = True
      return valid
  
  def _target_want_add(obj, cls_name, name, spec, origin, **kwargs):
      target_want_name = _target_want_decorate_class(
          obj, cls_name, name, "target", "target", obj._target_count, **kwargs)
      obj._targets[target_want_name]['spec'] = spec
      obj._targets[target_want_name]['origin'] = origin
      for key, val in kwargs.iteritems():
          valid = _target_want_add_check_key(obj, cls_name, target_want_name,
                                             key, val)
          if not valid:
              raise blocked_e("%s: unknown key @%s" % (key, origin))
      obj._target_count += 1
  
  def target_want_add(_tc, target_want_name, spec, origin, **kwargs):
      """\
      Add a requirement for a target to a testcase instance
  
      Given a testcase instance, add a requirement for it to need a
      target, filtered with the given specification (*spec*, which
      defaults to any), a name and optional arguments in the form of
      keywords.
  
      This is equivalent to the :func:`tcfl.tc.target` decorator, which
      adds the requirement to the class, not to the instance. Please
      refer to it for the arguments.
      """
      assert isinstance(_tc, tc_c)
      cls = type(_tc)
      if id(_tc._targets) == id(cls._targets):
          # this means that this testcase instance does not have a list
          # of targets specific to it. Because now we are adding one, we
          # make a private copy of all that information *for* the
          # testcase, separated from the type
          _tc._targets = copy.deepcopy(cls._targets)
          _tc._target_count = cls._target_count
          _tc._interconnects = copy.deepcopy(cls._interconnects)
      _target_want_add(_tc, cls.__name__, target_want_name,
                       spec, origin, **kwargs)
  
  def target(spec = None, name = None, **kwargs):
      """\
      Add a requirement for a target to a testcase instance
  
      For each target this testcase will need, a :ref:`filtering
      specification <target_specification>` can be given (*spec*), a
      name (or it will default to *targetN* except for the first one,
      which is just *target*) and optional arguments in the form of
      keywords.
  
      Of those optional arguments, the most important are the app_*
      arguments. An app_* argument is supplying a source path for an
      application that has to be (maybe) configured, built and deployed
      to the target. The decorator will add phase methods to the
      testcase to configure, build and deploy the application. Now,
      depending on the application drivers installed, the application
      can be built or not. FIXME make this explanation better.
  
      :param str spec: :ref:`specification <target_specification>` to
        filter against the tags the remote target exposes.
      :param str name: name for the target (must not exist already). If
        none, first declared target is called *target*, the next
        *target1*, then *target2* and so on.
      :param dict kwargs: extra keyword arguments are allowed, which
        might be used in different ways that are still TBD. Main ones
        recognized:
  
         - *app_NAME = dict(BSP1: PATH1, BSP1: PATH2)*: specify a list
           of paths to apps that shall be built and deployed to the
           given BSPs by App builder *app_NAME*; App builders exist for
           Zephyr, Arduino Sketch and other setups, so you don't
           manually have to build your apps. You can create your own
           too.
  
           When a board is being run in a multiple BSP mode, each BSP
           has to be added to an App builder if using the App builder
           support, otherwise is an error condition.
  
         - *app_NAME = PATH*: the same, but one for when one BSP is
           used; it applies to `any` BSP in a single :term:`BSP model`
           target.
  
           FIXME: add link to app builders
  
         - *app_NAME_options = STRING*: Extra options to pass to the APP
            builder FIXME: support also BSP_options?
  
         .. _tcf_target_modes:
  
         - *mode*: how to consider this target at the time of generating
            multiple permutations of targets to run a testcase:
  
            - *any*: run the testcase on any target that can
              be found to match the specification
  
            - *one-per-type*: run the testcase on one target of each
              type that meets the specification (so if five targets
              match the specification but they are all of the same type,
              only one will run it; however, if there are two different
              types in the set of five, one of each type will run it)
  
            - *all*: run on every single target that matches the
              specification
  
           Specially on testcases that require multiple targets, there
           can be a huge number of permutations on how to run the
           testcase to ensure maximum coverage of different combinations
           of targets; some experimentation is needed to decide how to
           tell TCF to run the testcase and balance how many resources
           are used.
  
      """
  
      def decorate_class(cls):
          # This uses quite a dirty hack to add a method. I'd
          # like to know a better way to do it.
          #
          # The constraint is that we need to add the method with
          # SPECIFIC argument names so that later on, the method
          # dispatcher can call it with the right target instance.
          #
          # So we literally compile a chunk of python code that defines
          # a function __CLASSNAME_PHASE_for_TARGETNAME(TARGETNAME) that
          # than can behave as a method, as it takes a testcase for
          # first argument (as if it were `self`). This it calls
          # tc_c._PHASE_50_for_target() to do the actual work. We use
          # setattr() to inject it as a method of the class, calling it
          # PHASE_for_TARGETNAME(TARGETNAME).
          #
          # The method dispatcher in tc_c.methods_call() will call
          # PHASE_for_TARGETNAME(TARGETNAME) with TARGETNAME being an
          # instance to the right target_c for the testcase from
          # tc_c._targets[]
          #
          assert issubclass(cls, tc_c), \
              "cls has to be a subclass of tcfl.tc.tc_c"
          # Ugly way of doing it; we want to build upon the tags of the
          # base class -- but not modify them; so when we add, we COPY the
          # _targets dictionary from our base class to modify it specific
          # to this class
          # FIXME: replace target_want_name with targettarget_want_name
          if id(super(cls, cls)._targets) == id(cls._targets):
              cls._targets = collections.OrderedDict(super(cls, cls)._targets)
  
          _target_want_add(cls, cls.__name__,
                           name, spec, tcfl.origin_get(2), **kwargs)
          return cls
  
      return decorate_class
  
  
  def interconnect(spec = None, name = None, **kwargs):
      """\
      Add a requirement for an interconnect to a testcase instance
  
      An interconect is a target that binds two or more targets
      together, maybe provides interconnectivity services (networking or
      any other); we declare it's need just like any other
      target--however, we add the name to an special list so it is
      easier to handle later.
  
      The arguments are the same as to :func:`tcfl.tc.target`.
      """
  
      def decorate_class(cls):
          # This uses quite a dirty hack to add a method. I'd
          # like to know a better way to do it.
          #
          # The constraint is that we need to add the method with
          # SPECIFIC argument names so that later on, the method
          # dispatcher can call it with the right target instance.
          #
          # So we literally compile a chunk of python code that defines
          # a function __CLASSNAME_PHASE_for_TARGETNAME(TARGETNAME) that
          # than can behave as a method, as it takes a testcase for
          # first argument (as if it were `self`). This it calls
          # tc_c._PHASE_50_for_target() to do the actual work. We use
          # setattr() to inject it as a method of the class, calling it
          # PHASE_for_TARGETNAME(TARGETNAME).
          #
          # The method dispatcher in tc_c.methods_call() will call
          # PHASE_for_TARGETNAME(TARGETNAME) with TARGETNAME being an
          # instance to the right target_c for the testcase from
          # tc_c._targets[]
          #
          assert issubclass(cls, tc_c), \
              "cls has to be a subclass of tcfl.tc.tc_c"
          # Ugly way of doing it; we want to build upon the tags of the
          # base class -- but not modify them; so when we add, we COPY the
          # _targets dictionary from our base class to modify it specific
          # to this class
          # FIXME: replace target_want_name with targettarget_want_name
          if id(super(cls, cls)._targets) == id(cls._targets):
              cls._targets = copy.deepcopy(super(cls, cls)._targets)
  
          origin = tcfl.origin_get(2)
          ic_want_name = _target_want_decorate_class(
              cls, cls.__name__,
              name, "interconnect", "ic", cls._ic_count, **kwargs)
          if id(super(cls, cls)._interconnects) == id(cls._interconnects):
              cls._interconnects = copy.deepcopy(super(cls, cls)._interconnects)
          cls._targets[ic_want_name]['spec'] = spec
          cls._targets[ic_want_name]['origin'] = origin
          cls._interconnects.add(ic_want_name)
          for key, val in kwargs.iteritems():
              valid = _target_want_add_check_key(
                  cls, cls.__name__, ic_want_name,
                  key, val)
              if not valid:
                  raise blocked_e("%s: unknown key @%s" % (key, origin))
          # By default, interconnects are explored all
          cls._targets[ic_want_name]['kws'].setdefault('unlimited', True)
          cls._ic_count += 1
          return cls
  
      return decorate_class
  
  
  #
  # Metaclass for tc_c, to initialize class specific fields
  #
  # Note we cannot define it nested due to some bug in the pickling
  # code that fails to pick up nested classes.
  #
  # https://stackoverflow.com/questions/1947904/how-can-i-pickle-a-nested-class-in-python
  #
  # Note we only manipulate these in the single-thread path of 'tcf
  # run'.
  #
  # Can't really use a multiprocess[ing].SyncManager here for the locks,
  # as in places where we run this the paths are too deep and it goes
  # into failing to create the Unix socket because the path is too long.
  # In any case, it doesn't really matter, because this part we only
  # touch it in the single-threaded path of TCF run.
  #
  class _tc_mc(type):
      def __init__(cls, name, bases, d):
          type.__init__(cls, name, bases, d)
  
          #: What is the current accumulated result of all the
          #: testcases of this class that have ran and finished
          cls.class_result = result_c(0, 0, 0, 0, 0)
  
  #
  # The core testcase object
  #
  class tc_c(object):
      r"""
      A testcase, with instructions for configuring, building, deploying,
      setting up, running, evaluating, tearing down and cleaning up.
  
      Derive this class to create a testcase, implementing the different
      testcase methods to build, deploy and evaluate if it is considered
      a pass or a failure:
  
      >>> class sometest(tcfl.tc.tc_c):
      >>>
      >>>     def eval_device_present(self):
      >>>          if not os.path.exists("/dev/expected_device"):
      >>>              raise tcfl.tc.error_e("Device not connected")
      >>>
      >>>     def eval_mode_correct(self):
      >>>          s = os.stat("/dev/expected_device"):
      >>>          if s.st_mode & 0x644 == 0:
      >>>              raise tcfl.tc.failed_e("wrong mode")
  
      .. note:: the class will be ignored as a testcase if its name
                starts with *_base_*; this is useful to create common
                code which will be instantiated in another class without
                it being confused with a testcase.
  
      The runner will call the testcase methods to evaluate the test;
      any failure/blockage causes the evaluation to stop and move on to
      the next testcase:
  
      - *configure\*()* for getting source code, configuring a buid,
        etc ..
  
      - *build\*()* for building anything that is needed to run the
        testcase
  
      - *deploy\*()* for deploying the build products or artifacts
        needed to run the testcase to the diffrent targets
  
      - For evaluating:
  
        - *setup\*()* to setup the system/fixture for an evaluation run
        - *start\*()* to start/power-on the targets or anything needed
          for the test case evaluation
        - *eval\*()* to actually do evaluation actions
        - *teardown\*()* for powering off
  
        As well, any *test\*()* methods will be run similarly, but for
        each, the sequence called will be setup/start/test/teardown (in
        contrast to *eval* methods, where they are run in sequence
        without calling setup/start/teardown in between).
  
      - *clean\*()* for cleaning up (ran only if *-L* is passed on the
        command line)
  
      - *class_teardown* is mostly used for self-testing and debugging,
         but are functions called whenever every single testcase of the
         same class has completed executing.
  
      Methods can take no arguments or the names of one or more targets
      they will operate with/on. These targets are declared using the
      :func:`tcfl.tc.target` (for a normal target) and
      :func:`tcfl.tc.interconnect` (for a target that
      interconnects/groups the rest of the targets together).
  
      The methods that take no targets will be called sequentially in
      **alphabetical order** (not in declaration order!). The methods
      that take different targets will be called in parallel (to
      maximize multiple cores, unless decorated with
      :func:`tcfl.tc.serially`). Evaluation functions are always called
      sequentially, except if decorated with :func:
  
      The testcase methods use the APIs exported by this class and module:
  
       - to report information at the appropiate log level:
         :meth:`report_pass`, :meth:`report_fail`, :meth:`report_blck`
         and :meth:`report_info`
  
       - raise an exception to indicate result of this method:
  
         - *pass*, raise :py:exc:`tcfl.tc.pass_e` (or simply return)
         - *failed*, raise :py:exc:`tcfl.tc.failed_e`,
         - *error*, raise :py:exc:`tcfl.tc.error_e`,
         - *blocked*, raise :py:exc:`tcfl.tc.blocked_e`; any other
           uncaught Python exception is also converted to this.
         - *skipped*, raise :py:exc:`tcfl.tc.skip_e`
  
       - run commands in the local machine with :meth:`shcmd_local`; the
         command can be formatted with *%(KEYWORD)[sd]* that will be
         substituted with values found in :attr:`kws`.
  
       - Interact with the remote targets through instances of
         :class:`target_c` that represent them:
  
         - via arguments to the method
         - via :attr:`targets`, a dictionary keyed by the names of the
           targets requested with the :func:`target` and
           :func:`interconnect` decorators; for example:
  
           >>> @tcfl.tc.interconnect()	# named "ic" by default
           >>> @tcfl.tc.target()		# named "target1" by default
           >>> @tcfl.tc.target()		# named "target" by default
           >>> class mytest(tcfl.tc.tc_c):
           >>>     ...
           >>>
           >>>     def start(self, ic, target, target1):
           >>>         ic.power.cycle()
           >>>         target.power.cycle()
           >>>         target1.power.cycle()
           >>>
           >>>     def eval_1(self, target):
           >>>         target.expect("Hello world")
           >>>
           >>>     def eval_2(self):
           >>>         target2 = self.target_group.target("target2")
           >>>         target2.expect("Ready")
           >>>
           >>>     def eval_3(self):
           >>>         mytarget = self.targets["ic"]
           >>>         ic.expect("targets are online")
           >>>
           >>>     def teardown(self):
           >>>         for _n, target in reversed(self.targets.iteritems()):
           >>>            target.power.off()
           >>>
  
         :class:`target_c` expose APIs to act on the targets, such as
         power control, serial console access, image deployment
  
      """
  
      #
      # Public testcase interface
      #
  
      #: List of places where we declared this testcase is build only
      build_only = []
  
      def __init__(self, name, tc_file_path, origin):
          for hook_pre in self.hook_pre:
              assert callable(hook_pre), \
                  "tcfl.tc.tc_c.hook_pre contains %s, defined as type '%s', " \
                  "which  is not callable" % type(hook_pre).__name__
  
          #: Keywords for *%(KEY)[sd]* substitution specific to this
          #: testcase.
          #:
          #: Note these do not include values gathered from remote
          #: targets (as they would collide with each other). Look at
          #: data:`target.kws <tcfl.tc.target_c.kws>` for that.
          #:
          #: These can be used to generate strings based on information,
          #: as:
          #:
          #:   >>>  print "Something %(FIELD)s" % target.kws
          #:   >>>  target.shcmd_local("cp %(FIELD)s.config final.config")
          #:
          #: Fields available:
          #:
          #:   - `runid`: string specified by the user that applies to
          #:     all the testcases
          #:
          #:   - `srcdir` and `srcdir_abs`: directory where this
          #:     testcase was found
          #:
          #:   - `thisfile`: file where this testscase as found
          #:
          #:   - `tc_hash`: unique four letter ID assigned to this
          #:     testcase instance. Note that this is the same for all
          #:     the targets it runs on. A unique ID for each target of
          #:     the same testcase instance is the field *tg_hash* in the
          #:     target's keywords :data:`target.kws
          #:     <tcfl.tc.target_c.kws>` (FIXME: generate, currently
          #:     only done by app builders)
          #:
          self.kws = {}
          #: Origin of the keyword in self.kws; the values for these are
          #: lists of places where the setting was set, or re-set
          self.kws_origin = {}
  	#: For use during execution of phases; testcase drivers can
          #: store anything they need during the execution of each phase.
          #: It will be, however, deleted when the phase is completed.
          self.buffers = {}
          #: Lock to access :attr:`buffers` safely from multiple threads
          #: at the same time for the same testcase.
          self.buffers_lock = threading.Lock()
          self.name = name
          self.kw_set('tc_name', self.name)
          self.kw_set('cwd', os.getcwd())
          # This one is left for drivers to do their thing in here
          self.kw_set('tc_name_short', self.name)
          self.origin = origin
          self.kw_set('tc_origin', self.origin)
          # Instantiate a copy of build_only to respect what the derived
          # class has set -- we need type(self) to ensure that we get
          # the type of the derived class -- and here I might not know
          # enough Python, mind you, there might be a better way to do
          # it-- and deepcopy to make sure we have an object that is
          # fully owned by this instance object in case it decides to
          # modify it.
          self.build_only = copy.deepcopy(type(self).build_only)
  
          #: dictionary of tags this test case has been stamped with
          self._tags = dict(self._tags)
          self._tags['name'] = (self.name, tcfl.origin_get())
          self._tags_update()
          #: Group of targets this testcase is being ran on
          self.target_group = None
  
          # Create a logger
          self.log = tc_logadapter_c(logging, None)
  
          if os.path.isabs(tc_file_path):
              srcdir = os.path.relpath(os.path.dirname(
                  os.path.abspath(tc_file_path)))
              thisfile = tc_file_path
          else:
              # This makes sure "FILE" resolves to "./FILE"
              srcdir = os.path.relpath(os.path.dirname(
                  os.path.abspath(tc_file_path)))
              thisfile = os.path.join(srcdir, os.path.basename(tc_file_path))
          self._kw_set("runid", "" if tc_c.runid == None else tc_c.runid,
                       origin = "cmdline")
          self._kw_set("srcdir", srcdir)
          self._kw_set("srcdir_abs",
                       os.path.dirname(os.path.abspath(tc_file_path)))
          self._kw_set("thisfile", thisfile)
  
          self.tls = threading.local()
          #: Expect loop to wait for things to happen
          self.tls.expecter = expecter.expecter_c(self._expecter_log, self,
                                                  timeout = 60)
          # Ticket ID for this testcase / target group
          self.ticket = None
          # The group of targets where the TC is running
          self._target_group = None
          #: :class:`Target objects <tcfl.tc.target_c>`) in which this
          #: testcase is running (keyed by target want name, as given to
          #: decorators :func:`tcfl.tc.target` and
          #: func:`tcfl.tc.interconnect`.
          #: Note this maps to ``self._target_groups_c.targets()`` for
          #: convenience.
          self.targets = {}
          # List of remote targets selected for this run
          self.rt_selected = None
          self.ic_selected = None
          # FIXME
          self.skip_reports = False
  
          # Prefix used for lines that print reports
          self._report_prefix = ""
          self._prefix_update()	# Update the prefix
  
          # Prefix used for lines that print reports
          self._report_prefix = ""
          self._prefix_update()	# Update the prefix
  
          # instance specific list of files/paths to wipe at the end
          self._cleanup_files = set()
  
          #: Result of the last evaluation run
          #:
          #: When an evaluation is run (setup/start/eval/teardown), this
          #: variable reflexts the evaluation status; it is meant to be
          #: used during the *teardown* phase, so for example, in case
          #: of failure, the teardown phase might decide to gather
          #: information about the current target's state.
          self.result_eval = result_c(0, 0, 0, 0, 0)
  
          #: Result of the last run of all phases in this testcase
          #:
          #: we might need to look at this in other testcases executed
          #: inmediately after (as added with :meth:`post_tc_append`).
          self.result = result_c(0, 0, 0, 0, 0)
  
          # Combination of runid/ident string
          self._ident = ""
  
          # Testcases we need to run when we are done running this one
          self._tcs_post = []
  
          # Initialize prefixes and a few keywords we need -- we might
          # override these later if/when we assign a target group.
          self.mkticket()
          self.tmpdir = os.path.join(tc_c.tmpdir, self.ticket)
          try:
              os.makedirs(self.tmpdir)
          except OSError:
              if not os.path.isdir(self.tmpdir):
                  raise
          self._kw_set("tmpdir", self.tmpdir)
          self._kw_set("tc_hash", self.ticket)
  
          self.ts_start = time.time()
          self.ts_end = None
  
          # Always before we start, run the site hook
          for hook in self.hook_pre:
              hook(self)
  
      def __thread_init__(self, expecter_parent):
          """
          When we run some methods of this object in a different thread,
          we need to initialize some parts first.
  
          This is currently quite a dirty hack, but it is what we
          have. We use it when we clone the object to run in a target
          group or when we spawn thredas to run methods in parallel.
          """
          self.tls.expecter = expecter.expecter_c(
              self._expecter_log, self, timeout = expecter_parent.timeout)
  
      def is_static(self):
          """
          Returns *True* if the testcase is *static* (needs to targets to
          execute), *False* otherwise.
          """
          return not self._targets
  
  
      # Reporting interface FIXME: document
      def report_pass(self, message, attachments = None,
                      level = None, dlevel = 0, alevel = 2, ulevel = 5):
          if level == None:		# default args are computed upon def'on
              level = msgid_c.depth()
          self._report_argcheck(message, attachments,
                                level, dlevel, alevel, ulevel)
          level += dlevel
          report.report_c.report(level, level + alevel, level + ulevel, self,
                                 "PASS", message, attachments)
  
      def report_error(self, message, attachments = None,
                       level = None, dlevel = 0, alevel = 2, ulevel = 5):
          if level == None:		# default args are computed upon def'on
              level = msgid_c.depth()
          self._report_argcheck(message, attachments,
                                level, dlevel, alevel, ulevel)
          level += dlevel
          report.report_c.report(level, level + alevel, level + ulevel, self,
                                 "ERRR", message, attachments)
  
      def report_fail(self, message, attachments = None,
                      level = None, dlevel = 0, alevel = 2, ulevel = 5):
          if level == None:		# default args are computed upon def'on
              level = msgid_c.depth()
          self._report_argcheck(message, attachments,
                                level, dlevel, alevel, ulevel)
          level += dlevel
          report.report_c.report(level, level + alevel, level + ulevel, self,
                                 "FAIL", message, attachments)
  
      def report_blck(self, message, attachments = None,
                      level = None, dlevel = 0, alevel = 2, ulevel = 5):
          if level == None:		# default args are computed upon def'on
              level = msgid_c.depth()
          self._report_argcheck(message, attachments,
                                level, dlevel, alevel, ulevel)
          level += dlevel
          report.report_c.report(level, level + alevel, level + ulevel, self,
                                 "BLCK", message, attachments)
  
      def report_skip(self,  message, attachments = None,
                      level = None, dlevel = 0, alevel = 2, ulevel = 5):
          if level == None:		# default args are computed upon def'on
              level = msgid_c.depth()
          self._report_argcheck(message, attachments,
                                level, dlevel, alevel, ulevel)
          level += dlevel
          report.report_c.report(level, level + alevel, level + ulevel, self,
                                 "SKIP", message, attachments)
  
      def report_info(self, message, attachments = None,
                      level = None, dlevel = 0, alevel = 2, ulevel = 5):
          if level == None:		# default args are computed upon def'on
              level = msgid_c.depth()
          self._report_argcheck(message, attachments,
                                level, dlevel, alevel, ulevel)
          level += dlevel
          report.report_c.report(level, level + alevel, level + ulevel, self,
                                 "INFO", message, attachments)
  
      def report_data(self, domain, name, value, expand = True):
          """Report measurable data
  
          When running a testcase, if data is collected that has to be
          reported for later analysis, use this function to report
          it. This will be reported by the report driver in a way that
          makes it easy to collect later on.
  
          Measured data is identified by a *domain* and a *name*, plus
          then the actual value.
  
          A way to picture how this data can look once aggregated is as
          a table per domain, on which each invocation is a row and each
          column will be the values for each name.
  
          :param str domain: to which domain this measurement applies
            (eg: "Latency Benchmark %(type)s");
  
          :param str name: name of the value  (eg: "context switch
            (microseconds)"); it is recommended to always add the unit
            the measurement represents.
  
          :param value: value to report for the given domain and name;
             any type can be reported.
  
          :param bool expand: (optional) by default, the *domain* and
            *name* fields will be %(FIELD)s expanded with the keywords
            of the testcase or target. If *False*, it will not be
            expanded.
  
            This enables to, for example, specify a domain of "Latency
            measurements for target %(type)s" which will automatically
            create a different domain for each type of target.
          """
          assert isinstance(domain, basestring)
          assert isinstance(name, basestring)
          if expand:
              domain = domain % self.kws
              name = name % self.kws
          report.report_c.report(
              2, 1000, 1000, self, "DATA", domain + "::" + name + "::%s" % value,
              dict(domain = domain, name = name, value = value))
  
      def report_tweet(self, what, result, extra_report = "",
                       ignore_nothing = False, attachments = None,
                       level = None, dlevel = 0, alevel = 2, ulevel = 5,
                       dlevel_failed = 0, dlevel_blocked = 0,
                       dlevel_passed = 0, dlevel_skipped = 0, dlevel_error = 0):
          if level == None:		# default args are computed upon def'on
              level = msgid_c.depth()
          self._report_argcheck(what, attachments, level, dlevel, alevel, ulevel)
          level += dlevel
          r = False
          if result.failed > 0:
              tag = "FAIL"
              msg = valid_results[tag][1]
              level += dlevel_failed
          elif result.errors > 0:
              tag = "ERRR"
              msg = valid_results[tag][1]
              level += dlevel_error
          elif result.blocked > 0:
              tag = "BLCK"
              msg = valid_results[tag][1]
              level += dlevel_blocked
          elif result.passed > 0:
              tag = "PASS"
              msg = valid_results[tag][1]
              r = True
              level += dlevel_passed
          elif result.skipped > 0:
              tag = "SKIP"
              msg = valid_results[tag][1]
              level += dlevel_skipped
              r = True
          else:            # When here, nothing was run, all the counts are zero
              if ignore_nothing == True:
                  return True
              report.report_c.report(level, level + alevel, level + ulevel, self,
                                     "BLCK",
                                     what + " / nothing ran " + extra_report,
                                     attachments)
              return False
          report.report_c.report(level, level + alevel, level + ulevel, self,
                                 tag, what + " " + msg + " " + extra_report,
                                 attachments)
          return r
  
  
      def shcmd_local(self, cmd, origin = None, reporter = None,
                      logfile = None):
          """
          Run a shell command in the local machine, substituting
          %(KEYWORD)[sd] with keywords defined by the testcase.
          """
          if origin == None:
              origin = tcfl.origin_get(2)
          return self._shcmd_local(cmd % self.kws, origin = origin,
                                   reporter = reporter, logfile = logfile)
  
  
      @classmethod
      def file_ignore_add_regex(cls, regex, origin = None):
          """
          Add a regex to match a file name to ignore when looking
          for testcase files
  
          :param str regex: Regular expression to match against the
            file name (not path)
          :param str origin: [optional] string describing where this
            regular expression comes from (eg: FILE:LINENO).
          """
          assert isinstance(regex, basestring)
          if origin != None:
              assert isinstance(origin, basestring)
          if origin == None:
              o = inspect.stack()[1]
              origin = "%s:%s" % (o[1], o[2])
          cls._ignore_regexs.append((re.compile(regex), origin))
  
      @classmethod
      def dir_ignore_add_regex(cls, regex, origin = None):
          """
          Add a regex to match a directory name to ignore when looking
          for testcase files
  
          :param str regex: Regular expression to match against the
            directory name (not path)
          :param str origin: [optional] string describing where this
            regular expression comes from (eg: FILE:LINENO).
          """
          assert isinstance(regex, basestring)
          if origin != None:
              assert isinstance(origin, basestring)
          if origin == None:
              o = inspect.stack()[1]
              origin = "%s:%s" % (o[1], o[2])
          cls._ignore_directory_regexs.append((re.compile(regex), origin))
  
      @classmethod
      def driver_add(cls, _cls, origin = None, *args):
          """
          Add a driver to handle test cases (a subclass of :class:tc_c)
  
          A testcase driver is a subclass of :class:`tcfl.tc.tc_c` which
          overrides the methods used to locate testcases and implements the
          different testcase configure/build/evaluation functions.
  
          >>> import tcfl.tc
          >>> class my_tc_driver(tcfl.tc.tc_c)
          >>> tcfl.tc.tc_c.driver_add(my_tc_driver)
  
          :param tcfl.tc.tc_c _cls: testcase driver
          :param str origin: (optional) origin of this call
          """
          assert issubclass(_cls, tc_c)
          if origin == None:
              o = inspect.stack()[1]
              origin = "%s:%s" % (o[1], o[2])
          setattr(_cls, "origin", origin)
          logger.info("%s: Added test case driver %s", origin, _cls)
          tcd_setup = getattr(_cls, "setup", None)
          if tcd_setup != None:	# base class tc_c has no setup()
              _cls.setup(*args)
          cls._tc_drivers.append(_cls)
  
      #: (list of callables) a list of functions to call before starting
      #: execution of each test case instance (right before any phases
      #: are run)
      #:
      #: Usable to do final testcase touch up, adding keywords needed
      #: for the site deployment. etc.
      #:
      #: Note these will be called as methods in the order in the list,
      #: so the first argument will be always be the the testcase
      #: instance.
      #:
      #: E.g.: in a TCF configuration file `.tcf/conf_hook.py` you can
      #: set:
      #:
      #: >>> def _my_hook_fn(tc):
      #: >>>     # Classify testcases based on category:
      #: >>>     # - red
      #: >>>     # - green
      #: >>>     # - blue
      #: >>>     #
      #: >>>     # tc_name keyword has the path of the testcase, which
      #: >>>     # we are using for the sake of example to categorize;
      #: >>>     # keywords can be dumped by running `tcf run
      #: >>>     # /usr/share/examples/test_dump_kws*py.
      #: >>>
      #: >>>     name = tc.kws['tc_name']
      #: >>>     categories = set()
      #: >>>     for category in [ 'red', 'green', 'blue' ]:
      #: >>>         # if test's path has CATEGORY, add it
      #: >>>         if category in name:
      #: >>>             categories.add(category)
      #: >>>     if not categories:
      #: >>>         categories.add('uncategorized')
      #: >>>     tc.kw_set('categories', ",".join(categories))
      #: >>>     tc.log.error("DEBUG categories: %s", ",".join(categories))
      #: >>>
      #: >>> tcfl.tc.tc_c.hook_pre.append(_my_hook_fn)
      #: >>>
      #:
      #: .. warning::
      #:
      #:    - this is a global variable for all testcases of all classes
      #:      and instances assigned to run in different targets
      #:
      #:    - these functions will execute on different threads
      #:      and processes, so **do not use shared data or global
      #:      variables**.
      #:
      #:    - only add to this list from configuration files, never from
      #:      testcases or testcase driver code.
      hook_pre = []
  
      #: (dict) a dictionary to translate target type names, from
      #: *TYPE[:BSP]* to another name to use when reporting as it is
      #: useful/convenient to your application (eg: if what you are
      #: testing prefers other type names); will be only translated if
      #: present. E.g.:
      #:
      #: >>> tcfl.tc_c.type_map = {
      #: >>>     # translate to Zephyr names
      #: >>>     "arduino-101:x86" = "arduino_101",
      #: >>>     "arduino-101:arc" = "arduino_101_ss",
      #: >>> }
      type_map = {}
  
      #: Map exception types to results
      #:
      #: this allows to automaticall map an exception raised
      #: automatically and be converted to a type. Any testcase can
      #: define their own version of this to decide how to convert
      #: exceptions from the default of them being considered blockage
      #: to skip, fail or pass
      #:
      #: >>> class _test(tcfl.tc.tc_c):
      #: >>>     def configure_exceptions(self):
      #: >>>         self.exception_to_result[OSError] = tcfl.tc.error_e
      exception_to_result = {
          AssertionError: blocked_e,
      }
  
      #: How many times do we repeat the evaluation (for stress/MTBF)
      eval_repeat = 1
  
      #: Which evaluation are we currently running (out of :data:`eval_repeat`)
      eval_count = 0
  
      #: List of callables that will be executed when a testcase is
      #: identified; these can modify as needed the testcase (eg:
      #: scanning for tags)
      testcase_patchers = []
  
      runid = None
      runid_visible = ""
  
      #: temporary directory where testcases can drop things; this will
      #: be specific to each testcase instance (testcase and target
      #: group where it runs).
      tmpdir = tempfile.mkdtemp(prefix = "tcf.run" + runid_visible + "-")
  
      #: temporary directory where to store information (serial console,
      #: whatever) that will be captured on each different evaluation;
      #: on each invocation of the evaluation, a new buffer dir will be
      #: allocated and code that captures things from the target will
      #: store captures in there.
      buffers = None
  
      #
      # Protected / private APIs
      #
      # For internal use only!
  
      # Number of testcases we are running (used to calculate how much
      # we wait for targets)
      _tcs_total = 0
  
      #: Number of testcases running on targets
      jobs = 1
  
      _hash_salt = ""
  
      _ignore_regexs = [
          # FIXME: there was a way to get the current file and line to
          # replace builtin
          (re.compile(".*~$"), "builtin"),
          (re.compile(r".*\.txt$"), "builtin"),
          (re.compile(r".*\.[oachS]$"), "builtin"),
          (re.compile(r".*\.asm$"), "builtin"),
          (re.compile(r".*\.so$"), "builtin"),
          (re.compile(r".*\.o\.cmd$"), "builtin"),
          (re.compile(r".*\.cmd$"), "builtin"),
          (re.compile(r".*\.pyc$"), "builtin"),
          (re.compile(r"\.git/"), "builtin"),
      ]
      # List of regular expressions of directory names to ignore
      _ignore_directory_regexs = [
          (re.compile("^outdir(-.*)?$"), "builtin"),
          (re.compile("^.git$"), "builtin"),
      ]
      # In an unlimited test run, we execute on ALL the targets we find; on
      # limited we only run on target of each type
      _mode = False
  
      # Extra report fields when a test case fails
      #
      # Can be set from configuration (extra_report_format) or with the
      # --extra-report-format command line to tcf. Fields are the same
      # as for the markup rules.
      _extra_report_format = None
  
      # Phases we are going to run (configure, build, deploy, eval,
      # clean) Note we use a dictionary where the key is the name of the
      # phase and the value is a list of the places where said decission
      # has been made. Likewise with _phases_skip.
      _phases = collections.defaultdict(set)
      _phases_skip = collections.defaultdict(set)
  
      #: tags that describe this testcase
      _tags = {}
      #: targets on which this testcase wants to run
      # FIXME: rename to _target_wants
      # This needs to be an OrderedDictionary so the same order in which
      # we specified the wanted targets is maintained by default.
      _targets = collections.OrderedDict()
      # Do we run also on targets that have been disabled
      _targets_disabled_too = False
      #: index for target names
      _target_count = 0
      #: interconnects this TC needs to run
      _interconnects = set()
      #: index for interconnect names
      _ic_count = 0
  
      # Do we actually run or pretend to?
      _dry_run = False
  
      # Where we cache all the remote target info
      rt_all = None
  
      #
      # Support to call the testcase methods
      # (configure|build|deploy|eval|clean)*()
      #
      _configure_serially = False
      _configure_serial = []
      _configure_parallel = []
  
      _build_serially = False
      _build_serial = []
      _build_parallel = []
  
      _deploy_serially = False
      _deploy_serial = []
      _deploy_parallel = []
  
      _setup_serial = []
      _start_serial = []
      _eval_serial = []
      _test_serial = []
      _teardown_serial = []
      _class_teardown_serial = []
  
      _clean_serially = False
      _clean_serial = []
      _clean_parallel = []
  
      # List of available testcase drivers
      _tc_drivers = []
  
      # Will targets be released at the end of the testcase
      release = True
  
      __metaclass__ = _tc_mc
  
      #
      # App Builder entry points; when you define an app_NAME builder in
      # the @target() decorator, _target_app_setup() creates
      # CLASSNAME.(configure|build|deploy|clean)_for_TARGET() calls for
      # you that call these functions.
      #
      # Internally, these are discovered and called by _methods_run /
      # _method_trampoline_call as they have been aliased via a
      # setattr() to a CLASNAME.FUNCTION_for_TARGETNAME() by the
      # @target() decorator via _target_app_setup().
      def _configure_50_for_target(self, _target):
          for bsp in _target.bsps:
              # FIXME: use msgid_c here
              _target.bsp_set(bsp)
              _target.report_info("configuring BSP %s" % bsp, dlevel = 3)
              _app, app_src = _target._app_get_for_bsp("configure")
              app.configure(_app, self, _target, app_src)
          for bsp, (_app, app_src, _) \
              in _target.bsps_stub.iteritems():
              if _app == None:
                  raise blocked_e("%s: BSP has to be stubbed, but no "
                                  "stubbing information was provided" % bsp)
              # FIXME: use msgid_c here
              _target.bsp_set(bsp)
              _target.report_info("configuring stub-BSP %s" % bsp, dlevel = 3)
              # FIXME: append app_src_options
              app.configure(_app, self, _target, app_src)
          if _target.bsps:
              _target.bsp_set()
  
      # same as _configure_50_for_target()
      def _build_50_for_target(self, _target):
          for bsp in _target.bsps:
              # FIXME: use msgid_c here
              _target.bsp_set(bsp)
              _target.report_info("building BSP %s" % bsp, dlevel = 3)
              _app, app_src = _target._app_get_for_bsp("build")
              app.build(_app, self, _target, app_src)
          for bsp, (_app, app_src, _) \
              in _target.bsps_stub.iteritems():
              if _app == None:
                  raise blocked_e("%s: BSP has to be stubbed, but no "
                                  "stubbing information was provided" % bsp)
              # FIXME: use msgid_c here
              _target.bsp_set(bsp)
              _target.report_info("building stub-BSP %s" % bsp, dlevel = 3)
              app.build(_app, self, _target, app_src)
          if _target.bsps:
              _target.bsp_set()
  
      # same as _configure_50_for_target()
      # Deployment methods can do all the work themselves or append a
      # list of images to the images set to be uploaded to the target
      # using the images interface.
      def _deploy_50_for_target(self, _target):
          images = set()
          result = result_c(0, 0, 0, 0, 0)
          for bsp in _target.bsps:
              # FIXME: use msgid_c here
              _target.bsp_set(bsp)
              _target.report_info("deploying BSP %s" % bsp, dlevel = 3)
              _app, app_src = _target._app_get_for_bsp("deploy")
              r = app.deploy(images, _app, self, _target, app_src)
              assert isinstance(r, result_c)
              result += r
          for bsp, (_app, app_src, _) \
              in _target.bsps_stub.iteritems():
              if _app == None:
                  raise blocked_e("%s: BSP has to be stubbed, but no "
                                  "stubbing information was provided" % bsp)
              # FIXME: use msgid_c here
              _target.bsp_set(bsp)
              _target.report_info("deploying stub-BSP %s" % bsp,
                                  dlevel = 3)
              r = app.deploy(images, _app, self, _target, app_src)
              assert isinstance(r, result_c)
              result += r
          if _target.bsps:
              _target.bsp_set()
  
          if images:
              if getattr(_target, "images", None) != None:
                  result = _target.images.upload_set(images)
              else:
                  result = result_c(0, 0, 0, 1, 0)
                  _target.report_info("Images collected by deploy phase, but "
                                      "no image upload interfaces available",
                                      dlevel = -1)
          return result
  
      # same as _configure_50_for_target()
      def _setup_50_for_target(self, _target):
          for bsp in _target.bsps:
              # FIXME: use msgid_c here
              _target.bsp_set(bsp)
              _target.report_info("setting up BSP %s" % bsp, dlevel = 3)
              _app, app_src = _target._app_get_for_bsp("setup")
              app.setup(_app, self, _target, app_src)
          # Stubs are assumed to need no setup
          if _target.bsps:
              _target.bsp_set()
  
      # same as _configure_50_for_target()
      def _start_50_for_target(self, _target):
          for bsp in _target.bsps:
              # FIXME: use msgid_c here
              _target.bsp_set(bsp)
              _target.report_info("starting BSP %s" % bsp, dlevel = 3)
              _app, app_src = _target._app_get_for_bsp("start")
              app.start(_app, self, _target, app_src)
          # Stubs are assumed to need no starting
          if _target.bsps:
              _target.bsp_set()
  
      # same as _configure_50_for_target()
      def _teardown_50_for_target(self, _target):
          for bsp in _target.bsps:
              # FIXME: use msgid_c here
              _target.bsp_set(bsp)
              _target.report_info("tearing down BSP %s" % bsp, dlevel = 3)
              _app, app_src = _target._app_get_for_bsp("teardown")
              app.teardown(_app, self, _target, app_src)
          # Stubs are assumed to need no teardown
          if _target.bsps:
              _target.bsp_set()
  
      # same as _configure_50_for_target()
      def _clean_50_for_target(self, _target):
          for bsp in _target.bsps:
              _target.bsp_set(bsp)
              _target.report_info("cleaning BSP %s" % bsp, dlevel = 3)
              _app, app_src = _target._app_get_for_bsp("clean")
              app.clean(_app, self, _target, app_src)
          for bsp, (_app, app_src, _) \
              in _target.bsps_stub.iteritems():
              if _app == None:
                  raise blocked_e("%s: BSP has to be stubbed, but no "
                                  "stubbing information was provided" % bsp)
              _target.bsp_set(bsp)
              _target.report_info("cleaning stub-BSP %s" % bsp, dlevel = 3)
              app.clean(_app, self, _target, app_src)
          if _target.bsps:
              _target.bsp_set()
  
      #
      # Linkage into the report API and support for it
      #
  
      @staticmethod
      def _report_argcheck(message, attachments, level, dlevel, alevel, ulevel):
          assert isinstance(message, basestring)
          if attachments:
              assert isinstance(attachments, dict)
          assert isinstance(level, int)
          assert isinstance(dlevel, int)
          assert isinstance(alevel, int)
          assert isinstance(ulevel, int)
  
      def _report_mk_prefix(self):
          """
          Update the prefix we use for the logging/reports when some
          parameter changes.
          """
          if self.target_group == None:
              self._report_prefix = self.name + " @local"
          elif self.target_group.len() == 1:
              self._report_prefix = \
                  self.name \
                  + " @" \
                  + self.target_group.targets.values()[0].fullid
          else:
              self._report_prefix = self.name + " @" + self.target_group.name
  
      def report_mk_prefix(self):
          """
          Update the prefix we use for the logging/reports when some
          parameter changes.
          """
          return self._report_prefix
  
      #
      # Method handling code
      #
      # these _method* methods are the infrastructure that helps locate
      # and call the methods
      # configure|build|deploy|setup|start|eval|clean* defined by the
      # user or dynamically added by App builders.
      #
      @staticmethod
      def _valid_id(ident):
          # Credit:
          # http://stackoverflow.com/questions/12700893/how-to-check-if-a-string-is-a-valid-python-identifier-including-keyword-check/29586366#29586366
          # Licensing: cc by-sa 3.0
          """Determines, if string is valid Python identifier."""
  
          # Smoke test - if it's not string, then it's not identifier,
          # but we don't want to just silence exception. It's better to
          # fail fast.
          if not isinstance(ident, str):
              raise TypeError('expected str, but got {!r}'.format(type(ident)))
  
          # Resulting AST of simple identifier is <Module [<Expr <Name "foo">>]>
          try:
              root = ast.parse(ident)
          except SyntaxError:
              return False
  
          if not isinstance(root, ast.Module):
              return False
  
          if len(root.body) != 1:
              return False
  
          if not isinstance(root.body[0], ast.Expr):
              return False
  
          if not isinstance(root.body[0].value, ast.Name):
              return False
  
          if root.body[0].value.id != ident:
              return False
  
          return True
  
      def __method_prepare(self, fname, fn, do_serially, serial, parallel):
          """
          Prepare a single method for later execution with __methods_call()
  
          Collects the information
          """
          # Evaluate a method from the class and decide if it goes in
          # the list of things that have to be run serially or in the
          # ones that can be parallelize.
          # Verifies that methods that declare variables match targets
          # that have beend declared.
          cls = type(self)
          argnames = list(fn.__code__.co_varnames[:fn.__code__.co_argcount])
          if inspect.ismethod(fn):
              if fn.__self__ is cls:
                  _type = False
              else:
                  _type = True
              try:
                  argnames.pop(0)
              except IndexError:
                  raise blocked_e(
                      "%s.%s(): not a valid method (missing 'self'?), "
                      "@classmethod or @staticmethod @%s"
                      % (cls.__name__, fname, tcfl.origin_get_object(fn)))
          elif isinstance(fn, types.UnboundMethodType) == False:
              _type = False
          else:
              raise blocked_e(
                  "%s.%s() (%s): can't figure out function type"
                  % (cls.__name__, fname, tcfl.origin_get_object(fn)))
  
          # Any arguments left MUST be names of targets that have been
          # declared with the @target decorator to the class
          for argname in argnames:
              if not self._valid_id(argname):
                  raise blocked_e(
                      "%s.%s() (%s): argument '%s' is not a "
                      "valid Python identifier"
                      % (cls.__name__, fname,
                         tcfl.origin_get_object(fn), argname))
              if not argname in self._targets.keys():
                  raise blocked_e(
                      "%s @ %s.%s() %s: needs target named '%s', which hasn't "
                      "been declared with the @target class decorator "
                      "(available: %s)"
                      % (self.name, cls.__name__, fname, cls._origin_fn(fn),
                         argname,
                         " ".join([i for i in sorted(self._targets.keys())])))
  
          # what kind of execution can we do for this method? serial or
          # parallel?
          if len(argnames) == 0:
              # not target specification, so it might use all
              # of them--force serial
              execution_mode = 'serial'
          elif len(argnames) == 1:
              # methods that use only one target can be parallelized
              execution_mode = 'parallel'
          else:
              # methods that use more than one target have to be serial
              execution_mode = 'serial'
  
          # our guess is overriden by the function's decoration with
          # :func:`tcfl.tc.serially` or :func:`tcfl.tc.concurrently`
          decorated_execution_mode = getattr(fn, "execution_mode", None)
          if decorated_execution_mode:
              # overriden by the user
              execution_mode = decorated_execution_mode
  
          # the final override comes from...taxaaaan an argument to this
          # function overriding it all?
          if do_serially:
              execution_mode = 'serial'
  
          # Do not transform into a set, we want to keep the order
          if len(argnames) == 0:
              argnames = None
          if execution_mode == 'serial':
              serial.append((fname, fn, _type, argnames))
          elif execution_mode == 'parallel':
              parallel.append((fname, fn, _type, argnames))
          else:
              raise AssertionError("unknown execution mode '%s'"
                                   % execution_mode)
  
      @classmethod
      def _origin_fn(cls, fn):
          # Find the origin of a function, being mindful that if it was
          # generated by an App builder, we need to look for the alias
          #
          try:
              return tcfl.origin_get_object(fn)
          except IOError as e:
              fname = fn.__name__
              if e.message == 'source code not available':
                  if fname.startswith("__" + cls.__name__) \
                     and any("_%s_for_" % phase in fname \
                             for phase in [ "configure", "build",
                                            "deploy", "clean"]):
                      return "internal app builder"
                  return "origin n/a"
              else:
                  raise
  
      def _methods_prepare(self):
          """
          Scan for methods in the testcase that can be used for
          running the diffferent phases.
  
          Store their information in a way that will be easily
          accessible later by __method_call().
          """
          cls = type(self)
          # Note we need to make sure we create NEW instances of all the
          # class-specific variables that are specific to cls (and not
          # modify the ones we inherit from the base class)
          self._configure_serial = \
              copy.deepcopy(super(cls, cls)._configure_serial)
          self._configure_parallel = \
              copy.deepcopy(super(cls, cls)._configure_parallel)
  
          self._build_serial = copy.deepcopy(super(cls, cls)._build_serial)
          self._build_parallel = copy.deepcopy(super(cls, cls)._build_parallel)
  
          self._deploy_serial = copy.deepcopy(super(cls, cls)._deploy_serial)
          self._deploy_parallel = copy.deepcopy(super(cls, cls)._deploy_parallel)
  
          self._setup_serial = copy.deepcopy(super(cls, cls)._setup_serial)
          self._start_serial = copy.deepcopy(super(cls, cls)._start_serial)
          self._eval_serial = copy.deepcopy(super(cls, cls)._eval_serial)
          self._test_serial = copy.deepcopy(super(cls, cls)._test_serial)
          self._teardown_serial = copy.deepcopy(super(cls, cls)._teardown_serial)
          # Class teardown inherits the class we inherit from too
          self._class_teardown_serial = copy.deepcopy(
              super(cls, cls)._class_teardown_serial)
  
          self._clean_serial = copy.deepcopy(super(cls, cls)._clean_serial)
          self._clean_parallel = copy.deepcopy(super(cls, cls)._clean_parallel)
  
          # We might have added methods later to the
          # instance, not to the class, so inspect the object
          for fname, fn in sorted(inspect.getmembers(self),
                                  key = lambda x: x[0]):
              if not callable(fn):
                  continue
              if fname.startswith("configure"):
                  self.__method_prepare(fname, fn, self._configure_serially,
                                        self._configure_serial,
                                        self._configure_parallel)
              elif fname.startswith("build"):
                  self.__method_prepare(fname, fn, self._build_serially,
                                        self._build_serial, self._build_parallel)
              elif fname.startswith("deploy"):
                  self.__method_prepare(
                      fname, fn, self._deploy_serially,
                      self._deploy_serial, self._deploy_parallel)
              elif fname.startswith("setup"):
                  self.__method_prepare(fname, fn, True,
                                        self._setup_serial, None)
              elif fname.startswith("start"):
                  self.__method_prepare(fname, fn, True,
                                        self._start_serial, None)
              elif fname.startswith("eval"):
                  self.__method_prepare(fname, fn, True, self._eval_serial, None)
              elif fname.startswith("test"):
                  self.__method_prepare(fname, fn, True, self._test_serial, None)
              elif fname.startswith("teardown"):
                  self.__method_prepare(fname, fn, True,
                                        self._teardown_serial, None)
              elif fname.startswith("class_teardown"):
                  self.__method_prepare(fname, fn, True,
                                        self._class_teardown_serial, None)
              elif fname.startswith("clean"):
                  self.__method_prepare(fname, fn, self._clean_serially,
                                        self._clean_serial, self._clean_parallel)
              else:
                  continue
          # Now that the (configure|build|deploy|clean)*() methods are
          # classified, we need to check that the ones that are to be
          # run in parallel don't have conflicting arguments (we can't
          # do more than one operation in the same target at the same
          # time). For that we collect all the parallels argument's list
          # and make sets out of that have to be disjuntive
          for method_list in [self._configure_parallel, self._build_parallel,
                              self._deploy_parallel, self._clean_parallel]:
              for fname, fn, _type, argnames in method_list:
                  for fname2, fn2, _type2, argnames2 in method_list:
                      if fn == fn2:
                          continue
                      common_argnames = set(argnames) & set(argnames2)
                      decorated_execution_mode1 = getattr(fn, 'execution_mode', None)
                      decorated_execution_mode2 = getattr(fn2, 'execution_mode', None)
                      if decorated_execution_mode1 == 'parallel' \
                         and decorated_execution_mode2 == 'parallel':
                          # do not bother checking, as both has been
                          # decorated as 'have to execute concurrently'
                          # even if they share targets.
                          continue
                      if common_argnames:
                          raise blocked_e(               # FIXME: add origin
                              "%s.%s() (%s) and %s.%s() (%s) conflict in using "
                              "target/s '%s' at the same time (and thus they "
                              "cannot be run in parallel. "
                              "You have to focus actions for a target in "
                              "a single method or run them serially by "
                              "prefixing a @tcfl.tc.serially() decorator."
                              # FIXME: offer fixing methodologies
                              % (cls.__name__, fname,
                                 cls._origin_fn(fn),
                                 cls.__name__, fname2,
                                 cls._origin_fn(fn2),
                                 ", ".join(sorted(list(common_argnames)))))
          # And finally, sort the methods alphabetically (the ones that
          # will be run serially and also the parallel in case config
          # says we have to run them serially none-the-less)
          for method_list in [
                  self._configure_serial, self._configure_parallel,
                  self._build_serial, self._build_parallel,
                  self._deploy_serial, self._deploy_parallel,
                  self._clean_serial, self._clean_parallel]:
              method_list.sort(key = lambda i: i[0])
  
  
      def _mk_target_args_for_fn(self, fn, args):
          """
          Given a list of arguments for a method, translate it into the
          target instantations so it can be called
  
          When a method declaration requires in its argument list the
          names of the targets it wants (from the testcase's list of
          declared targets), this will pull the actual insances of the
          targets so they can be passed when calling the method.
  
          :param func fn: function to call
          :param tuple args: list of argument names
          :returns list: list of target_c instances matching each target name
          """
          if args == None:
              return []
          _args = []
          for wanted_target in args:
              if wanted_target in self.target_group.targets:
                  _args.append(self.target_group.target(wanted_target))
              else:
                  raise blocked_e(
                      "BUG? testcase function `%s.%s` (%s) wants an "
                      "interconnect or target '%s' not present in target "
                      "group `%s` (available: %s)"
                      % (type(self).__name__, fn.__name__,
                         tcfl.origin_get_object(fn), wanted_target,
                         self.target_group.name,
                         ", ".join([
                             i for i in self.target_group.targets.keys()
                         ])))
          return _args
  
      def __method_trampoline_call(self, fname, fn, _type, targets):
          # runs a class function and return a return_c
          #
          # convers the return value to return_c from bool, or None, or
          # passes a return_c. Passes exceptions.
          #
          # We use fname instead of fn.__name__ because if we have made
          # an alias out of the method in the class (with setattr(), we
          # want to see the alias name, not the original function name).
          if fname == fn.__name__:
              self.report_info("running %s.%s()"
                               % (type(self).__name__, fn.__name__),
                               dlevel = 3)
          else:
              self.report_info("running %s.%s() [App builder's alias for "
                               "%s() which calls tc_c._*_50_for_target()]"
                               % (type(self).__name__, fname, fn.__name__),
                               dlevel = 3)
          if type(fn) == types.MethodType:	# instance needed
              if fn.im_self == None:
                  # This was added from target_wanted_add(), binding to an
                  # specific object; we forced it unbound, so we have to
                  # call it with 'self'
                  r = getattr(self, fname)(self, *targets)
              else:
                  # This was added from the @target() decorator, binding
                  # to a class, it doesn't need a self as Python will
                  # add it for us, current self.
                  r = getattr(self, fname)(*targets)
          else:	# static/classmethod
              r = fn(*targets)
  
          # Now, let's see what did it return
          if isinstance(r, result_c):
              return r
          elif r == None or r == True:
              return result_c(1, 0, 0, 0, 0)
          elif r == False:
              return result_c(0, 1, 0, 0, 0)
          else:
              raise blocked_e(
                  "%s.%s(): don't know what to do with "
                  "return value of type %s; return nothing or "
                  "True for success, False for failure, raise "
                  "any exception for blockage, or "
                  "tcfl.{passed|blocked|error|failed|skip}_e "
                  % (type(self).__name__, fn.__name__,
                     type(r).__name))
  
      def __method_trampoline_thread(self, msgid, fname, fn, _type, targets,
                                     thread_init_args):
          # runs a function and returns a tuple (result, exception
          # info). If there was no exception, [1] will be None;
          # otherwise it will contain the exception information so it
          # can be re-raised when called from a thread.
          #
          # Because we are in the context of a newly initialized thread,
          # we need to initialize some things
          if thread_init_args == None:
              thread_init_args = {}
          else:
              assert isinstance(thread_init_args, dict)
          try:
              with msgid_c(parent = msgid, l = 2):
                  self.__thread_init__(**thread_init_args)
                  return (
                      self.__method_trampoline_call(fname, fn, _type, targets),
                      None)
          except:
              return (None, sys.exc_info())
  
      @result_c.from_exception
      def _method_run(self, fname, fn, _type, args):
          """
          Implement the guts of _methods_call()
  
          Runs and aggregates results of serial methods, then launches a
          threadpool for the parallel methods; aggregates their results
          and raise exception if any happened.
  
          :returns result_c: aggregated result of all the methods.
          """
          targets = self._mk_target_args_for_fn(fn, args)
          return self.__method_trampoline_call(fname, fn, _type, targets)
  
      def _methods_run(self, do_serially, serial_list, parallel_list):
          """
          Implement the guts of _methods_call()
  
          Runs and aggregates results of serial methods, then launches a
          threadpool for the parallel methods; aggregates their results
          and raise exception if any happened.
  
          :returns result_c: aggregated result of all the methods.
          """
          result = result_c(0, 0, 0, 0, 0)
          if do_serially:
              for fname, fn, _type, args in sorted(serial_list + parallel_list):
                  retval = self._method_run(fname, fn, _type, args)
                  result += retval
                  if retval.errors or retval.failed \
                     or retval.blocked or retval.skipped:
                      return result
          else:
              for fname, fn, _type, args in serial_list:
                  retval = self._method_run(fname, fn, _type, args)
                  result += retval
                  if retval.errors or retval.failed \
                     or retval.blocked or retval.skipped:
                      return result
              # FIXME: set from from config
              # All these are supposedly I/O bound jobs as either they
              # are chitchatting with the network or spawning jobs to do
              # stuff
              thread_pool = _multiprocessing_method_pool_c(processes = 10)
              threads = {}
              for fname, fn, _type, args in parallel_list:
                  targets = self._mk_target_args_for_fn(fn, args)
                  threads[fname] = thread_pool.apply_async(
                      self.__method_trampoline_thread,
                      (msgid_c(l = 2), fname, fn, _type, targets,
                       dict(expecter_parent = self.tls.expecter)))
              thread_pool.close()
              thread_pool.join()
              for thread in threads.values():
                  r = thread.get()
                  if r[1] != None:	# re-raise thrown exceptions
                      raise r[1][0], r[1][1], r[1][2]
                  result += r[0]
              del thread_pool
          return result
  
  
      @result_c.from_exception
      def _methods_call(self, base):
          """
          Call all the methods of a class' instance whose name starts with
          @base.
  
          Methods who take no arguments (other than classmethods or
          instance method's cls and self) are called serially in
          alphabetical order.
  
          Methods that take targets as arguments are called in
          parallel (unless the corresponding switch that forces
          serialization is flipped).
  
          No two methods that take targets might share a target, an
          exception will be raised.
  
          :returns result_c: aggregated result of all method calls.
          :raises: whatever exception any method raised is passed up
          """
          if base == "configure":
              return self._methods_run(self._configure_serially,
                                       self._configure_serial,
                                       self._configure_parallel)
          elif base == "build":
              return self._methods_run(self._build_serially,
                                       self._build_serial, self._build_parallel)
          elif base == "deploy":
              return self._methods_run(
                  self._deploy_serially,
                  self._deploy_serial, self._deploy_parallel)
          elif base == "setup":
              return self._methods_run(True, self._setup_serial, [])
          elif base == "start":
              return self._methods_run(True, self._start_serial, [])
          elif base == "eval":
              return self._methods_run(True, self._eval_serial, [])
          elif base == "test":
              return self._methods_run(True, self._test_serial, [])
          elif base == "teardown":
              return self._methods_run(True, self._teardown_serial, [])
          elif base == "class_teardown":
              return self._methods_run(True, self._class_teardown_serial, [])
          elif base == "clean":
              return self._methods_run(self._clean_serially,
                                       self._clean_serial, self._clean_parallel)
          else:
              assert False, "Unknown base '%s'" % base
  
  
      #
      # Helpers for the public API
      #
      def _shcmd_local(self, cmd, origin = None, reporter = None,
                       logfile = None, nonzero_e = error_e):
          """
          Run a single shell command
  
          :param tc_c _tc: test case
          :param str cmd: shell command to run
          :param str origin: where the shell command was found
          :return: True if successful, False if failed, None if blocked
  
          """
          if reporter == None:
              reporter = self
          if origin == None:
              origin = tcfl.origin_get(2)
          # This is safe as it runs in the client, as the current user
          # FIXME: we might want to jail it so rogue test cases don't
          # run silly things...
          phase = msgid_c.phase()
          if phase != "":
              output_tag = "%s output" % phase
              ex_trace = "%s trace" % phase
          else:
              output_tag = "output"
              ex_trace = "trace"
          # Make the tag file-name approved -- FIXME: do this in logfiel_open
          if phase:
              _phase = "-" + phase.translate(string.maketrans("/.", "__"))
              phase += " "
          else:
              _phase = ""
  
          # delete = False, needed so we can open it from other
          # places to report
          if logfile == None:
              logf = commonl.logfile_open(
                  tag = _phase + "stdouterr",
                  delete = False, directory = self.tmpdir)
          else:
              logf = logfile
          try:
              p = subprocess.Popen([ cmd ], shell = True, close_fds = False,
                                   stdout = logf, stderr = subprocess.STDOUT)
              rc = p.wait()
              del p
              logf.flush()
              logf.seek(0,0)
  
              if rc == 0:
                  reporter.report_pass("%spassed: '%s' @%s"
                                       % (phase, cmd, origin),
                                       { output_tag: logf })
                  logf.seek(0,0)
                  return logf.read()
              elif rc == 127:
                  raise blocked_e("exit code %d from '%s' @%s" \
                                  % (rc, cmd, origin),
                                  { output_tag: logf, "alevel": 0})
              else:
                  raise nonzero_e("exit code %d from '%s' @%s" \
                                  % (rc, cmd, origin),
                                  { output_tag: logf, "alevel": 0})
          except (failed_e, error_e, blocked_e):
              raise
          except Exception as e:
              raise blocked_e(
                  "exception ('%s' from %s): %s %s" % (cmd, origin, type(e), e),
                  { ex_trace: "".join(traceback.format_tb(sys.exc_info()[2])) })
          finally:
              if logfile == None:
                  del logf
  
  
      @property
      def target_group(self):
          return self._target_group
  
      @target_group.setter
      def target_group(self, target_group):
          if target_group != None:
              assert isinstance(target_group, target_group_c)
          self._target_group = target_group
          servers = set()		# We don't care about reps in servers
          if target_group:
              self.targets = target_group.targets
              # Generate a string with the target types to place in
              # the test keywords so it can be reported
              if target_group.targets:
                  target_types = []	# Here we want one per target
                  # Note this is sorted by target-want-name, the names
                  # assigned by the testcase to the targets, so all the
                  # types and server lists are sorted by that.
                  target_names = []
                  for tgname in sorted(target_group.targets.keys()):
                      _target = target_group.targets[tgname]
                      servers.add(_target.rtb.aka)
                      if len(_target.rt.get('bsp_models', {})) > 1:
                          target_type = _target.type + ":" + _target.bsp_model
                      else:
                          target_type = _target.type
                      target_names.append(_target.fullid
                                          + _target.bsp_model_suffix()
                                          + _target.bsp_suffix())
                      target_type = self.type_map.get(target_type, target_type)
                      target_types.append(target_type)
                  target_types = ",".join(target_types)
              else:
                  target_type = self.type_map.get('static', 'static')
                  target_types = target_type
                  target_names = [ ]
              target_group_targets = ",".join(target_names)
              target_group_name = target_group.name
              target_group_info = target_group.descr
          else:
              self.targets = {}
              target_group_targets = 'static'
              target_group_name = 'static'
              target_group_info = 'static'
              target_types = "static"
  
          self.kw_set('target_group_targets', target_group_targets )
          self.kw_set('target_group_info', target_group_info)
          self.kw_set('target_group_name', target_group_name)
          self.kw_set("type", target_types)
          self.kw_set("target_group_types", target_types)
          self.kw_set('target_group_servers', ",".join(servers))
          self._report_mk_prefix()
          return target_group
  
      def _extra_report(self, kws):
          if self._extra_report_format != None:
              while True:
                  try:
                      return " " + self._extra_report_format % kws
                  except KeyError as e:
                      # If the key is not available, n/a it -- this can
                      # happen when we have a generic message for a
                      # dynamic test from (let's say) the target or that
                      # is not available in every target; or a static vs
                      # dynamic test, in which there are keys for static
                      # that are not there for dynamic
                      # e.message is the key that we failed to find
                      kws[e.message] = "n/a"
                      continue
                  except Exception as e:
                      self.log.error("extra-report error: %s", e)
                      return " {bad extra-report format}"
          else:
              return ""
  
      def _tags_update(self, tags = None):
          if not tags:
              tags = []
          # Tag/s were updated, see if there are any special ones we
          # need to handle
          if not tags:
              tags = self._tags.keys()
          for tagname in tags:
              value, origin = self._tags[tagname]
              if tagname == 'build_only' and value == True:
                  self.build_only.append('tag:' + origin)
  
      def tag_set(self, tagname, value = None, origin = None):
          """
          Set a testcase tag.
  
          :param str tagname: name of the tag (string)
          :param value: (optional) value for the tag; can be a string,
            bool; if none specified, it is set to True
          :param str origin: (optional) origin of the tag; defaults to
            the calling function
  
          Note that there are a few tags that have speciall conventions:
  
          - *component/COMPONENTNAME* is a tag with value
            *COMPONENTNAME* and it is used to classify the testcases by
            component. Multiple tags like this might exist if the
            testcase belongs to multiple components. Note it should be a
            single word.
  
            TCF will create a tag *components* with value *COMPONENTNAME1
            COMPONENTNAME2 ...* (space separated list of components)
            which shall match the *component/COMPONENTx*
            *name* contains the name of the testcase after testcase
            instantiation.
          """
  
          assert isinstance(tagname, basestring), (
              "tagname has to be a string, not a %s" % type(tagname).__name__)
          if value == None:
              value = True
          else:
              assert isinstance(value, basestring) \
                  or isinstance(value, bool)
          if origin == None:
              origin = "[builtin default] " + tcfl.origin_get(1)
          else:
              assert isinstance(origin, basestring)
          self._tags[tagname] = (value, origin)
          self._tags_update([ tagname ])
  
      def tags_set(self, _tags, origin = None, overwrite = True):
          """
          Set multiple testcase tags.
  
          :param dict tags: dictionary of tags and values
          :param str origin: (optional) origin of the tag; defaults to
            the calling function
  
          Same notes as for :meth:`tag_set` apply
          """
  
          for tagname, (value, _origin) in _tags.iteritems():
              assert isinstance(tagname, basestring), \
                  "tagname has to be a string, not a %s" \
                  % type(tagname).__name__
              if value == None:
                  value = True
              else:
                  assert isinstance(value, (basestring, bool)), \
                      "tag value has to be None (taken as True), bool, " \
                      "string, not a %s" % type(value).__name__
              if origin:
                  _origin = origin
              if _origin == None:
                  _origin = "[builtin default] " + tcfl.origin_get(1)
              if tagname in self._tags and overwrite == False:
                  continue
              self._tags[tagname] = (value, _origin)
          self._tags_update(_tags.keys())
  
      def kw_set(self, key, value, origin = None):
          """
          Set a testcase's keyword and value
  
          :param str key: keyword name
          :param str value: value
          :param str origin: (optional) where this comes from
          """
          self._kw_set(key, value, origin)
  
      def kw_unset(self, kw):
          """
          Unset a string keyword for later substitution in commands
  
          :param str kw: keyword name
          """
          assert isinstance(kw, basestring)
          if kw in self.kws:
              del self.kws[kw]
          if kw in self.kws_origin:
              del self.kws_origin[kw]
  
      def kws_set(self, d, origin = None):
          """
          Set a bunch of testcase's keywords and values
  
          :param dict d: A dictionary of keywords and values
          """
          assert isinstance(d, dict)
          if origin == None:
              o = inspect.stack()[1]
              origin = "%s:%s" % (o[1], o[2])
          else:
              assert isinstance(origin, basestring)
          for key, value in d.iteritems():
              self._kw_set(key, value, origin)
      #
      # Helpers for private APIs
      #
  
  
      def tag_get(self, tagname, value_default, origin_default = None):
          """
          Return a tuple (value, origin) with the value of the tag and
          where it was defined.
          """
  
          if origin_default == None:
              origin_default = "[builtin default] " + tcfl.origin_get(1)
          return self._tags.get(tagname, (value_default, origin_default))
  
      def _prefix_update(self):
          """
          Update the prefix we use for the logging/reports when some
          parameter changes.
          """
          self._report_mk_prefix()
          self.log.prefix = " " + self.report_mk_prefix()
          # Replace the logger's format based on the prefix
          self.log.id = id(self)
  
      def _phase_skip(self, phase):
          """
          Determine if a phase has to be skipped
  
          :returns bool: True if @phase has to be skipped, False otherwise.
          """
          retval = False
          if not phase in self._phases:
              self.report_info("phase %s not in [%s]"
                               % (phase, ', '.join(self._phases.keys())),
                               dlevel = 5)
              retval = True
          if phase in self._phases_skip:
              self.report_info("phase %s skipped due to [%s]" %
                               (phase, ", ".join(self._phases_skip[phase])),
                               dlevel = 5)
              retval = True
          return retval
  
      def _kw_set(self, kw, value, origin = None):
          """
          Set a string keyword for later substitution in commands
  
          :param str kw: keyword name
          :param str value: value for the keyword
          :param str origin: origin of this setting; if none, it will be
            taken from the stack
          """
          assert isinstance(kw, basestring)
          assert isinstance(value, basestring)
          if origin == None:
              o = inspect.stack()[1]
              origin = "%s:%s" % (o[1], o[2])
          else:
              assert isinstance(origin, basestring)
          self.kws[kw] = value
          self.kws_origin.setdefault(kw, []).append(origin)
  
  
      def _expecter_log(self, msg, **attachments):
          # FIXME: move this to report to the target it is expecting from
          self.report_info(msg, **attachments)
  
      def _components_fixup(self):
          # Based on tags called component/COMPONENTNAME define a tag
          # called "components" with the list of component names for
          # this testcase
          componentl = []
          for tag_name in self._tags:
              if tag_name.startswith('component/'):
                  componentl.append(tag_name[len('component/'):])
          self.tag_set("components", " ".join(componentl), commonl.origin_get(1))
  
      #
      # Remote target acquisition
      #
  
      _busy_regex = re.compile(r"tried to use busy target "
                               r"\(owned by '(?P<owner>[^']+)'\)")
  
      @classmethod
      def _busy_msg_get_owner(cls, msg):
          m = cls._busy_regex.search(msg)
          if m:
              owner = m.groupdict()['owner']
              if owner == "":
                  owner = "n/a"
          else:
              owner = "n/a"
          return owner
  
      # When a target is released, this event is set so anyone who is
      # waiting can check if the target they were waiting for was
      # released.
      # LAME, need a better way for all this, see comments in _targets_assign()
      target_event = threading.Event()
  
      def _target_acquire_robust(self, _target, timeout, wait):
          # calls shielding from some temporary conditions (eg: doing a
          # few simple retries) related to a server restaring or network
          # glitches
          #
          # Need this soft retry loop here as we want to do it before we
          # attempt to release any target we have already acquired
          t0 = time.time()
          t = t0
          while t - t0 < timeout:
              try:
                  _target.rtb.rest_tb_target_acquire(
                      _target.rt, ticket = self.ticket)
                  break
              except Exception as e:
                  # If read timesout or the server power cycles
                  # in between/resets, treat it as a soft retry
                  t = time.time()
                  if t - t0 > timeout:
                      _target.report_info("retry timeout (%.2fs) exceeded"
                                          % timeout)
                      raise
                  message = str(e.message)	# easier matching for stuff
                  if "Read timed out" in message \
                     or 'ECONNRESET' in message \
                     or 'Connection refused' in message:
                      _target.report_info(
                          "acquisition failed (connection condition) "
                          "@%.2f/%.2fs, waiting for %.2fs and retrying"
                          % (t - t0, timeout, wait), dlevel = 4)
                      time.sleep(wait)
                      continue
                  raise
  
      # FIXME: add phase "assign"
      @contextlib.contextmanager
      def _targets_assign(self, timeout = 1000, period = assign_period):
          if self.is_static():
              yield
              return
  
          acquired = []
          pending = []
          for target in self.target_group.targets.values():
              if target.acquire:
                  pending.append(target)
          if not pending:
              yield
              return
          # FIXME: lame method to avoid deadlocks; always
          #        lock/unlock in the same alphabetical order, by fullid
          pending.sort(key = lambda _target: _target.fullid)
  
          # If any of the targets declare it takes it a long time to
          # power up, consider that.
          target_max_power_up_time = max(
              pending, key = lambda target: target.rt.get('power_up_time', 0))
          timeout += target_max_power_up_time.rt.get('power_up_time', 0) * 5
  
          # Increase the timeout based on how many testcases are waiting
          # for it from us FIXME this can't take into account other
          # clients or how long tests are expected to last; thus, we
          # need a better fix
          timeout *= 1 + tc_c._tcs_total / 3.0
  
          ts0 = time.time()
          ts = time.time()
          ts_reported = { target.fullid : ts for target in pending }
          try:
              while ts - ts0 < timeout and pending:
                  _period = period * (1 + random.random())
                  # Note this loop is kinda weird.
                  #
                  # We will progress in the for loop only if we are
                  # succesful in locking the first target in the list --
                  # that's because we have to acquire all the targets in
                  # the same order, to avoid deadlocks -- and we choose
                  # alphabetical order of the target's fullid, so it is
                  # valid across servers FIXME: this will have to change
                  # to some sort of UUID.
                  #
                  # Why do we also re-acquire the acquired ones? To tell
                  # the server we are still using them, otherwise they
                  # might be marked as idle and released if the process
                  # with the other ones is taking a long time.
                  for _target in list(acquired) + list(pending):
                      try:
                          if not self._targets_disabled_too:
                              # Refresh info
                              _rtb, _rt = tcfl.ttb_client._rest_target_find_by_id(_target.fullid)
                              disabled = _rt.get('disabled', False)
                              if disabled:
                                  raise blocked_e(
                                      "%s: acquisition failed (target has "
                                      "been disabled)" % _target.fullid)
                          _target.report_info(
                              "acquiring with tix '%s'" % self.ticket,
                              dlevel = 6)
                          # acquire a single target, soft retrying half as
                          # much as we have of timeout left
                          self._target_acquire_robust(
                              _target, (ts0 + timeout - time.time()) / 2, period)
                          # Clear progress information we might have printed
                          commonl.progress("")
                          _target.report_info("acquired", dlevel = 5)
                          if _target in pending:
                              pending.remove(_target)
                              acquired.append(_target)
                          # Stays in the for loop to acquire the next
                      except requests.exceptions.HTTPError as e:
                          # FIXME: this is DIRTY, need a more
                          # programatical way to do it
                          if not "tried to use busy target" in e.message:
                              raise
                          owner = self._busy_msg_get_owner(e.message)
                          _target.report_info(
                              "acquisition failed (busy, owned by '%s') "
                              "@%.2f/%.2fs, waiting for %.2fs and retrying"
                              % (owner, ts - ts0, timeout, _period),
                              dlevel = 5)
                          if ts - ts_reported[_target.fullid] > 2:
                              ts_reported[_target.fullid] = ts
                              commonl.progress(
                                  "[%s: waiting to acquire %s from %s, "
                                  "%ds so far]" % (
                                      self.ticket, _target.fullid,
                                      owner, ts - ts0)
                              )
                          break	# Exit for, wait
  
                  self.target_event.wait(_period)
                  ts = time.time()
              if pending != []:
                  msg = "can't acquire %s, still busy after %.2fs" \
                        % (",".join([i.fullid for i in pending]), timeout)
                  self.log.error("%s: " % self.name + msg)
                  raise blocked_e(msg)
          except:
              if tc_c.release:
                  for _target in reversed(acquired):
                      _target.report_info("releasing", dlevel = 6)
                      _target.rtb.rest_tb_target_release(_target.rt,
                                                         force = False,
                                                         ticket = self.ticket)
                      _target.report_info("released", dlevel = 5)
              else:
                  self.report_info("WARNING!! not releasing targets",
                                   dlevel = -10)
              raise
  
          self._prefix_update()
          try:
              yield
          finally:
              self._prefix_update()
              if tc_c.release:
                  for _target in reversed(acquired):
                      _target.report_info("releasing", dlevel = 6)
                      try:
                          _target.rtb.rest_tb_target_release(
                              _target.rt, force = False, ticket = self.ticket)
                      except requests.HTTPError as e:
                          # the script has released the target itself
                          if 'tried to use non-acquired target' in e:
                              pass
                      _target.report_info("released", dlevel = 5)
              else:
                  self.report_info("WARNING!! not releasing targets",
                                   dlevel = -10)
              self.target_event.set()
  
      def targets_active(self, *skip_targets):
          """
          Mark each target this testcase use as being used
  
          This is to be called when operations are being done on the
          background that the daemon can't see and thus consider the
          target active (e.g.: you are copying a big file over SSH)
  
          >>> class mytest(tcfl.tc.tc_c):
          >>>     ...
          >>>     def eval_some(self):
          >>>         ...
          >>>         self.targets_active()
          >>>         ...
  
          If any target is to be skipped, they can be passed as arguments:
  
          >>> @tcfl.tc.interconnect()
          >>> @tcfl.tc.target()
          >>> @tcfl.tc.target()
          >>> class mytest(tcfl.tc.tc_c):
          >>>     ...
          >>>     def eval_some(self, target):
          >>>         ...
          >>>         self.targets_active(target)
          >>>         ...
          """
          for target in self.targets.values():
              if not target in skip_targets and target.keep_active:
                  target.active()
  
      def _targets_active(self):
          # backwards compat
          self.targets_active()
  
      #
      # Remote target selection
      #
      # Given all the list of available, this helps _run_on_targets() to
      # generate the minimum set of unique permutations of interconnects
      # and targets where we can / need to run the TC for optimal
      # coverage.
      #
      # _target_wants_find_candidates() finds candidate remote targets
      # _target_want_list_permutations() computes the permutations of
      # wanted targets with suitable candidate remote targets
      #
      # _target_wants_find_candidates()
      #   _target_want_find_candidates()
      #     _target_want_rt_candidates()
      #       _targets_select_by_spec()
      #       _targets_select_by_app()
      #
      def _target_select_by_app(self, target_want_name, target_want,
                                rt, bsp_model, origin):
          rt_full_id = rt['fullid']
          rt_type = rt['type']
  
          if 'app' not in target_want or not target_want['app']:
              self.report_info(
                  "target '%s': %s:%s (type:%s): candidate by App builder; "
                  "there are no App builder specifications, so have no say @%s"
                  % (target_want_name, rt_full_id, bsp_model, rt_type,
                     origin),
                  dlevel = 7)
              return True
  
          wanted_bsps = set(target_want['app'].keys())
          wanted_bsp_count = len(wanted_bsps)
          if 'bsp_models' in rt:
              bsps = rt['bsp_models'][bsp_model]
          else:
              bsps = []
          if bsps == None:
              bsps = set([bsp_model])
          else:
              bsps = set(bsps)
          bsp_count = len(bsps)
          if bsp_count != wanted_bsp_count:
              self.report_info(
                  "target '%s': %s:%s (type:%s): ignoring by App builder; "
                  "BSP count %d doesn't match needed %d by app_* constraints "
                  "set in @%s"
                  % (target_want_name, rt_full_id, bsp_model, rt_type,
                     bsp_count, wanted_bsp_count, origin),
                  dlevel = 8)
              return False
  
          # If we have a list of BSPs we need to match, make sure the
          # remote target has them. The list of wanted BSPs comes from
          # the ones we might have configured by specifying apps
          # to the @target decorator.
          # Note that per definition, @target allows only ONE wildcard
          # BSP, for the case when a single BSP/app is specified
          # and we don't care to specify the BSP.
          if "*"  in wanted_bsps:
              # This could never happen, but be anal
              assert wanted_bsp_count == 1
              # the first check above in the count has made sure we are
              # only asking for a target with as many BSPs as the ones
              # we are asking for in the wanted section. By means of
              # having a "*" BSP, it ensures we only want one BSP, so we
              # know this is a match
              self.report_info(
                  "target '%s': %s:%s (type:%s): candidate by App buider; "
                  "wildcard BSP in app matches single BSP "
                  "target/BSP-model @%s"
                  % (target_want_name, rt_full_id, bsp_model, rt_type, origin),
                  dlevel = 7)
              return True
  
          # If we get here, we have a list of wanted BSPs in the
          # app specification of the target/want. We also have a
          # list of BSPs the remote target/bsp_model combination
          # offers. We should have a one-on-one mapping
          orphan_bsps = bsps - wanted_bsps
          if orphan_bsps:
              self.report_info(
                  "target '%s': %s:%s (type:%s): ignoring by App builder; "
                  "BSPS %s are left with no build information with "
                  "the current app_* information "
                  "(you might want to specify 'app_manual' if those "
                  "are going to be built manually)"
                  " @%s"
                  % (target_want_name, rt_full_id, bsp_model, rt_type,
                     ",".join(list(orphan_bsps)), origin),
                  dlevel = 8)
              return False
          orphan_wanted_bsps = wanted_bsps - bsps
          if orphan_wanted_bsps:
              self.report_info(
                  "target '%s': %s:%s (type:%s): ignoring by App builder; "
                  "BSPS %s specified to build in @%s are not exposed "
                  "by the target"
                  % (target_want_name, rt_full_id, bsp_model, rt_type,
                     origin, ",".join(list(orphan_bsps))),
                  dlevel = 8)
              return False
          # I guess there is a match
          self.report_info(
              "target '%s': %s:%s (type:%s):  candidate by App builder; "
              "selected by app_* specifications on BSP count match "
              "(%d) and serviced BSPs @%s"
              % (target_want_name, rt_full_id, bsp_model, rt_type,
                 bsp_count, origin),
              dlevel = 7)
          return True
  
      def _targets_select_by_spec(self, target_want_name,
                                  rt, bsp_model, spec, origin,
                                  _kws = None):
          if not _kws:
              _kws = {}
          # We are going to modify the _kws dict, so make a copy!
          kws = dict(_kws)
          # Given a remote target and BSP-model and a filtering spec,
          # iterate over the BSPs that apply to said BSP-model and see
          # where the spec matches -- if so, consider that rt/bsp-model
          # as candidate for the target need specified by the testcase in
          # target_want.
          #
          # For each spec that matches, then we are good to consider
          # that rt/bsp-model combination as candidate, we don't need to
          # keep iterating over BSPs
          #
          # Note it does not matter if we select the same remote target
          # in two different target_wants -- the creator of the target
          # groups will discard permutations that are not valid.
          if bsp_model:
              kws['bsp_model'] = bsp_model
              if 'bsp_models' in rt:
                  bsps = rt['bsp_models'][bsp_model]
                  if bsps == None:
                      bsps = [ bsp_model ]
              else:
                  bsps = []
          else:
              bsps = []
              kws['bsp_model'] = ""
          kws['bsp_count'] = len(bsps)
          kws_bsp = dict()
          commonl.kws_update_from_rt(kws, rt)
          rt_full_id = rt['fullid']
          rt_type = rt['type']
  
          for bsp in bsps:
              kws_bsp.clear()
              kws_bsp.update(kws)
              kws_bsp['bsp'] = bsp
              if 'bsps' in rt:
                  commonl.kws_update_type_string(kws_bsp, rt['bsps'][bsp])
              self.report_info(
                  "target '%s': %s:%s/%s (type:%s): considering by spec"
                  % (target_want_name, rt_full_id, bsp_model, bsp, rt_type),
                  dlevel = 9)
              if commonl.conditional_eval("target selection", kws_bsp,
                                          spec, origin, kind = "specification"):
                  # This remote target matches the specification for
                  # this target want
                  self.report_info(
                      "target '%s': %s:%s/%s (type:%s): candidate by spec"
                      % (target_want_name, rt_full_id, bsp_model, bsp, rt_type),
                      dlevel = 7)
                  return True
              else:
                  self.report_info(
                      "target '%s': %s:%s/%s (type:%s): ignoring by spec; "
                      "didn't match '%s'"
                      % (target_want_name, rt_full_id, bsp_model, bsp,
                         rt_type, spec),
                      dlevel = 8)
          if bsps == []:
          # If there are no BSPs, just match on the core keywords
              # No match on BSPs? try without a BSP
              if commonl.conditional_eval("target selection", kws,
                                          spec, origin, kind = "specification"):
                  # This remote target matches the specification for
                  # this target want
                  self.report_info(
                      "target '%s': %s:%s (type:%s): candidate by spec w/o BSP"
                      % (target_want_name, rt_full_id, bsp_model, rt_type),
                      dlevel = 7)
                  return True
              else:
                  self.report_info(
                      "target '%s': %s:%s (type:%s): ignoring by spec w/o BSP; "
                      "didn't match '%s'"
                      % (target_want_name, rt_full_id, bsp_model,
                         rt_type, spec),
                      dlevel = 8)
                  return False
  
      def _target_want_rt_candidate(self, target_want_name, target_want,
                                    rt_full_id, bsp_model, kws = None):
          if not kws:
              kws = {}
          # Decide if a remote target and BSP model shall be considered
          # by a testcase's want of a target
          rt = self.rt_all[rt_full_id]
          rt_type = rt['type']
          spec = target_want.get('spec', None)
          if spec == None:
              wanted_by_specs = True
          elif isinstance(spec, basestring):
              wanted_by_specs = self._targets_select_by_spec(
                  target_want_name, rt, bsp_model,
                  spec, target_want['origin'], _kws = kws)
          elif callable(spec):
              wanted_by_specs = spec(
                  target_want_name, target_want,
                  rt, bsp_model,
                  spec, target_want['origin'])
          else:
              raise blocked_e("Unknown type %s for target_want['spec'] @%s"
                              % (type(spec).__name__, target_want['origin']))
          if not wanted_by_specs:
              return False
          wanted_by_app = self._target_select_by_app(
              target_want_name, target_want,
              rt, bsp_model, target_want['origin'])
  
          # If the user specified a spec (filter or function)
          # and it conflicts with what the App builder says,
          # that's something is not ok.
          if spec != None and not wanted_by_app:
              wanted_bsps = set(target_want['app'].keys())
              if 'bsp_models' in rt:
                  bsps = rt['bsp_models'][bsp_model]
              else:
                  bsps = []
              if bsps == None:
                  bsps = set([bsp_model])
              else:
                  bsps = set(bsps)
              orphan_bsps = bsps - wanted_bsps
              if orphan_bsps:
                  orphan_bsps_str = \
                      ". Missing app assignment to " \
                      "BSPs " + " ".join(list(orphan_bsps)) + \
                      "; use `app_manual` for those who will " \
                      "be manually configured/built/deployed... " \
                      "in configure/build/... methods"
              else:
                  orphan_bsps_str = ""
              self.report_info(
                  "target '%s': %s:%s (type:%s): can't use; "
                  "conditions imposed by the spec @%s conflict "
                  "with the App builder's"
                  % (target_want_name, rt_full_id, bsp_model,
                     rt_type, target_want['origin'])
                  +  orphan_bsps_str, dlevel = 3)
              return False
  
          if not wanted_by_app:
              return False
  
          return True
  
      def _target_want_find_candidates(self, target_want_name,
                                       target_want, rts_bsp_models,
                                       kws):
          candidates = set()
          # Note the shuffling -- we want to access in random order
          # so we spread access around all the selected targets in
          # case we only choose one of each type
          rts_bsp_models = random.sample(rts_bsp_models, len(rts_bsp_models))
          for rt_full_id, bsp_model in rts_bsp_models:
              if self._target_want_rt_candidate(target_want_name, target_want,
                                                rt_full_id, bsp_model,
                                                kws = kws):
                  candidates.add((rt_full_id, bsp_model))
  
          if not candidates:
              self.report_skip(
                  "target '%s': No targets can be used (all %d "
                  "selected from %d available eliminated by "
                  "testcase filtering)" %
                  (target_want_name, len(rts_bsp_models),
                   len(self.rt_all)))
          return candidates
  
      def _rt_types(self, rt_bsp_model_set):
          """
          Given a list/set of remote targets in (fullid,BSP-MODEL), return a set
          of the remote target types / BSPmodels represented in said list
          """
          rt_types = set()
          for rt_fullid, bsp_model in rt_bsp_model_set:
              rt_types.add((self.rt_all[rt_fullid]['type'], bsp_model))
          return rt_types
  
      def _target_wants_find_candidates(self, target_want_name_list,
                                        rt_selected, kws = None):
          """
          For the given wanted targets, find which of the remote targets given
          are candidates to be considered for assignment.
  
          :param list target_want_name_list: list of wanted target names (in
            self._targets), targets/interconnects which will be passed to
            testcase methods.
  
          :param dict rt_selected: dict of available remote targets to use, keyed
            by remote target full ID and set of available :term:`BSP
            models`s for each.
  
          :returns: dictionary of candidate assignment keyed by the wanted target
            name; each contains a set of suitable remote target (in a tuple
            (FULLID,BSP-MODEL).
  
          """
          if not kws:
              kws = {}
          rts_bsp_models = []
          candidates = {}
          # Pregenerate a list of <rt_fullid, bspmodel>
          for rt_full_id, bsp_models in rt_selected.items():
              if bsp_models:
                  for bsp_model in bsp_models:
                      rts_bsp_models.append((rt_full_id, bsp_model))
              else:
                  rts_bsp_models.append((rt_full_id, None))
          for target_want_name in target_want_name_list:
              candidates[target_want_name] = \
                  self._target_want_find_candidates(
                      target_want_name, self._targets[target_want_name],
                      rts_bsp_models, kws)
          return candidates
  
      # FIXME: use this everywhere where we are trying to print rt_selected
      @staticmethod
      def _selected_str(rt_selected):
          return " ".join([fullid + ":" + ",".join(bsp_models)
                           for fullid, bsp_models in rt_selected.iteritems()])
  
      @staticmethod
      def _tg_str(icg):
          s=""
          for twn, (rt_full_id, bsp_model) in sorted(icg.iteritems(),
                                                     key = lambda x: x[0]):
              if bsp_model == None:
                  s += "%s=%s " % (twn, rt_full_id)
              else:
                  s += "%s=%s:%s " % (twn, rt_full_id, bsp_model)
          return s[:-1]
  
      def _target_wants_list_permutations(self, target_want_candidates, n,
                                          tag = "target", name_prefix = ""):
  
          # Compute how many unique candidates we have
          candidates = set()
          for c in target_want_candidates.values():
              candidates |= c
  
          # max_repeats is how many repetitions we will allow before we
          # call it quits
          #
          # A repetition is when we have to reject a candidate target
          # group because:
          #
          #  - we have already considered that type
          #
          #  - it involves an invalid combination of targets
          #
          # Now, this algorithm is anything but efficient now and this
          # is a hack that will be rid of when we make this algorithm
          # proper.
          #
          # As we have more candidates, we need more chances to find it
          max_repeats = max(100, 5 * len(candidates))
  
          # Limited evaluation: this means to only do one target of each
          # type. This can be general (set on the command line, applies to all
          # targets) or per per target (declared by the target
          # decorator. _types_seen decides which is applied to each.
          _types_seen = {}
          for twn in target_want_candidates:
              target_want = self._targets[twn]
              mode = target_want['kws'].get('mode', None)
              if mode == None:
                  mode = self._mode
              if mode == "one-per-type":
                  _types_seen[twn] = 'one-per-type'
              elif mode == "any":
                  _types_seen[twn] = 'any'
              elif mode == "all":
                  _types_seen[twn] = 'all'
                  # If the user is asking to run EVERYWHERE *and* we
                  # have only one target,  increase the max number of
                  # iterations to the number of available remote
                  # targets and BSP model combinations, to try to pick'em all
                  # We don't do this when more than one target, the
                  # permutations could go crazy really quick -- use the
                  # command line to control that.
                  if len(target_want_candidates) == 1:
                      _n = 0
                      for rt_fullid, rt in self.rt_all.iteritems():
                          # Yes, if no BSP model, count as 1
                          bsp_model_count = len(rt.get('bsp_models', [None ]))
                          _n += bsp_model_count
                      if _n > n:
                          n = _n
                          max_repeats = 3 * _n
              else:
                  assert True, "Unknown mode in self._mode (%s) "\
                      "or target_want['kws']['mode'] (%s)" \
                      % (self._mode, target_want['kws']['mode'])
  
  
          # We are going to generate permutations of assigning candidate remote
          # targets (A*, B* in the example below) to each target want declared by
          # the testcsae (T* in the example); if we have:
          #
          # T1 candidates A1, A2, B1
          # T2 candidates A1, A2, B1
          #
          # Then the possible groups would be:
          #
          #       1  2  3   4  5  6   7  8  9
          #  T1  A1 A1 A1  A2 A2 A2  B1 B1 B1
          #  T2  A1 A2 B1  A1 A2 B1  A1 A2 B1
          #
          # Removing dups (as two target wants can't use the same remote target)
          #
          #       2  3   4  6   7  8
          #  T1  A1 A1  A2 A2  B1 B1
          #  T2  A2 B1  A1 B1  A1 A2
          #
          # Now, that's  six groups; but A1 and A2 are type A and B1 is type B,
          # so we don  have to redo targets that have the same types in limited
          # mode (that'd be 4--repeats 2, 6-3, 8-7):
          #
          #      2  3   7
          #  T1  A1 A1  B1
          #  T2  A2 B1  A1
          #
          # We are left with three unique type / target assignment permutations
          #
          # Note that if now we mark T2 as 'don't care' (as in any target type
          # works)
          #
          #       2  3   4  6   7  8
          #  T1  A1 A1  A2 A2  B1 B1
          #  T2  N  N   N  N   N  N
          #
          # Then
          #
          #       2  7
          #  T1  A1 B1
          #  T2  N  N
          #
          # We are left with just two groups to test
          #
          # For tracking limited mode: for each permutation we generate,
          # we have a string [TARGET-WANT-NAME1:TARGET-TYPE1
          # TARGET-WANT-NAME2:TARGET-TYPE2 ...]; if the string is in the
          # @types_seen set, then it means we already have that type
          # covered, so we ignore it.
          #
          # If we generate this many groups that are repeated, we are maybe out
          # of permutations to generate. LAME, but quick to implement
          types_seen = set()
  
          permutations = {}
          itr = 0
          target_counter = 0
          while itr < n:
              if max_repeats == 0:
                  # Lame way to avoid getting stuck in not being able to generate
                  # good permutations
                  self.report_info("Exiting after generating %d permutations "
                                   "(tried to generate more, but discarded "
                                   "too many)" % len(permutations), dlevel = 6)
                  break
  
              # Instead of brute forcing a random number, converting it to our
              # base and discarding the invalids, we are going to go for each
              # target want at random order in the list and pick up a candidate
              # at random, making sure it is not taken already
  
              # This keeps track of which physical remote targets have already
              # been taken, so we don't try to pick it again
              rts_taken = set()
              _types = dict()
              perm = {}
              cnt = 0
              for target_want_name in random.sample(target_want_candidates,
                                                    len(target_want_candidates)):
                  target_want = self._targets[target_want_name]
                  # Of the want's candidates, get which aren't taken already
                  rts_not_taken = set()
                  for rt_full_id, bsp_model in target_want_candidates[target_want_name]:
                      if rt_full_id in rts_taken:
                          continue
                      rts_not_taken.add((rt_full_id, bsp_model))
                  if len(rts_not_taken) == 0:
                      self.report_skip(
                          "%s group %s: no remote targets (or group of) "
                          "can satisfy the conditions for wanted target '%s'" %
                          (tag, name_prefix, target_want_name))
                      perm.clear()
                      type(self).class_result += result_c(0, 0, 0, 0, 1)
                      break
                  # From available candidates, pick one at random and
                  # mark it taken
                  (rt_fullid, bsp_model) = random.sample(rts_not_taken, 1)[0]
                  rts_taken.add(rt_fullid)
                  perm[target_want_name] = (rt_fullid, bsp_model)
                  rt_type = self.rt_all[rt_fullid]['type']
                  if _types_seen[target_want_name] == 'one-per-type':
                      _types[target_want_name] = (rt_type, bsp_model)
                  elif _types_seen[target_want_name] == 'any':
                      # Use always the same string, so it is simplified
                      # by the set
                      _types[target_want_name] = ('any', None)
                  else: 	#  run in 'ALL' targets: use the sequential
                          #  counter as a "type" so they are all
                          #  different and thus are unique so it is not
                          #  eliminated by the set reduction
                      _types[target_want_name] = ("%d" % target_counter, None)
                  self.report_info(
                      "%s group %s: target '%s': candidate %s:%s"
                      % (tag, name_prefix, target_want_name,
                         rt_fullid, bsp_model),
                      dlevel = 8)
                  cnt += 1
                  target_counter += 1
  
              if perm == {}:
                  # This meant we don't have enough targets to assign
                  break
              perm_id = commonl.mkid(self._tg_str(perm), l = 4)
              # Generate a unique string that lists the target want
              # names and the type of remote target they got assigned
              perm_types_id = " ".join([
                  twn + "=" + rt_type + \
                    ((":" + bsp_model) if bsp_model else "")\
                  for twn, (rt_type, bsp_model) in sorted(_types.iteritems(),
                                                          key = lambda x: x[0])])
              # Have we seen that permutations of target wants / remote
              # target types?
              if perm_types_id in types_seen:
                  max_repeats -=1
                  self.report_info("%s group %s: "
                                   "ignoring (types already considered) %s"
                                   % (tag, perm_id, perm_types_id), dlevel = 7)
                  continue
              elif perm_id in permutations:
                  # We have one like this already
                  _name_prefix = "-" + name_prefix
                  self.report_info("%s group %s%s: "
                                   "ignoring (repeated)"
                                   % (tag, _name_prefix, perm_id), dlevel = 7)
                  max_repeats -=1
                  continue
  
              itr += 1
              types_seen.add(perm_types_id)
              permutations[perm_id] = perm
          return permutations
  
  
      #
      # The top level testcase running interface
      #
      # _run_on_targets() is given all the available remote targets,
      # figures out the permutations of remote targets to wanted targets
      # and spawns on one thread per TC/targetgroup _run(), which calls
      # _run_on_target_group()
  
      def _eval_prepare(self, cnt1, name):
          # Make each run a fresh run
          self.buffers.clear()
          self.tls.expecter.buffers.clear()
          self.tls.expecter.buffers_persistent.clear()
          self.buffersdir = os.path.join(self.tmpdir,
                                         "eval-buffers-%02d-%s" % (cnt1, name))
          # Wipe the directory (if it exists, and recreate it fresh)
          # --nothing interesting should be left over here anyway.
          shutil.rmtree(self.buffersdir, True)
          os.makedirs(self.buffersdir)
  
      def _do_the_eval(self):
          # on each eval repetition, we want self.result_eval to only
          # have the eval data for that repetition; for all the
          # repetitions, we accumulate it in result_all_evals.
          result_all_evals = result_c(0, 0, 0, 0, 0)
          for self.eval_count in range(0, self.eval_repeat):
              try:
                  # Repeat the evaluation as many times as we were asked
                  # too, even if it fails.
                  #
                  # Note we run setup/start/eval/teardown for all the eval*
                  # methods; however, for the test* methods, we do for each
                  # setup/start/testX/teardown
                  with msgid_c("E#%d" % (self.eval_count + 1), phase = 'eval'):
                      self.result_eval = result_c(0, 0, 0, 0, 0)
                      if self._eval_serial:
                          self._eval_prepare(self.eval_count, 0)
                          retval = self._methods_call("setup")
                          self.result_eval += retval
                          if retval.errors or retval.failed or retval.blocked:
                              self._methods_call("teardown")
                              continue
                          retval = self._methods_call("start")
                          self.result_eval += retval
                          if retval.errors or retval.failed or retval.blocked:
                              self._methods_call("teardown")
                              continue
                          self.result_eval += self._methods_call("eval")
                          self._methods_call("teardown")
                      for fname, fn, _type, args in self._test_serial:
                          self.result_eval = result_c(0, 0, 0, 0, 0)
                          self._eval_prepare(self.eval_count, fname)
                          retval = self._methods_call("setup")
                          self.result_eval += retval
                          if retval.errors or retval.failed or retval.blocked:
                              self._methods_call("teardown")
                              continue
                          retval = self._methods_call("start")
                          self.result_eval += retval
                          if retval.errors or retval.failed or retval.blocked:
                              self._methods_call("teardown")
                              continue
                          self.result_eval += self._method_run(fname, fn,
                                                               _type, args)
                          self._methods_call("teardown")
              finally:
                  result_all_evals += self.result_eval
                  # However, we always update self.result_eval() at the end with
                  # the totals in case we exit and are not taking the
                  # loop again
                  self.result_eval = result_all_evals
          return self.result_eval.summary()
  
  
      def _run_on_target_group(self, msgid):
          """
          Run the five phases of evaluating a test case and collect the
          result of each.
  
          Note that for deployment and evolution we need the targets assigned.
          """
          # Note dlevel_passed; we don't want the PASS messages for
          # level-0 verbosity, only when things FAIL or are BLOCKed
          result = result_c(0, 0, 0, 0, 0)
          while True:	# Fugly but beats not having goto
              # This we ran during __init__, but then because this
              # *self* was a copy of that self that ran __init__
              # (FIXME: ugly hack that has to be fixed), we need to
              # run it again so it generates the right keywords for
              # the targets where it is assigned to run.
              for hook in self.hook_pre:
                  hook(self)
              # specific, we can take it out to the TC level
              if not self._phase_skip("configure"):
                  with msgid_c("C", phase = 'config'):
                      retval = self._methods_call("configure")
                  result += retval
                  # This testcase might have no build, so ignore if nothing
                  # is ran in the build
                  if self.report_tweet(
                          "configure", retval, ignore_nothing = True,
                          extra_report = self._extra_report(self.kws),
                          dlevel = -1, dlevel_skipped = 2, dlevel_passed = 2) \
                          == False \
                          or retval.skipped > 0:
                      break
              # We don't want to count build success as a separate
              # test-case when we have evaluation--hence why we reassign
              # `result` insted of adding to it.
              if not self._phase_skip("build"):
                  with msgid_c("B", phase = 'build'):
                      result = self._methods_call("build")
                  # This testcase might have no build, so ignore if nothing
                  # is ran in the build
                  if self.report_tweet(
                          "build", result, ignore_nothing = True,
                          extra_report = self._extra_report(self.kws),
                          dlevel = -1, dlevel_skipped = 2, dlevel_passed = 1) \
                          == False \
                          or result.skipped > 0:
                      break
  
              # are the test target or testcase declaring as build only?
              if self.build_only:
                  self.report_info("testcase is build only: "
                                   "skipping deploy and eval (%s)"
                                   % ", ".join(self.build_only), dlevel = 4)
                  break
  
              # We don't want to count build success as a separate
              # test-case when we have evaluation--hence why we reassign
              # `result` insted of adding to it.
  
              deploy_skip = self._phase_skip("deploy")
              eval_skip = self._phase_skip("eval")
              if deploy_skip and eval_skip:	# Skip assigning targets
                  break				# if we know we won't run
  
              # We use the current message id as the ticket for target
              # assignment
              try:
                  with self._targets_assign():
                      if not deploy_skip:
                          with msgid_c("D", phase = "deploy"):
                              self._eval_prepare(0, "deploy")
                              retval = self._methods_call("deploy")
                          result += retval
                          # We don't report deploy as much as build/eval
                          if self.report_tweet(
                                  "deploy", retval, ignore_nothing = True,
                                  extra_report = self._extra_report(self.kws),
                                  dlevel_passed = 1) \
                                  == False or retval.skipped > 0:
                              break
                      if not eval_skip:
                          retval = self._do_the_eval()
                          result += retval
                          if self.report_tweet(
                                  "evaluation", retval,
                                  extra_report = self._extra_report(self.kws),
                                  dlevel = -1, dlevel_skipped = 1,
                                  dlevel_passed = 1) \
                                  == False:
                              break
              # Catch stuff in _targets_assign()
              except Exception as e:
                  self.log.error("exception: %s %s\n%s" %
                                 (type(e).__name__, e, traceback.format_exc()))
                  self.report_blck("exception: %s %s" % (type(e).__name__, e),
                                   { 'exception info': traceback.format_exc() },
                                   alevel = 1)
                  result += result_c(0, 0, 0, 1, 0)
  
              break # we are in a while loop that has to happen only once
  
          # Run clean rules
          if not self._phase_skip("clean"):
              with msgid_c("L", phase = 'clean'):
                  retval = self._methods_call("clean")
              self.report_tweet("clean", retval,
                                extra_report = self._extra_report(self.kws),
                                ignore_nothing = True,
                                dlevel_passed = 1)
              # cleanup success does not count, only if it fails
              # or blocks, so it is noticed and can be addressed
              result += result_c(0, retval.errors, retval.failed,
                                 retval.blocked, 0)
  
          return result.summary()
  
      def _cleanup(self):
          # Remove and wipe files left behind
          # FIXME: move this to set all the files in a tempdir specific
          # to each TC instantation and just wipe that dir at the end / __del__
          for pathname in self._cleanup_files:
              try:
                  os.unlink(pathname)
              except OSError as e:
                  if e.errno != errno.ENOENT:
                      raise
  
      def finalize(self, result):
          assert isinstance(result, result_c)
          self.ts_end = time.time()
          self.report_tweet(
              "COMPLETION", result,
              extra_report = self._extra_report(self.kws),
              # Trick: won't see this, report driver will
              level = 1000,
              ignore_nothing = False
          )
          self._cleanup()
  
      def mkticket(self):
          # Note we use this msgid's string as tc_hash for subsitution,
          # it is a unique name based on target name and BSP model, test
          # case name (which might be more than just the tescase path if
          # a single file yields multiple test cases).
          global ticket
          target_group_name = self.target_group.name \
                              if self.target_group else 'n/a'
          if ticket == None:
              self.ticket = msgid_c.encode(
                  self._hash_salt + self.name + target_group_name, 4)
          else:
              self.ticket = self._ident
  
      def _run(self, msgid, thread_init_args):
          """
          High level executor for the five phases of the testcase
  
          Note that self.target_group contains the names of the targets
          and their description
  
          Note seems the msgid is overriden, but context is obtaiend via
          TLS.
          """
          try:
              self.__thread_init__(**thread_init_args)
              result = result_c(0, 0, 0, 0, 0)
              self.mkticket()
              # temporary directory specialization for this TC
              self.tmpdir = os.path.join(tc_c.tmpdir, self.ticket)
              if not os.path.isdir(self.tmpdir):
                  # And fail if exists and it is not a directory
                  os.makedirs(self.tmpdir)
  
              self._kw_set("tmpdir", self.tmpdir)
              self._kw_set("tc_hash", self.ticket)
              with msgid_c(self.ticket, depth = 1, l = 4) as msgid:
                  self._ident = msgid_c.ident()
                  try:
                      self.report_info("will run on target group '%s'"
                                       % (self.target_group.descr))
                      for _target in self.target_group.targets.values():
                          # We need to update all the target's KWS, as we
                          # have added KWS to the tescase (tc_hash and
                          # tg_hash!) , which we could only add once we had
                          # the target group created -- FIXME: this is
                          # currently a hack and we need a better way to do
                          # it--unifying with the target group creation and
                          # setting in run_on_targets()
                          _target.ticket = self.ticket
                          if _target.bsp_model == None:
                              bsp_model = ""
                          else:
                              bsp_model = _target.bsp_model
                          _target.kw_set('tg_hash', tcfl.msgid_c.encode(
                              self.ticket + _target.fullid + bsp_model, 4))
                          _target._ident = msgid_c.ident()
                      result += self._run_on_target_group(msgid)
                  except Exception as e:
                      self.log.error(
                          "exception: %s %s\n%s" %
                          (type(e).__name__, e, traceback.format_exc()))
                      self.report_blck(
                          "exception: %s %s" % (type(e).__name__, e),
                          { 'exception info': traceback.format_exc() },
                          alevel = 1)
                      result = result_c(0, 0, 0, 1, 0)
                  finally:
                      self.finalize(result)
          except Exception as e:
              # This msgid_c context is a hack so that this exception
              # report has a proper RUNID and HASH prefix
              with msgid_c(self.ticket, depth = 0, l = 4) as msgid:
                  self.report_blck(
                      "BUG exception: %s %s" % (type(e).__name__, e),
                      { 'exception info': traceback.format_exc() },
                      alevel = 1)
              result = result_c(0, 0, 0, 1, 0)
  
          self.result = result
          results = [ (type(self), result.summary()) ]
  
          for tc in self._tcs_post:
              # Well, this is quite a hack -- for reporting to work ok,
              # rebind each's target's testcase pointer to this
              # subtestcase.
              for _target in self.targets.values():
                  _target.testcase = tc
              tc.target_group = self.target_group
              tc._targets = self._targets
              tc._kw_set("type", self.kws['type'])
              tc._methods_prepare()	# setup phase running methods
              # remember this returns a list, so we have to concatenate them
              results += tc._run(msgid_c.parent(),
                                 dict(expecter_parent = self.tls.expecter))
  
          # Undo the hack from before, as we might need these values to
          # be proper for reporting later on
          for _target in self.targets.values():
              _target.testcase = self
  
          return results
  
      def post_tc_append(self, tc):
          """
          Append a testcase that shall be executed inmediately after
          this testcase is done executing in the same target group.
  
          This is a construct that can be used for:
  
          - execute other testcases that have been detected as needed
            only during runtime
  
          - reporting subtestcases of a main testcase (relying only on
            the output of the main testcase execution, such as in
            :class:`tcfl.tc_zephyr_sanity.tc_zephyr_subsanity_c`.
  
          :param tc_c tc: [instance of a] testcase to append; note this
            testcase will be executed in the same target group as this
            testcase is being executed. So the testcase has to declare
            at the same targets (with the same names) or a subset of
            them. Example:
  
            >>> @tcfl.tc.target("target1")
            >>> @tcfl.tc.target("target2")
            >>> @tcfl.tc.target("target3")
            >>> class some_tc(tcfl.tc.tc_c):
            >>>     ...
            >>>     def eval_something(self, target2):
            >>>         new_tc = another_tc(SOMEPARAMS)
            >>>         self.post_tc_append(new_tc)
            >>>
            >>>
            >>> @tcfl.tc.target("target2")
            >>> class another_tc(tcfl.tc.tc_c):
            >>>     ...
            >>>     def eval_something(self, target2):
            >>>         self.report_info("I'm running on target2")
            >>>
            >>> @tcfl.tc.target("target1")
            >>> @tcfl.tc.target("target3")
            >>> class yet_another_tc(tcfl.tc.tc_c):
            >>>     ...
            >>>     def eval_something(self, target1, target3):
            >>>         self.report_info("I'm running on target1 and target3")
  
          """
          self._tcs_post.append(tc)
  
      def _clone(self):
          """Return a hybrid shallow/deep copy of this object
  
          This for now a dirty workaround around the fact that we have to
          split a testcase object between the information for it and what
          has to be specific to each thread.
  
          Anything that is not going to be modified, we don't need to deep
          copy, the rest, yes.
  
          # FIXME: it makes little sense to try to load upon TCPY writers
          to split deep vs shallow, shareable vs not.
          So let's have this clone function default to do a deep copy,
          except of the parts we know are ok to be shallow and those we
          share.
  
          - this is really bad in general becuase it means the
          constructor is not called, so we are shortcircuting a lot of
          implementation details for inheriting classes?. We have to
          stop doing this, and just do a new object and copying?
  
          """
          c = copy.copy(self)
          c.buffers = {}	# For use during execution of actions
          c.kws = dict(self.kws)
          c.kws_origin = dict(self.kws_origin)
          c.log = tc_logadapter_c(logging, None)
          c._prefix_update()
          c.tls = threading.local()
          c._cleanup_files = set()
          return c
  
      @result_c.from_exception
      def _run_on_targets(self, tp, rt_all, rt_selected, ic_selected):
          """
          Launch the test case execution, in one or more background
          threads
  
          :param dict targets: Dictionary of targets where the test
            would have to be run. Can be empty (to indicate
            auto-selection of targets or when unneeded--eg for static
            test cases).
  
          :param dict rt_selected: Dictionary keyed by
            fullid listing the BSP models that have to be run for
            each target. This comes after all the filtering, and more
            filtering can happen here, target specific.
  
          :param dict ic_selected: Dictionary keyed by fullid listing
            the interconnects available, also listing (if it provides)
            the BSP models that have to be run for each target. This
            comes after all the filtering, and more filtering can happen
            here, target specific.
  
          :returns: dict of thread descriptors as returned by
            ThreadPool.apply_async(); if {}, it means that no targets
            could be located and this TC is blocked.
  
          :exceptions: any, treat them as a blocker
  
          """
          try:
              threads = []
  
              # FIXME: maybe move this to __init__
              self._methods_prepare()	# setup phase running methods
              self.rt_all = rt_all
              if self.is_static():
                  if self._dry_run:
                      self.report_info("will run")
                  rt_selected = { 'local': rt_all['local']['bsp_models'].keys() }
                  ic_selected = { }
  
              # This will be now testcase-specific, so make a deep copy
              # so it can be modified by the testcase -- note we might alter
              # FIXME: note we might not need this deep coy --
              # verify once the reorg is done
              self.rt_selected = copy.deepcopy(rt_selected)
              self.ic_selected = copy.deepcopy(ic_selected)
  
              # list candidates to interconnects
              #
              # We requested R interconnects [len(self._interconnects)] and we
              # have ic_selected as candidates for those interconnects. Each
              # interconnect requested might put restrictons to which
              # interconnects can be used. So in ic_candidates, return which of
              # ic_selected can be used for each interconnect.
              self.report_info("interconnect groups: finding remote ic targets",
                               dlevel = 6)
              ic_candidates = self._target_wants_find_candidates(
                  self._interconnects, ic_selected)
              self.report_info("interconnect groups: available remote ic "
                               "targets: %s" % self._selected_str(ic_selected),
                               dlevel = 5)
              # The list of candidates to interconnects could be arranged in
              # different ways. Permutations will make it grow quick. If IC0 =
              # {AB}, IC1={ABC} and IC2={ABC}, there is an upper ceiling of
              # N!/(N-R)!, where R is the number of interconnects and N is the
              # max number of available candidates (len(ic_selected) (the real
              # ceiling will be lower as this is not a pure permutation
              # problem--some of the sets of candidates don't include all the
              # available N candidates in ic_selected and I don't really know if
              # there is a math way to compute it -- internet got too dense for
              # my limited set theory knowledge quite quick).
              #
              # So we are just going to generate a few at random; if we only have
              # one interconnect, we'll try to get one for each type of
              # interconnect. Otherwise we use the command line to limit the
              # amount of IC groups we create.
              if len(self._interconnects) == 1:
                  ic_want_name = list(self._interconnects)[0]
                  max_permutations = \
                      len(self._rt_types(ic_candidates[ic_want_name]))
              else:
                  # FIXME: get from command line and defaults
                  max_permutations = 10
              self.report_info("interconnect groups: generating %d by "
                               "permuting remote ic targets" %
                               max_permutations, dlevel = 6)
              ic_permutations = self._target_wants_list_permutations(
                  ic_candidates, max_permutations, tag = "interconnect")
              if len(self._interconnects):
                  if len(ic_permutations) == 0:
                      type(self).class_result += result_c(0, 0, 0, 0, 1)
                      raise skip_e("WARNING! No interconnects available")
              elif len(self._interconnects) == 0:
                  # Cheat, when we don't use interconnects, just define an empty
                  # one so we enter into the loop below
                  ic_permutations = { "all": { } }
              for icgid, icg in ic_permutations.iteritems():
                  self.report_info("interconnect group %s: %s"
                                   % (icgid, self._tg_str(icg)), dlevel = 5)
  
  
              # FIXME: now filter them
  
              # Now that we have a set of interconnect permutations, we need
              # to select, for each, which targets fit in the permutation
              # Basically, this is a simple OR, if the target is in any of
              # the interconnects, it's good to go.
              self.report_info("interconnect groups: finding remote "
                               "targets available for each",
                               dlevel = 6)
              icg_selected = collections.defaultdict(dict)
              for icgid, icg in ic_permutations.iteritems():
                  for icwn, (rtic, _) in icg.iteritems():
                      rtic_id = self.rt_all[rtic]['id']
                      for rt_full_id, bsp_models in rt_selected.iteritems():
                          rt = self.rt_all[rt_full_id]
                          if rtic_id in rt.get('interconnects', {}):
                              icg_selected[icgid][rt_full_id] = bsp_models
                  if icgid == "all":	# No interconnects, take'em all
                      icg_selected[icgid] = dict(rt_selected)
  
                  if not icg_selected[icgid]:
                      type(self).class_result += result_c(0, 0, 0, 0, 1)
                      self.report_skip(
                          "interconnect group %s (%s): no targets can be used!!"
                          % (icgid, self._tg_str(icg)), dlevel = 3)
                  else:
                      self.report_info(
                          "interconnect group %s (%s): candidate targets "
                          "available: %s" %
                          (icgid, self._tg_str(icg),
                           self._selected_str(icg_selected[icgid])),
                          dlevel = 4)
  
              # So now icg_selected is a dict, keyed by interconnet
              # group name, which contains the remote targets that are
              # valid candidates to be used for that interconnect group
              # (so they satisfy the condition of being in any of the
              # interconnects that are required for the group).
              #
              # So now, for each of those interconnect groups we need to
              # generate a list of target groups.
              self.report_info("interconnect groups: generating target "
                               "groups for each", dlevel = 6)
              rt_candidates = {}
              rt_permutations = {}
              target_want_list = set(self._targets.keys()) - self._interconnects
              for icgid, rt_selected in icg_selected.iteritems():
                  # Generate keywords that describe the current
                  # interconnects
                  kws_ics = dict()
                  for icwn, ic in ic_permutations[icgid].iteritems():
                      commonl.kws_update_from_rt(kws_ics, self.rt_all[ic[0]],
                                                 prefix = icwn + ".")
                  # list candidates to targets in this interconnect group
                  self.report_info("interconnect group %s: "
                                   "finding remote target candidates" %
                                   icgid, dlevel = 8)
                  rt_candidates = self._target_wants_find_candidates(
                      target_want_list, rt_selected, kws_ics)
                  self.report_info("interconnect group %s: "
                                   "remote target candidates: %s"
                                   % (icgid, pprint.pformat(rt_candidates)),
                                   dlevel = 7)
  
                  # Generate permutations of targets wants vs available
                  # targets for this interconnect group
                  if len(target_want_list) == 1:
                      twn = list(target_want_list)[0]
                      max_permutations = len(self._rt_types(rt_candidates[twn]))
                  else:
                      max_permutations = tc_c.max_permutations
  
                  self.report_info("interconnect group %s: generating "
                                   "target groups" % icgid, dlevel = 6)
                  tgs = self._target_wants_list_permutations(
                      rt_candidates, max_permutations, name_prefix = icgid)
                  if tgs:
                      for tgid, tg in tgs.iteritems():
                          rt_permutations[(icgid, tgid)] = tg
                  elif len(self._targets) == 0: # static TC
                      ic_permutations["localic"] = {}
                      rt_permutations = { ("localic", "localtg") : {} }
                  elif len(target_want_list) == 0: # only ICs?
                      self.report_info("interconnect group %s: "
                                       "no targets needed" % (icgid),
                                       dlevel = 4)
                      # FIXME: fix so if tgid == None, nothjing is printed
                      rt_permutations[(icgid, None)] = {}
                  else:
                      self.report_info("interconnect group %s: "
                                       "not enough targets" % (icgid),
                                       dlevel = 4)
  
              # So now we are going to iterate over all the groups
              for (icgid, tgid), tg in rt_permutations.iteritems():
                  if tgid == None:
                      # no targets, but interconnects
                      tg_name = icgid
                  else:
                      if icgid == "all":	# No interconnects?
                          tg_name = tgid
                      else:
                          tg_name = "%s-%s" % (icgid, tgid)
                  strs = []
                  icg_str = self._tg_str(icg)
                  if len(icg_str):
                      strs.append(icg_str)
                  tg_str = self._tg_str(tg)
                  if len(tg_str) > 0:
                      strs.append(tg_str)
                  if not strs:
                      strs = [ tg_name ]
                  group_str = " ".join(strs)
                  del strs
  
                  icg = ic_permutations[icgid]
                  tc_for_tg = self._clone()
                  if self._dry_run:
                      self.report_info("will run on target group '%s'"
                                       % group_str, dlevel = 1)
                      continue
                  # FIXME: this order could be better
                  target_group = target_group_c(group_str)
                  # Ids of the interconnect targets we'll be using
                  icg_ids = set()
                  for target_want_name in reversed(self._targets.keys()):
                      # iterate over the list of wanted targets to add
                      # them in the right order to the
                      # target_group.
                      #
                      # FIXME: reversed for decorator workaround
                      if target_want_name in icg:
                          rt_full_id, bsp_model = icg[target_want_name]
                          icg_ids.add(rt_all[rt_full_id]['id'])
                      elif target_want_name in tg:
                          rt_full_id, bsp_model = tg[target_want_name]
                      else:
                          raise RuntimeError("BUG? target want %s neither "
                                             "in icf or tg?" % target_want_name)
                      target_group.target_add(
                          target_want_name,
                          target_c(rt_all[rt_full_id], tc_for_tg,
                                   bsp_model, target_want_name))
                  # this should be the unique ID
                  target_group.name_set(tg_name)
                  tc_for_tg.target_group = target_group
                  tc_for_tg.targets = target_group.targets
                  for target in target_group.targets.values():
                      target._kws_update_interconnect_addrs(icg_ids)
                      # this just updates the core keys, but later calls
                      # to kw_set() and company will refresh the main
                      # target.kws dict.
                  thread = tp.apply_async(
                      tc_for_tg._run,
                      (
                          msgid_c.current(),
                          dict(expecter_parent = self.tls.expecter)
                      )
                  )
                  threads.append(thread)
              self.log.info("%d jobs launched" % len(threads))
              return threads
          finally:
              self._cleanup()
  
      #
      # Testcase driver internal interface
      #
  
      @classmethod
      def _classes_enumerate(cls, obj):
          # Recursively enumerate classes in the object that are a base
          # of tc_c -- this allows us to pick up classes that are
          # defined inside other classes, which is quite useful for unit
          # testcases and other odd ends
          l = []
          for x in inspect.getmembers(obj):
              if x == ("__class__", type):
                  pass
              elif inspect.isclass(x[1]):
                  # Well, FIXME: this is not ok -- we are trying to
                  # ignore here any old style unit class from the
                  # ./tests/ tree as we move to the new style of unit
                  # test cases.
                  # But because we have a mess with how we import files
                  # now, some times they are found as
                  # commonl.testing.test_ttbd_mixin, some as
                  # commonl.testing.test_ttbd_mixin and we can't match
                  # on that, so for the time being, we match on name.
                  base_classes = [ i.__name__ for i in inspect.getmro(x[1]) ]
                  for base_class in base_classes:
                      if base_class in  [ "test_ttbd_mixin", "test_tcf_mixin" ]:
                          logging.info("ignoring unit test case %s",
                                       x[1].__name__)
                          break
                  else:
                      l += cls._classes_enumerate(x[1])
                      if issubclass(x[1], tc_c):
                          if x[1].__name__.endswith("_base"):
                              logger.info(
                                  "%s: %s: considering a base class "
                                  "due to name starting with `_base`, "
                                  "ignoring", obj.__file__, x[1].__name__)
                          else:
                              l.append(x[1])
          return l
  
      # Testcases are any file that start with `test` and ends with `.py`
      file_regex = re.compile(r'^test[^/]*\.py$')
  
      # Default driver loader; most test case drivers would over load
      # this to determine if a file is a testcase or not.
      @classmethod
      def is_testcase(cls, path):
          if not cls.file_regex.search(os.path.basename(path)):
              return []
          try:
              # Include where path is in the sys.path, so modules that
              # are in the same directory are loaded
              sys.path.insert(0, os.path.dirname(path))
              name = path.translate(string.maketrans("/.", "__"))
              module = imp.load_source(name, path)
          except tcfl.tc.exception as e:
              raise
          except (ImportError, Exception) as e:
              if 'base.util.tag' in e.message \
                 or 'util.const' in e.message \
                 or 'util.tag' in e.message:
                  raise skip_e(
                      "unsupported: %s" % e.message,
                      { "ex_trace": traceback.format_exc(), "dlevel": 4 })
              else:
                  raise blocked_e(
                      "Cannot import: %s (%s)" % (e, type(e).__name__),
                      { "ex_trace": traceback.format_exc() })
          finally:
              sys.path.pop(0)
          l = cls._classes_enumerate(module)
          # Get instances of subclassess of tc_c class as testcases
          if l == []:
              logger.warning("%s: no suitable classes found in %s",
                             path, module)
              # Nothing found, so let's try to fully unload & delete the
              # module and garbage collect it
              del sys.modules[module.__name__]
              del module
          tcs = []
          for _cls in l:
              path = inspect.getsourcefile(_cls)
              tc = _cls(path + "#" + _cls.__name__,	# Name
                        path,				# File, origin
                        path + ":" + "%d" % inspect.getsourcelines(_cls)[1])
              tcs.append(tc)
          return tcs
  
      @classmethod
      def _create_from_file_name(cls, tcis, file_name):
          """
          Given a filename that contains a possible test case, create one or
          more TC structures from it and return them in a list
  
          :param list tcis: list where to append found test case instances
          :param str file_name: path to file to consider
          :returns: result_c with counts of tests passed/failed (zero,
            as at this stage we cannot know), blocked (due to error
            importing) or skipped(due to whichever condition).
          """
          # FIXME: not working well to ignore .git
          result = result_c(0, 0, 0, 0, 0)
          for ignore_regex, origin in cls._ignore_regexs:
              if ignore_regex.match(file_name):
                  logger.log(6, "%s: ignored by regex %s [%s]",
                             file_name, ignore_regex.pattern, origin)
                  return result
  
          for _tc_driver in cls._tc_drivers:
              tc_instances = []
              try:
                  tc_instances += _tc_driver.is_testcase(file_name)
              except Exception as e:
                  tc_fake = tc_c(file_name, file_name, "builtin")
                  tc_fake.mkticket()
                  with msgid_c(tc_fake.ticket, depth = 1, l = 4) as _msgid:
                      tc_fake._ident = msgid_c.ident()
                      retval = result_c.report_from_exception(tc_fake, e)
                      tc_fake.finalize(retval)
                      result += retval
                  continue
  
              if not tc_instances:
                  continue
  
              for _tc in tc_instances:
                  for testcase_patcher in cls.testcase_patchers:
                      testcase_patcher(_tc)
                  _tc._components_fixup()
                  logger.info("testcase found @ %s", _tc.origin)
              tcis += tc_instances
              break
          else:
              logger.log(7, "%s: no testcase driver got it", file_name)
  
          return result
  
      @classmethod
      def find_in_path(cls, tcs, path):
          """
          Given a path, scan for test cases and put them in the
          dictionary @tcs based on filename where found.
          list of zero or more paths, scan them for files that
          contain testcase tc information and report them.
          :param dict tcs: dictionary where to add the test cases found
  
          :param str path: path where to scan for test cases
  
          :returns: result_c with counts of tests passed/failed (zero,
            as at this stage we cannot know), blocked (due to error
            importing) or skipped(due to whichever condition).
          """
          assert isinstance(tcs, dict)
          # FIXME this should cache stuff so we don't have to rescan all the
          # times--if the ctime hasn't changed since our cache entry, then
          # we use the cached value
          result = result_c(0, 0, 0, 0, 0)
          logger.info("%s: scanning argument", path)
          tc_global.report_info("%s: scanning directory from arguments" % path,
                                dlevel = 4)
          if os.path.isdir(path):
              for tc_path, _dirnames, _filenames in os.walk(path):
                  logger.log(5, "%s: scanning directory", tc_path)
                  tc_global.report_info("%s: scanning directory" % tc_path,
                                        dlevel = 5)
                  # Remove directories we don't care about
                  # FIXME: very o(n)
                  keep_dirs = []
                  for dirname in list(_dirnames):
                      for dir_ignore_regex, _origin in tc_c._ignore_directory_regexs:
                          if dir_ignore_regex.match(dirname):
                              break
                      else:
                          keep_dirs.append(dirname)
                  del _dirnames[:]
                  _dirnames.extend(keep_dirs)
                  for filename in sorted(_filenames):
                      tc_instances = []
                      file_name = os.path.join(tc_path, filename)
                      result += cls._create_from_file_name(tc_instances, file_name)
                      for _tc in tc_instances:
                          tcs[_tc.name] = _tc
          elif os.path.isfile(path):
              tc_instances = []
              result += cls._create_from_file_name(tc_instances, path)
              for _tc in tc_instances:
                  tcs[_tc.name] = _tc
          return result
  
      def _class_teardowns_run(self):
          """
          Runs all the class teardown methods for this testcase class
          """
          return self._methods_call("class_teardown")
  
  tc_c.driver_add(tc_c)
  
  
  def find(args):
      """
      Discover test cases in a list of paths
      """
      # discover test cases
      tcs = {}
      if len(args.testcase) == 0:
          logger.warning("No testcases specified, searching in "
                         "current directory, %s", os.getcwd())
          args.testcase = [ '.' ]
      for tc_path in args.testcase:
          tc_c.find_in_path(tcs, tc_path)
          for tcd in tc_c._tc_drivers:
              # If a driver has a different find function, use it to
              # find more
              tcd_find_in_path = getattr(tcd, "find_in_path", None)
              if tcd_find_in_path is not None and\
                 id(getattr(tcd_find_in_path, "im_func", tcd_find_in_path)) \
                 != id(tc_c.find_in_path.im_func):
                  tcd.find_in_path(tcs, tc_path)
      if len(tcs) == 0:
          logger.warning("No testcases found")
  
  
  tc_global = tc_c("toplevel", "", "builtin")
  tc_global.skip_reports = True
  
  def _testcase_match_tags(tc, tags_spec, origin = None):
      """
      :param str tags: string describing tag selection expression
      for :py:mod:`commonl.expr_parser`
      :param bool not_found_mistmatch: if a tag filter specifies a
        tag that is not found in the test case, treat it as a mismatch
        versus ignoring it.
      """
      if tags_spec == None:
          tc.report_info("selected by no-tag specification", dlevel = 4)
          return
      else:
          assert isinstance(tags_spec, basestring)
      if origin == None:
          origin = "[builtin]"
      kws = dict()
      for name, (value, _vorigin) in tc._tags.iteritems():
          kws[name] = value
      if not commonl.conditional_eval("testcase tag match", kws, tags_spec,
                                      origin, kind = "specification"):
          raise skip_e("because of tag specification '%s' @ %s" %
                       (tags_spec, origin), dict(dlevel = 4))
      tc.report_info("selected by tag specification '%s' @ %s" %
                     (tags_spec, origin), dlevel = 4)
  
  
  def testcases_discover(tcs_filtered, args):
      result = result_c(0, 0, 0, 0, 0)
  
      # discover test cases
      tcs_filtered.clear()
      if len(args.testcase) == 0 and len(args.manifest) == 0:
          logger.warning("No testcases specified, searching in "
                         "current directory, %s", os.getcwd())
          args.testcase = [ '.' ]
      tcs = {}
      tc_global.report_info("scanning for test cases", dlevel = 2)
  
      ignore_r = re.compile(r"^(\s*#.*|\s*)$")
      for manifest_file in args.manifest:
          try:
              with open(manifest_file) as manifest_fp:
                  for tc_path_line in manifest_fp:
                      if not ignore_r.match(tc_path_line):
                          args.testcase.append(tc_path_line.strip())
          except OSError:
              file_error = sys.exc_info()[1]
              logger.error("Error reading file: " + str(file_error))
              result.blocked += 1
  
      for tc_path in args.testcase:
          if not os.path.exists(tc_path):
              logger.error("%s: does not exist; ignoring", tc_path)
              continue
          result += tc_c.find_in_path(tcs, tc_path)
          for tcd in tc_c._tc_drivers:
              # If a driver has a different find function, use it to
              # find more
              tcd_find_in_path = getattr(tcd, "find_in_path", None)
              if tcd_find_in_path is not None and\
                 id(getattr(tcd_find_in_path, "im_func", tcd_find_in_path)) \
                 != id(tc_c.find_in_path.im_func):
                  result += tcd.find_in_path(tcs, tc_path)
      if len(tcs) == 0:
          logger.error("WARNING! No testcases found")
          return result
  
      # Now that we have them testcases, filter them based on the
      # tag filters specified in the command line with '-s'. Multiple's
      # are ORed together. Then for each testcase, apply the filter see
      # if it selects it or not.
      if args.tags_spec == []:
          tags_spec = None
      else:
          tags_spec = "(" + ") or (".join(args.tags_spec) +  ")"
  
      for tc_path, tc in tcs.iteritems():
          try:
              # This is a TC unit testcase aid
              if args.testcase_name and tc.name != args.testcase_name:
                  # Note this is only use for unit testing, so we don't
                  # account it in the list of skipped TCs
                  tc_global.report_info(
                      "ignoring because of testcase name '%s' not "
                      "matching args.testcase_name %s'"
                      % (tc.name, args.testcase_name),
                      dict(dlevel = 1))
                  continue
              _testcase_match_tags(tc, tags_spec, "command line")
              # Anything with a skip tag shall be skipped
              skip_value, skip_origin = tc.tag_get('skip', False)
              if skip_value != False:
                  if isinstance(skip_value, basestring):
                      raise skip_e("because of 'skip' tag @%s: %s"
                                   % (skip_origin, skip_value))
                  else:
                      raise skip_e("because of 'skip' tag @%s" % skip_origin)
              # Fill in any arguments from the command line
              # We will consider this testcase
              tcs_filtered[tc_path] = tc
          except exception as e:
              tc.mkticket()
              with msgid_c(tc.ticket, l = 4) as _msgid:
                  tc._ident = msgid_c.ident()
                  result += result_c.report_from_exception(tc, e)
  
      if not tcs_filtered:
          logger.error("All testcases skipped or filtered out by command "
                       "line -s options")
          return result
      logger.warning("Testcases filtered by command line to: %s",
                     ", ".join(tcs_filtered.keys()))
      return result
  
  def _targets_discover_local(targets_all):
      """
      Discover the 'local' target, used for running static tests
      """
      bsp = platform.machine()
      targets_all['local'] = {
          # FIXME: obtain from local system
          'bsp_models': { bsp: None },
          'bsps': {
              bsp: {
              },
          },
          'type': platform.system(),
          'id': 'local',
          'fullid': 'local',
          'url' : 'file://' + tcfl.origin_get_object_path(_targets_discover_local)
      }
  
  
  def _targets_discover(args, rt_all, rt_selected, ic_selected):
      """
      Do initial discovery of targets based on the command line options
      """
      rt_selected.clear()
      ic_selected.clear()
      rt_all.clear()
      _targets_discover_local(rt_all)
  
      rt_selected_all = {}
      ic_selected_all = {}
      # List all the targets available
      rt_all_list = ttb_client.rest_target_find_all(all_targets = args.all)
      if not rt_all_list:
          logger.error("WARNING! No targets available")
      for rt in rt_all_list:
          rt_all[rt['fullid']] = rt
          if 'interconnect_c' in rt.get('interfaces', []):
              ic_selected_all[rt['fullid']] = set(rt.get('bsp_models', {}).keys())
          else:
              rt_selected_all[rt['fullid']] = set(rt.get('bsp_models', {}).keys())
      del rt_all_list
  
      tc_global.report_info(
          "targets available: %s"
          % tc_global._selected_str(rt_selected_all), dlevel = 4)
      tc_global.report_info(
          "interconnects available: %s"
          % tc_global._selected_str(ic_selected_all), dlevel = 4)
  
      # Now filter based on the -t specs given in the command line
      # later each test case might filter more
      if not args.target:
          rt_selected.update(rt_selected_all)
          ic_selected.update(ic_selected_all)
      else:
          spec = "(" + ") or (".join(args.target) +  ")"
  
          # Select interconnects
          for rt_fullid, bsp_models in ic_selected_all.iteritems():
              rt = rt_all[rt_fullid]
              # Introduce two symbols after the ID and fullid, so "-t
              # TARGETNAME" works
              rt[rt_fullid] = True
              rt[rt['id']] = True
              for bsp_model in bsp_models:
                  if tc_global._targets_select_by_spec(
                          "command line", rt, bsp_model, spec, "command line"):
                      ic_selected.setdefault(rt_fullid, set()).add(bsp_model)
                      break
              else:
                  if tc_global._targets_select_by_spec(
                          "command line", rt, None, spec, "command line"):
                      ic_selected.setdefault(rt_fullid, set())
  
          # Select targets
          for rt_fullid, bsp_models in rt_selected_all.iteritems():
              rt = rt_all[rt_fullid]
              # Introduce two symbols after the ID and fullid, so "-t
              # TARGETNAME" works
              rt[rt_fullid] = True
              rt[rt['id']] = True
              for bsp_model in bsp_models:
                  if tc_global._targets_select_by_spec(
                          "command line", rt, bsp_model, spec, "command line"):
                      rt_selected.setdefault(rt_fullid, set()).add(bsp_model)
              if not bsp_models:
                  if tc_global._targets_select_by_spec(
                          "command line", rt, None, spec, "command line"):
                      rt_selected.setdefault(rt_fullid, set())
  
  
      tc_global.report_info(
          "targets selected by command line: %s"
          % tc_c._selected_str(rt_selected), dlevel = 4)
  
      tc_global.report_info(
          "interconnects selected by command line: %s"
          % tc_c._selected_str(ic_selected), dlevel = 4)
  
  
  def _run(args):
      """
      Runs one ore more test cases
      """
      _globals_init()
      if args.shard:
          try:
              if '/' in args.shard:
                  _shard, _shards = args.shard.split("/", 1)
              elif ':' in args.shard:
                  _shard, _shards = args.shard.split(":", 1)
              elif '-' in args.shard:
                  _shard, _shards = args.shard.split("-", 1)
              else:
                  raise ValueError
              shard = int(_shard)
              shards = int(_shards)
              if shard < 1 or shard > shards:
                  raise ValueError
          except:
              raise ValueError("%s: bad shard spec; expecting N/M, N:M or N-M "
                               "with 1 <= N <= M" % args.shard)
          shards_log = "-" + args.shard.replace("/", "-")
      else:
          shards_log = ""
          shards = 0
  
      tc_c._hash_salt = args.hash_salt
  
      # Is there a Run ID specified? or we asked to generate one? or none?
      global runid
      if args.id != None:
          if args.id == "":
              tc_c.runid_visible = ""
              tc_c.runid = msgid_c.generate()
          else:
              tc_c.runid_visible = args.id
              tc_c.runid = args.id
          if args.log_file == None:
              args.log_file = tc_c.runid + shards_log + ".log"
      else:
          tc_c.runid_visible = ""
  
      tc_c.max_permutations = args.max_permutations
  
      # Establish what is our log directory
      global log_dir
      if args.log_dir != None:
          if args.log_dir == "":
              if tc_c.runid == None:
                  logger.error("No log directory or RunID specified "
                               "(use -i or --log-dir)")
                  return 1
              log_dir = os.path.abspath(tc_c.runid)
          else:
              log_dir = os.path.abspath(args.log_dir)
          try:
              os.makedirs(log_dir)
          except OSError:
              # Lame check, as long as we have a dir, I don't care much
              if not os.path.isdir(log_dir):
                  raise
      else:
          log_dir = os.getcwd()
  
  
      # Create a tempdirectory for testcases to go wild
      # Why don't we change current directory to tc_c.tmpdir?
      #
      # - each testcase runs in a separate thread, and the change
      #   directory is per-process, so we can't have tstcases magically
      #   writing to CWD and not collide with each other
      #
      # - we want trash being written in a noticeable place, so it is
      #   noticed and fixed.
      #
      if args.tmpdir:
          commonl.check_dir_writeable(args.tmpdir,
                                      "testcases' run temporary directory")
          # We don't check if the tempdir is empty -- we might want to
          # reuse build stuff from a prev build and such -- it's up to
          # the user to use the right tempdir
          tc_c.tmpdir = os.path.abspath(args.tmpdir)
      else:
          # use defaul tc_c.tmpdir set in class tc_c
          if args.remove_tmpdir:
              atexit.register(shutil.rmtree, tc_c.tmpdir, True)
          else:
              atexit.register(
                  sys.stderr.write,
                  "I: %s: not removing temporary directory\n" % tc_c.tmpdir)
  
      # Now settle if we need a log file where to place it
      if args.log_file != None:
          # If no dir path, add one and anyway make it absolute
          if os.path.dirname(args.log_file) == '':
              log_file = os.path.join(log_dir, args.log_file)
          else:
              log_file = args.log_file
          log_file = os.path.abspath(log_file)
      else:
          log_file = None
  
      # if there is no log file given by the user *but* it did --tmpdir
      # or --no-remove-tmpdir, well, we are doing one by default
      if log_file == None and (args.tmpdir or not args.remove_tmpdir):
          log_file = os.path.join(tc_c.tmpdir, "run.log")
  
      # Init report drivers FIXME: need a hook to add more
      global report_console
      global report_failures
      report_console = report.report_console_c(
          args.verbosity - args.quietosity, log_dir, log_file,
          verbosity_logf = args.log_file_verbosity)
      report.report_c.driver_add(report_console)
      report_failures = report.file_c(log_dir)
      report.report_c.driver_add(report_failures)
  
      # Setup defaults in the base testcase class
      global ticket
      if args.ticket == '':	# default for tcf run means generate a ticket
          ticket = None
      elif args.ticket == ' ':	# default for tcf run means generate a ticket
          ticket = ""
      else:
          ticket = args.ticket	# Or use this ticket
      tc_c.release = args.release
      tc_c._dry_run = args.dry_run
      tc_c.eval_repeat = args.repeat_evaluation
      tc_c._targets_disabled_too = args.all
      tc_c._mode = args.mode
      if args.extra_report != None:
          tc_c._extra_report_format = args.extra_report
  
      # Which phases are we running?
      # Note these are class defaults--but there are test cases
      # or targets that might declare themselves build only (and thus
      # cancel deploy and eval, for example). So on __init__ we'll just
      # clone these two arrays, wasteful in memory as it is.
      tc_c._phases['build'] = set(['builtin-config'])
      tc_c._phases['deploy'] = set(['builtin-config'])
      tc_c._phases['eval'] = set(['builtin-config'])
      tc_c._phases['configure'] = set(['builtin-config'])
      tc_c._phases_skip['clean'] = set(['builtin-config'])
      if 'configure' in args.phases:
          tc_c._phases["configure"].add('command line')
          tc_c._phases_skip.pop('configure')
      if 'build' in args.phases:
          tc_c._phases['build'].add('command line')
      if 'deploy' in args.phases:
          tc_c._phases['build'].add('command line')
          tc_c._phases['deploy'].add('command line')
      if 'eval' in args.phases:
          tc_c._phases['build'].add('command line')
          tc_c._phases['deploy'].add('command line')
          tc_c._phases['eval'].add('command line')
      if 'clean' in args.phases:
          tc_c._phases['clean'].add('command line')
          tc_c._phases_skip.pop('clean')
  
      if 'configure' in args.phases_skip:
          tc_c._phases.pop('configure', None)
          tc_c._phases_skip["configure"].add('command line')
      if 'build' in args.phases_skip:
          tc_c._phases.pop('build', None)
          tc_c._phases_skip["build"].add('command line')
      if 'deploy' in args.phases_skip:
          tc_c._phases.pop('deploy', None)
          tc_c._phases_skip["deploy"].add('command line')
      if 'eval' in args.phases_skip:
          tc_c._phases.pop('eval', None)
          tc_c._phases_skip["eval"].add('command line')
      if 'clean' in args.phases_skip:
          tc_c._phases.pop('clean', None)
          tc_c._phases_skip["clean"].add('command line')
  
      with msgid_c(root = tc_c.runid, s = ""):
          tc_global.report_info("version %s"
                                % commonl.version_get(tcfl, "tcf"), dlevel = 3)
  
          tcs_filtered = {}
          result = testcases_discover(tcs_filtered, args)
          if shards:
              shard -= 1 # everything is easier zero based
              tcs_sorted = sorted(tcs_filtered.keys())
              tcs_shard = set(tcs_sorted[shard::shards])
              # Yes, there is a more efficient way to do this, but this
              # is cleaner
              tc_global.report_info(
                  "shard %s: selecting %s TCs of %d available" % (
                      args.shard, len(tcs_shard), len(tcs_sorted)),
                  dlevel = 2)
              for tc in set(tcs_sorted) - tcs_shard:
                  del tcs_filtered[tc]
  
          rt_all = {}
          rt_selected = {}
          ic_selected = {}
          _targets_discover(args, rt_all, rt_selected, ic_selected)
          if not rt_selected:
              logger.error("WARNING! No targets selected")
  
          tc_c._tcs_total = len(tcs_filtered)
          threads_no = int(args.threads)
          tp = _multiprocessing_tc_pool_c(processes = threads_no)
          # So now run as many testcases as possible
          threads = []
          time_start = time.time()
          tc_c.jobs = len(tcs_filtered)
          for tc in tcs_filtered.values():
              tc.mkticket()
              with msgid_c(tc.ticket, l = 4) as _msgid:
                  tc._ident = msgid_c.ident()
                  _threads = tc._run_on_targets(tp, rt_all,
                                                rt_selected, ic_selected)
              if isinstance(_threads, result_c):
                  result += _threads
              elif _threads == []:
                  # No targets could be found, SKIPPED
                  result += result_c(0, 0, 0, 0, 1)
              else:
                  threads += _threads
          tp.close()
          # FIXME: maybe we can use tp.imap_iter stuff
          tp.join()
          for thread in threads:
              for cls, retval in thread.get():
                  # Note we do this here vs. in tc_c._run() because that
                  # will be running in a different process.
                  cls.class_result += retval
                  result += retval
          time_end = time.time()
          # Run the class' teardown processes, if any -- mostly used for
          # self-testing. Note this is run in the top level process.
          seen_classes = set()
          for tc in tcs_filtered.values():
              cls = type(tc)
              if cls in seen_classes:
                  continue
              else:
                  seen_classes.add(cls)
                  with msgid_c(tc.ticket, l = 4, depth = 0,
                               phase = "class_teardown") as _msgid:
                      result += tc._class_teardowns_run()
          # If something failed or blocked, report totals verbosely
          tc_global.report_tweet(
              "%d tests (%d passed, %d error, %d failed, "
              "%d blocked, %d skipped, in %s) -"
              % (result.total(), result.passed, result.errors, result.failed,
                 result.blocked, result.skipped,
                 (datetime.timedelta(0, time_end - time_start))),
              result)
          tc_global.finalize(result)
  
          if result.errors or result.failed:
              return 1
          elif result.blocked > 0:
              return 127
          return 0
  
  def argp_setup(arg_subparsers):
      ap = arg_subparsers.add_parser("find",
                                     help = "List testcases")
      ap.add_argument(
          "-v", dest = "verbosity", action = "count", default = 0,
          help = "Increase information to display about the tests")
      ap.add_argument("-t", "--target", metavar = "TARGET",
                      help = "Test target on which to run the "
                      "test cases--if none, if will be autodetermined")
      ap.add_argument("testcase", metavar = "TEST-CASE",
                      nargs = "*",
                      help = "Test cases files or directories "
                      "where to find test cases")
      ap.set_defaults(func = find)
  
      ap = arg_subparsers.add_parser("run",
                                     help = "Run testcases")
      ap.add_argument(
          "-v", dest = "verbosity", action = "count", default = 0,
          help = "Increase information to display about the tests")
      ap.add_argument(
          "-q", dest = "quietosity", action = "count", default = 0,
          help = "Decrease information to display about the tests")
      ap.add_argument(
          "--no-remove-tmpdir", dest = "remove_tmpdir",
          action = "store_false", default = True,
          help = "Do not remove temporary directory upon exit")
      ap.add_argument(
          "--tmpdir", dest = "tmpdir", action = "store", type = str,
          default = None,
          help = "Directory where to place temporary files "
          "(they will not be removed upon exit; defaults to be "
          "autogenerated and removed upon exit")
      ap.add_argument(
          "--single-thread", action = "store_const", const = "1",
          dest = "threads",
          help = "Run things in a single thread of execution")
      ap.add_argument(
          "--threads", action = "store", type = int, dest = "threads",
          help = "Use this many threads of execution [%(default)s] for "
          "concurrent test case execution--note this can be way more than "
          "the number of CPUs in the system as it might be waiting for a "
          "targets to be available and that should not block buildin other "
          "test cases")
      ap.add_argument(
          "--make-j", action = "store", type = int,
          help = "Use this many threads of execution [%(default)s] "
          "for make (-jN)")
      ap.add_argument(
          "--log-file", action = "store", default = None,
          help = "Log full output to file")
      ap.add_argument(
          "--log-file-verbosity", action = "store", type = int, default = 999,
          metavar = "LOGLEVEL",
          help = "Unsigned integer describing the maximum verbosity of log "
          "file messages (0 is less, greater is more, defaults to 999, all)")
      ap.add_argument(
          "--log-dir", metavar = "DIRECTORY", action = "store", nargs = "?",
          type = str, const = "", default = None,
          help = "Write all log files and reports to said directory"
          "(defaults to autogenerated RunID)")
      ap.add_argument(
          "-u", action = "store_const", const = "all", dest = "mode",
          help = "Do an unlimited test run (on all targets of each type)")
      ap.add_argument(
          "-U", action = "store_const", const = "one-per-type", dest = "mode",
          help = "Do a limited test run (only one target of each type "
          "[default])")
      ap.add_argument(
          "-y", action = "store_const", const = "any", dest = "mode",
          help = "Do a limited test run on any target of any type")
      ap.add_argument(
          "-n", "--dry-run", action = "store_true", default = False,
          help = "Describe only what'd be run and where")
      ap.add_argument(
          "-i", "--id", action = "store", nargs = "?", metavar = "ID",
          type = str,
          const = "",	# Means auto-generate one
          default = None,
          help = "Store a run ID, accesible to the reporting engine (this "
          "also will create a log file ID.log)")
      ap.add_argument("-t", "--target", metavar = "TARGETSPECs",
                      action = "append", default = [],
                      help = "Test target in which to run the test case; "
                      "can be NAME, NAME:BSP-MODEL or TAG:PYTHONREGEX, where "
                      "all the target/bsp-models that match said TAG and regex "
                      "will be used--please note that each test case may "
                      "specify extra filtering that will be applied to these. "
                      "Multiple TARGETSPECs in a single string are ANDed "
                      "together; multiple -t TARGETSPECs are ORed together.")
      ap.add_argument(
          "--configure", "-c", action = "append_const",
          dest = 'phases', const = "configure",
          help = "Tell the testcases to run configure phase "
          " (off by default)")
      ap.add_argument(
          "--build", "-b", action = "append_const",
          dest = 'phases', const = "build",
          help = "Tell the testcases to run the build phase"
          " (default)")
      ap.add_argument(
          "--deploy", "-d", action = "append_const",
          dest = 'phases', const = "deploy",
          help = "Tell the testcases to run until the deploy phase"
          " (default)")
      ap.add_argument(
          "--eval", "-e", action = "append_const",
          dest = 'phases', const = "eval",
          help = "Tell the testcases to run until the eval phase"
          " (default)")
      ap.add_argument(
          "--skip-configure", "-C", action = "append_const",
          dest = 'phases_skip', const = "configure",
          help = "Tell the testcases to skip the configure phase")
      ap.add_argument(
          "--skip-build", "-B", action = "append_const",
          dest = 'phases_skip', const = "build",
          help = "Tell the testcases to skip the build phase")
      ap.add_argument(
          "--skip-deploy", "-D", action = "append_const",
          dest = 'phases_skip', const = "deploy",
          help = "Tell the testcases to skip the deploy phase")
      ap.add_argument(
          "--skip-eval", "-E", action = "append_const",
          dest = 'phases_skip', const = "eval",
          help = "Tell the testcases to skip the eval phase")
      ap.add_argument(
          "--clean", "-l", action = "append_const",
          dest = 'phases', const = "clean",
          help = "Tell the testcases to clean after themselves"
          " (off by default)")
      ap.add_argument(
          "-s", action = "append", dest = 'tags_spec', default = [],
          help = "Specify testcase filters, based on the testcase tags; "
          "multiple tags specifications are ORed together")
      ap.add_argument(
          "-r", "--repeat-evaluation",
          action = "store", type = int, default = 1,
          help = "How many times to repeat the evaluation phase "
          " (%(default)d by default); this is used for stress or "
          "MTBF testing without having to re-deploy the target")
      ap.add_argument(
          "--no-release", dest = "release", action = "store_false",
          default = True,
          help = "(internal) do not release targets once complete")
      ap.add_argument(
          "-P", "--max-permutations",
          action = "store", type = int, default = 10,
          help = "Maximum number of permutuations of targets for a "
          "single test that shall be considered")
      ap.add_argument(
          "--extra-report", action = "store", default = None,
          help = "Specify extra format information that is to be printed for "
          "failed testcases; fields are the same as for the rules "
          "%%(FIELDNAME)s (or d, or whichever type the field is)")
      ap.add_argument(
          "-a", "--all", action = "store_true", default = False,
          help = "Consider also disabled targets")
      ap.add_argument("-m", "--manifest", metavar = "MANIFESTFILEs",
                      action = "append", default = [],
                      help = "Specify one or more manifest files containing "
                      "test case paths (one per line) or '#' prefixed comment "
                      "lines. Test case paths are acumulated, keeping other "
                      "specified paths")
      ap.add_argument(
          "--shard", metavar = "N/M", action = "store", type = str,
          default = None,
          help = "Divide the sorted list of discovered testcases in "
          "M pieces and consider on the Nth")
      ap.add_argument(
          "--hash-salt", metavar = "SALT", action = "store", type = str,
          default = "",
          help = "Add SALT to the string used to compute the "
          "testcase's hash ID; useful when multiple runs of the same "
          "cases and targets are going to be run with different external "
          "variations, such as different compiler or library versions")
      ap.add_argument("testcase", metavar = "TEST-CASE",
                      nargs = "*",
                      help = "Files describing testcases")
      ap.set_defaults(func = _run,
                      testcase_name = None,
                      phases_skip = [], phases = [], tc_filter_specs = [],
                      threads = 4 * _multiprocessing.cpu_count(),
                      make_j = 2 * _multiprocessing.cpu_count(),
                      mode = 'one-per-type')
  
  # Get drivers registered
  
  # FIXME: move to config if sketch package is installed
  import tcfl.app_sketch
  tcfl.app.driver_add(tcfl.app_sketch.app_sketch)
  
  import app_manual
  tcfl.app.driver_add(app_manual.app_manual)
  
  # Target API extensions
  import target_ext_broker_files
  target_c.extension_register(target_ext_broker_files.broker_files)
  import target_ext_console
  target_c.extension_register(target_ext_console.console)
  import target_ext_power
  target_c.extension_register(target_ext_power.power)
  import target_ext_images
  target_c.extension_register(target_ext_images.images)
  import target_ext_debug
  target_c.extension_register(target_ext_debug.debug)
  import target_ext_tcob
  target_c.extension_register(target_ext_tcob.tcob)
  import target_ext_tunnel
  target_c.extension_register(target_ext_tunnel.tunnel)
  import target_ext_shell
  target_c.extension_register(target_ext_shell.shell)
  import target_ext_ssh
  target_c.extension_register(target_ext_ssh.ssh)
  import target_ext_ioc_flash_server_app
  target_c.extension_register(target_ext_ioc_flash_server_app.extension)
  import target_ext_buttons
  target_c.extension_register(target_ext_buttons.buttons)
  import target_ext_fastboot
  target_c.extension_register(target_ext_fastboot.fastboot)
  import pos
  target_c.extension_register(pos.extension, "pos")
  