text: # Copyright 2014 The Chromium Authors. All rights reserved.
  # Use of this source code is governed by a BSD-style license that can be
  # found in the LICENSE file.
  
  import logging
  import optparse
  
  from telemetry import decorators
  from telemetry.internal import story_runner
  from telemetry.internal.util import command_line
  from telemetry.page import legacy_page_test
  from telemetry.story import expectations as expectations_module
  from telemetry.web_perf import story_test
  from telemetry.web_perf import timeline_based_measurement
  
  Info = decorators.Info
  
  # TODO(crbug.com/859524): remove this once we update all the benchmarks in
  # tools/perf to use Info decorator.
  Owner = decorators.Info # pylint: disable=invalid-name
  
  
  class InvalidOptionsError(Exception):
    """Raised for invalid benchmark options."""
    pass
  
  
  class Benchmark(command_line.Command):
    """Base class for a Telemetry benchmark.
  
    A benchmark packages a measurement and a PageSet together.
    Benchmarks default to using TBM unless you override the value of
    Benchmark.test, or override the CreatePageTest method.
  
    New benchmarks should override CreateStorySet.
    """
    options = {}
    page_set = None
    test = timeline_based_measurement.TimelineBasedMeasurement
    SUPPORTED_PLATFORMS = [expectations_module.ALL]
    SUPPORTED_PLATFORM_TAGS = []
  
    def __init__(self, max_failures=None):
      """Creates a new Benchmark.
  
      Args:
        max_failures: The number of story run's failures before bailing
            from executing subsequent page runs. If None, we never bail.
      """
      self._max_failures = max_failures
      # TODO: There should be an assertion here that checks that only one of
      # the following is true:
      # * It's a TBM benchmark, with CreateCoreTimelineBasedMeasurementOptions
      #   defined.
      # * It's a legacy benchmark, with either CreatePageTest defined or
      #   Benchmark.test set.
      # See https://github.com/catapult-project/catapult/issues/3708
  
    def CanRunOnPlatform(self, platform, finder_options):
      """Figures out if the benchmark is meant to support this platform.
  
      This is based on the SUPPORTED_PLATFORMS class member of the benchmark.
  
      This method should not be overriden or called outside of the Telemetry
      framework.
  
      Note that finder_options object in practice sometimes is actually not
      a BrowserFinderOptions object but a PossibleBrowser object.
      The key is that it can be passed to ShouldDisable, which only uses
      finder_options.browser_type, which is available on both PossibleBrowser
      and BrowserFinderOptions.
      """
      for p in self.SUPPORTED_PLATFORMS:
        # This is reusing StoryExpectation code, so it is a bit unintuitive. We
        # are trying to detect the opposite of the usual case in StoryExpectations
        # so we want to return True when ShouldDisable returns true, even though
        # we do not want to disable.
        if p.ShouldDisable(platform, finder_options):
          return True
      return False
  
    def Run(self, finder_options):
      """Do not override this method."""
      finder_options.target_platforms = self.GetSupportedPlatformNames(
          self.SUPPORTED_PLATFORMS)
      return story_runner.RunBenchmark(self, finder_options)
  
    @property
    def max_failures(self):
      return self._max_failures
  
    @classmethod
    def Name(cls):
      return '%s.%s' % (cls.__module__.split('.')[-1], cls.__name__)
  
    @classmethod
    def AddCommandLineArgs(cls, parser):
      group = optparse.OptionGroup(parser, '%s test options' % cls.Name())
      cls.AddBenchmarkCommandLineArgs(group)
      if group.option_list:
        parser.add_option_group(group)
  
    @classmethod
    def AddBenchmarkCommandLineArgs(cls, group):
      del group  # unused
  
    @classmethod
    def GetSupportedPlatformNames(cls, supported_platforms):
      """Returns a list of platforms supported by this benchmark.
  
      Returns:
        A set of names of supported platforms. The supported platforms are a list
        of strings that would match possible values from platform.GetOsName().
      """
      result = set()
      for p in supported_platforms:
        result.update(p.GetSupportedPlatformNames())
      return frozenset(result)
  
    @classmethod
    def SetArgumentDefaults(cls, parser):
      default_values = parser.get_default_values()
      invalid_options = [o for o in cls.options if not hasattr(default_values, o)]
      if invalid_options:
        raise InvalidOptionsError(
            'Invalid benchmark options: %s' % ', '.join(invalid_options))
      parser.set_defaults(**cls.options)
  
    @classmethod
    def ProcessCommandLineArgs(cls, parser, args):
      pass
  
    def CustomizeOptions(self, finder_options, possible_browser=None):
      """Add options that are required by this benchmark."""
  
    def GetBugComponents(self):
      """Return the benchmark's Monorail component as a string."""
      return decorators.GetComponent(self)
  
    def GetOwners(self):
      """Return the benchmark's owners' emails in a list."""
      return decorators.GetEmails(self)
  
    def GetDocumentationLinks(self):
      """Return the benchmark's documentation links.
  
      Returns:
        A list of [title, url] pairs. This is the form that allows Dashboard
        to display links properly.
      """
      links = []
      url = decorators.GetDocumentationLink(self)
      if url is not None:
        links.append(['Benchmark documentation link', url])
      return links
  
    def GetInfoBlurb(self):
      """Return any info blurb associated with the the benchmark"""
      return decorators.GetInfoBlurb(self)
  
    def CreateCoreTimelineBasedMeasurementOptions(self):
      """Return the base TimelineBasedMeasurementOptions for this Benchmark.
  
      Additional chrome and atrace categories can be appended when running the
      benchmark with the --extra-chrome-categories and --extra-atrace-categories
      flags.
  
      Override this method to configure a TimelineBasedMeasurement benchmark. If
      this is not a TimelineBasedMeasurement benchmark, override CreatePageTest
      for PageTest tests. Do not override both methods.
      """
      return timeline_based_measurement.Options()
  
    def _GetTimelineBasedMeasurementOptions(self, options):
      """Return all timeline based measurements for the curren benchmark run.
  
      This includes the benchmark-configured measurements in
      CreateCoreTimelineBasedMeasurementOptions as well as the user-flag-
      configured options from --extra-chrome-categories and
      --extra-atrace-categories.
      """
      tbm_options = self.CreateCoreTimelineBasedMeasurementOptions()
      if options and options.extra_chrome_categories:
        # If Chrome tracing categories for this benchmark are not already
        # enabled, there is probably a good reason why. Don't change whether
        # Chrome tracing is enabled.
        assert tbm_options.config.enable_chrome_trace, (
            'This benchmark does not support Chrome tracing.')
        tbm_options.config.chrome_trace_config.category_filter.AddFilterString(
            options.extra_chrome_categories)
      if options and options.extra_atrace_categories:
        # Many benchmarks on Android run without atrace by default. Hopefully the
        # user understands that atrace is only supported on Android when setting
        # this option.
        tbm_options.config.enable_atrace_trace = True
  
        categories = tbm_options.config.atrace_config.categories
        if isinstance(categories, basestring):
          # Categories can either be a list or comma-separated string.
          # https://github.com/catapult-project/catapult/issues/3712
          categories = categories.split(',')
        for category in options.extra_atrace_categories.split(','):
          if category not in categories:
            categories.append(category)
        tbm_options.config.atrace_config.categories = categories
      if options and options.enable_systrace:
        tbm_options.config.chrome_trace_config.SetEnableSystrace()
      legacy_json_format = options and options.legacy_json_trace_format
      if legacy_json_format:
        tbm_options.config.chrome_trace_config.SetJsonTraceFormat()
      else:
        tbm_options.config.chrome_trace_config.SetProtoTraceFormat()
      if options and options.experimental_system_tracing:
        assert not legacy_json_format
        logging.warning('Enabling experimental system tracing!')
        tbm_options.config.enable_experimental_system_tracing = True
        tbm_options.config.system_trace_config.EnableChrome(
            chrome_trace_config=tbm_options.config.chrome_trace_config)
      if options and options.experimental_system_data_sources:
        assert not legacy_json_format
        tbm_options.config.enable_experimental_system_tracing = True
        tbm_options.config.system_trace_config.EnablePower()
        tbm_options.config.system_trace_config.EnableSysStatsCpu()
        tbm_options.config.system_trace_config.EnableFtraceCpu()
        tbm_options.config.system_trace_config.EnableFtraceSched()
  
      if options and options.force_sideload_perfetto:
        assert tbm_options.config.enable_experimental_system_tracing
        tbm_options.config.force_sideload_perfetto = True
  
      # TODO(crbug.com/1012687): Remove or adjust the following warnings as the
      # development of TBMv3 progresses.
      tbmv3_metrics = [m[6:] for m in tbm_options.GetTimelineBasedMetrics()
                       if m.startswith('tbmv3:')]
      if tbmv3_metrics:
        if legacy_json_format:
          logging.warning(
              'Selected TBMv3 metrics will not be computed because they are not '
              "supported in Chrome's JSON trace format.")
        else:
          logging.warning(
              'The following TBMv3 metrics have been selected to run: %s. '
              'Please note that TBMv3 is an experimental feature in active '
              'development, and may not be supported in the future in its '
              'current form. Follow crbug.com/1012687 for updates and to '
              'discuss your use case before deciding to rely on this feature.',
              ', '.join(tbmv3_metrics))
      return tbm_options
  
    def CreatePageTest(self, options):  # pylint: disable=unused-argument
      """Return the PageTest for this Benchmark.
  
      Override this method for PageTest tests.
      Override, CreateCoreTimelineBasedMeasurementOptions to configure
      TimelineBasedMeasurement tests. Do not override both methods.
  
      Args:
        options: a browser_options.BrowserFinderOptions instance
      Returns:
        |test()| if |test| is a PageTest class.
        Otherwise, a TimelineBasedMeasurement instance.
      """
      is_page_test = issubclass(self.test, legacy_page_test.LegacyPageTest)
      is_story_test = issubclass(self.test, story_test.StoryTest)
      if not is_page_test and not is_story_test:
        raise TypeError('"%s" is not a PageTest or a StoryTest.' %
                        self.test.__name__)
      if is_page_test:
        # TODO: assert that CreateCoreTimelineBasedMeasurementOptions is not
        # defined. That's incorrect for a page test. See
        # https://github.com/catapult-project/catapult/issues/3708
        return self.test()  # pylint: disable=no-value-for-parameter
  
      opts = self._GetTimelineBasedMeasurementOptions(options)
      return self.test(opts)
  
    def CreateStorySet(self, options):
      """Creates the instance of StorySet used to run the benchmark.
  
      Can be overridden by subclasses.
      """
      del options  # unused
      # TODO(aiolos, nednguyen, eakufner): replace class attribute page_set with
      # story_set.
      if not self.page_set:
        raise NotImplementedError('This test has no "page_set" attribute.')
      return self.page_set()  # pylint: disable=not-callable
  