text: #!/usr/bin/env python
  # Copyright 2015 The Chromium Authors. All rights reserved.
  # Use of this source code is governed by a BSD-style license that can be
  # found in the LICENSE file.
  
  """Toolbox to manage all the json files in this directory.
  
  It can reformat them in their canonical format or ensures they are well
  formatted.
  """
  
  import argparse
  import ast
  import collections
  import glob
  import json
  import os
  import subprocess
  import sys
  
  
  THIS_DIR = os.path.dirname(os.path.abspath(__file__))
  SRC_DIR = os.path.dirname(os.path.dirname(THIS_DIR))
  BLINK_DIR = os.path.join(SRC_DIR, 'third_party', 'WebKit')
  sys.path.insert(0, os.path.join(SRC_DIR, 'third_party', 'colorama', 'src'))
  
  import colorama
  
  
  SKIP = {
    # These are not 'builders'.
    'compile_targets', 'gtest_tests', 'filter_compile_builders',
    'non_filter_builders', 'non_filter_tests_builders',
  
    # These are not supported on Swarming yet.
  
    # Android Cloud is still experimental and involves spinning up an Android
    # instance on GCE.  Swarming doesn't work in that environment yet.
    'Android Cloud Tests',
  
    # Recipes don't promise execution on jelly bean.  This could work if the
    # OS dimensions go into the recipe, they're set in the json file, and
    # jelly bean devices are in the pool.  For now, just blacklist.
    'Jelly Bean Tester',
    'Lollipop Consumer Tester',
    'Lollipop Low-end Tester',
  
    # Android bots need custom dimension_sets entries for swarming, and capacity
    # is not there yet -- so don't let manage.py add swarming automatically there.
    'Android User Builder Tests',
    'Android GN',
  
    # http://crbug.com/441429
    'Linux Trusty (32)', 'Linux Trusty (dbg)(32)',
  
    # Swarming may not work on Mac10.10,11,12; need to
    # re-investigate and confirm.
    'WebKit Mac10.10',
    'WebKit Mac10.11',
    'WebKit Mac10.12',
    'WebKit Mac10.11 (dbg)',
    'WebKit Mac10.13 (retina)',
    'Chromium Mac10.10 Tests',
    'Chromium Mac10.11 Tests',
  
    # One off builders. Note that Swarming does support ARM.
    'Linux ARM Cross-Compile',
    'Site Isolation Android',
    'Site Isolation Linux',
    'Site Isolation Win',
  }
  
  
  SKIP_GN_ISOLATE_MAP_TARGETS = {
    # This target is magic and not present in gn_isolate_map.pyl.
    'all',
    'remoting/client:client',
    'remoting/host:host',
  
    # These targets are listed only in build-side recipes.
    'All_syzygy',
    'blink_tests',
    'captured_sites_interactive_tests',
    'cast_shell',
    'cast_shell_apk',
    'chrome_official_builder',
    'chrome_official_builder_no_unittests',
    'chrome_sandbox',
    'chromium_builder_asan',
    'chromium_builder_perf',
    'chromiumos_preflight',
    'linux_symbols',
    'mini_installer',
    'previous_version_mini_installer',
    'symupload',
  
    # iOS tests are listed in //ios/build/bots.
    'cronet_test',
    'cronet_unittests_ios',
    'ios_chrome_bookmarks_egtests',
    'ios_chrome_integration_egtests',
    'ios_chrome_manual_fill_egtests',
    'ios_chrome_reading_list_egtests',
    'ios_chrome_settings_egtests',
    'ios_chrome_smoke_egtests',
    'ios_chrome_ui_egtests',
    'ios_chrome_unittests',
    'ios_chrome_web_egtests',
    'ios_components_unittests',
    'ios_net_unittests',
    "ios_remoting_unittests",
    'ios_showcase_egtests',
    'ios_web_inttests',
    'ios_web_shell_egtests',
    'ios_web_unittests',
    'ios_web_view_inttests',
    'ios_web_view_unittests',
    'ocmock_support_unittests',
  
    # These are listed in Builders that are skipped for other reasons.
    'chrome_junit_tests',
    'components_background_task_scheduler_junit_tests',
    'components_gcm_driver_junit_tests',
    'components_invalidation_impl_junit_tests',
    'components_policy_junit_tests',
    'components_variations_junit_tests',
    'components_web_restrictions_junit_tests',
    'content_junit_tests',
    'content_junit_tests',
    'device_junit_tests',
    'junit_unit_tests',
    'media_router_e2e_tests',
    'media_router_junit_tests',
    'media_router_perf_tests',
    'motopho_latency_test',
    'net_junit_tests',
    'net_junit_tests',
    'service_junit_tests',
    'shipped_binaries',
    'system_webview_apk',
    'ui_junit_tests',
    'vr_common_perftests',
    'vr_perf_tests',
    'vrcore_fps_test',
    'webapk_client_junit_tests',
    'webapk_shell_apk_h2o_junit_tests',
    'webapk_shell_apk_junit_tests',
  
    # These tests are only run on WebRTC CI.
    'AppRTCMobileTest',
    'android_junit_tests',
    'audio_decoder_unittests',
    'common_audio_unittests',
    'common_video_unittests',
    'frame_analyzer',
    'libjingle_peerconnection_android_unittest',
    'modules_tests',
    'modules_unittests',
    'peerconnection_unittests',
    'rtc_media_unittests',
    'rtc_pc_unittests',
    'rtc_stats_unittests',
    'rtc_unittests',
    'system_wrappers_unittests',
    'test_support_unittests',
    'tools_unittests',
    'video_engine_tests',
    'voice_engine_unittests',
    'webrtc_nonparallel_tests',
    'xmllite_xmpp_unittests',
  
    # isolate is currently too slow for this target.
    # http://crbug.com/524758
    'webkit_layout_tests',
    'webkit_layout_tests_exparchive',
  
    # These are only run on V8 CI.
    'pdfium_test',
    'postmortem-metadata',
  
    # These are only for developer convenience and not on any bots.
    'telemetry_gpu_integration_test_scripts_only',
  
    # These are defined by an android internal gn_isolate_map.pyl file.
    'chrome_apk',
  }
  
  
  class Error(Exception):
    """Processing error."""
  
  
  def get_isolates():
    """Returns the list of all isolate files."""
  
    def git_ls_files(cwd):
      return subprocess.check_output(['git', 'ls-files'], cwd=cwd).splitlines()
  
    files = git_ls_files(SRC_DIR) + git_ls_files(BLINK_DIR)
    return [os.path.basename(f) for f in files if f.endswith('.isolate')]
  
  
  def process_builder_convert(data, test_name):
    """Converts 'test_name' to run on Swarming in 'data'.
  
    Returns True if 'test_name' was found.
    """
    result = False
    for test in data['gtest_tests']:
      if test['test'] != test_name:
        continue
      test.setdefault('swarming', {})
      if not test['swarming'].get('can_use_on_swarming_builders'):
        test['swarming']['can_use_on_swarming_builders'] = True
      result = True
    return result
  
  
  def process_builder_remaining(data, filename, builder, tests_location):
    """Calculates tests_location when mode is --remaining."""
    for test in data['gtest_tests']:
      name = test['test']
      if test.get('swarming', {}).get('can_use_on_swarming_builders'):
        tests_location[name]['count_run_on_swarming'] += 1
      else:
        tests_location[name]['count_run_local'] += 1
        tests_location[name]['local_configs'].setdefault(
            filename, []).append(builder)
  
  
  def process_file(mode, test_name, tests_location, filepath, ninja_targets,
                   ninja_targets_seen):
    """Processes a json file describing what tests should be run for each recipe.
  
    The action depends on mode. Updates tests_location.
  
    Return False if the process exit code should be 1.
    """
    filename = os.path.basename(filepath)
    with open(filepath) as f:
      content = f.read()
    try:
      config = json.loads(content)
    except ValueError as e:
      raise Error('Exception raised while checking %s: %s' % (filepath, e))
  
    for builder, data in sorted(config.iteritems()):
      if builder in SKIP:
        # Oddities.
        continue
      if not isinstance(data, dict):
        raise Error('%s: %s is broken: %s' % (filename, builder, data))
      if ('gtest_tests' not in data and
          'isolated_scripts' not in data and
          'additional_compile_targets' not in data and
          'instrumentation_tests' not in data):
        continue
  
      for d in data.get('junit_tests', []):
        test = d['test']
        if (test not in ninja_targets and
            test not in SKIP_GN_ISOLATE_MAP_TARGETS):
          raise Error('%s: %s / %s is not listed in gn_isolate_map.pyl' %
                      (filename, builder, test))
        elif test in ninja_targets:
          ninja_targets_seen.add(test)
  
      for target in data.get('additional_compile_targets', []):
        if (target not in ninja_targets and
            target not in SKIP_GN_ISOLATE_MAP_TARGETS):
          raise Error('%s: %s / %s is not listed in gn_isolate_map.pyl' %
                      (filename, builder, target))
        elif target in ninja_targets:
          ninja_targets_seen.add(target)
  
      gtest_tests = data.get('gtest_tests', [])
      if not isinstance(gtest_tests, list):
        raise Error(
            '%s: %s is broken: %s' % (filename, builder, gtest_tests))
      if not all(isinstance(g, dict) for g in gtest_tests):
        raise Error(
            '%s: %s is broken: %s' % (filename, builder, gtest_tests))
  
      seen = set()
      for d in gtest_tests:
        test = d['test']
        if (test not in ninja_targets and
            test not in SKIP_GN_ISOLATE_MAP_TARGETS):
          raise Error('%s: %s / %s is not listed in gn_isolate_map.pyl.' %
                      (filename, builder, test))
        elif test in ninja_targets:
          ninja_targets_seen.add(test)
  
        name = d.get('name', d['test'])
        if name in seen:
          raise Error('%s: %s / %s is listed multiple times.' %
                      (filename, builder, name))
        seen.add(name)
        d.setdefault('swarming', {}).setdefault(
            'can_use_on_swarming_builders', False)
  
      if gtest_tests:
        config[builder]['gtest_tests'] = sorted(
            gtest_tests, key=lambda x: x['test'])
  
      for d in data.get('isolated_scripts', []):
        name = d['isolate_name']
        if (name not in ninja_targets and
            name not in SKIP_GN_ISOLATE_MAP_TARGETS):
          raise Error('%s: %s / %s is not listed in gn_isolate_map.pyl.' %
                      (filename, builder, name))
        elif name in ninja_targets:
          ninja_targets_seen.add(name)
  
      for d in data.get('instrumentation_tests', []):
        name = d['test']
        if (name not in ninja_targets and
            name not in SKIP_GN_ISOLATE_MAP_TARGETS):
          raise Error('%s: %s / %s is not listed in gn_isolate_map.pyl.' %
                      (filename, builder, name))
        elif name in ninja_targets:
          ninja_targets_seen.add(name)
  
      # The trick here is that process_builder_remaining() is called before
      # process_builder_convert() so tests_location can be used to know how many
      # tests were converted.
      if mode in ('convert', 'remaining'):
        process_builder_remaining(data, filename, builder, tests_location)
      if mode == 'convert':
        process_builder_convert(data, test_name)
  
    expected = json.dumps(
        config, sort_keys=True, indent=2, separators=(',', ': ')) + '\n'
    if content != expected:
      if mode in ('convert', 'write'):
        with open(filepath, 'wb') as f:
          f.write(expected)
        if mode == 'write':
          print('Updated %s' % filename)
      else:
        print('%s is not in canonical format' % filename)
        print('run `testing/buildbot/manage.py -w` to fix')
      return mode != 'check'
    return True
  
  
  def print_convert(test_name, tests_location):
    """Prints statistics for a test being converted for use in a CL description.
    """
    data = tests_location[test_name]
    print('Convert %s to run exclusively on Swarming' % test_name)
    print('')
    print('%d configs already ran on Swarming' % data['count_run_on_swarming'])
    print('%d used to run locally and were converted:' % data['count_run_local'])
    for master, builders in sorted(data['local_configs'].iteritems()):
      for builder in builders:
        print('- %s: %s' % (master, builder))
    print('')
    print('Ran:')
    print('  ./manage.py --convert %s' % test_name)
    print('')
    print('R=')
    print('BUG=98637')
  
  
  def print_remaining(test_name, tests_location):
    """Prints a visual summary of what tests are yet to be converted to run on
    Swarming.
    """
    if test_name:
      if test_name not in tests_location:
        raise Error('Unknown test %s' % test_name)
      for config, builders in sorted(
          tests_location[test_name]['local_configs'].iteritems()):
        print('%s:' % config)
        for builder in sorted(builders):
          print('  %s' % builder)
      return
  
    isolates = get_isolates()
    l = max(map(len, tests_location))
    print('%-*s%sLocal       %sSwarming  %sMissing isolate' %
        (l, 'Test', colorama.Fore.RED, colorama.Fore.GREEN,
          colorama.Fore.MAGENTA))
    total_local = 0
    total_swarming = 0
    for name, location in sorted(tests_location.iteritems()):
      if not location['count_run_on_swarming']:
        c = colorama.Fore.RED
      elif location['count_run_local']:
        c = colorama.Fore.YELLOW
      else:
        c = colorama.Fore.GREEN
      total_local += location['count_run_local']
      total_swarming += location['count_run_on_swarming']
      missing_isolate = ''
      if name + '.isolate' not in isolates:
        missing_isolate = colorama.Fore.MAGENTA + '*'
      print('%s%-*s %4d           %4d    %s' %
          (c, l, name, location['count_run_local'],
            location['count_run_on_swarming'], missing_isolate))
  
    total = total_local + total_swarming
    p_local = 100. * total_local / total
    p_swarming = 100. * total_swarming / total
    print('%s%-*s %4d (%4.1f%%)   %4d (%4.1f%%)' %
        (colorama.Fore.WHITE, l, 'Total:', total_local, p_local,
          total_swarming, p_swarming))
    print('%-*s                %4d' % (l, 'Total executions:', total))
  
  
  def main():
    colorama.init()
    parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__)
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument(
        '-c', '--check', dest='mode', action='store_const', const='check',
        default='check', help='Only check the files')
    group.add_argument(
        '--convert', dest='mode', action='store_const', const='convert',
        help='Convert a test to run on Swarming everywhere')
    group.add_argument(
        '--remaining', dest='mode', action='store_const', const='remaining',
        help='Count the number of tests not yet running on Swarming')
    group.add_argument(
        '-w', '--write', dest='mode', action='store_const', const='write',
        help='Rewrite the files')
    parser.add_argument(
        'test_name', nargs='?',
        help='The test name to print which configs to update; only to be used '
             'with --remaining')
    args = parser.parse_args()
  
    if args.mode == 'convert':
      if not args.test_name:
        parser.error('A test name is required with --convert')
      if args.test_name + '.isolate' not in get_isolates():
        parser.error('Create %s.isolate first' % args.test_name)
  
    # Stats when running in --remaining mode;
    tests_location = collections.defaultdict(
        lambda: {
          'count_run_local': 0, 'count_run_on_swarming': 0, 'local_configs': {}
        })
  
    with open(os.path.join(THIS_DIR, "gn_isolate_map.pyl")) as fp:
      gn_isolate_map = ast.literal_eval(fp.read())
      ninja_targets = {k: v['label'] for k, v in gn_isolate_map.items()}
  
    try:
      result = 0
      ninja_targets_seen = set()
      for filepath in glob.glob(os.path.join(THIS_DIR, '*.json')):
        if not process_file(args.mode, args.test_name, tests_location, filepath,
                            ninja_targets, ninja_targets_seen):
          result = 1
  
      skip_targets = [k for k, v in gn_isolate_map.items() if
                      ('skip_usage_check' in v and v['skip_usage_check'])]
      extra_targets = (set(ninja_targets) - set(skip_targets) -
                       ninja_targets_seen - SKIP_GN_ISOLATE_MAP_TARGETS)
      if extra_targets:
        if len(extra_targets) > 1:
          extra_targets_str = ', '.join(extra_targets) + ' are'
        else:
          extra_targets_str = list(extra_targets)[0] + ' is'
        raise Error('%s listed in gn_isolate_map.pyl but not in any .json '
                    'files' % extra_targets_str)
  
      if args.mode == 'convert':
        print_convert(args.test_name, tests_location)
      elif args.mode == 'remaining':
        print_remaining(args.test_name, tests_location)
      return result
    except Error as e:
      sys.stderr.write('%s\n' % e)
      return 1
  
  
  if __name__ == "__main__":
    sys.exit(main())
  