tools/perf/generate_perf_sharding.py
def _ValidateShardMaps(args):
  """Validate that the shard maps, csv files, etc. are consistent."""
  del args
  errors = []

  tempdir = tempfile.mkdtemp()
  try:
    builders = _GetBuilderPlatforms(builders=None, waterfall='all')
    for builder in builders:
      output_file = os.path.join(
          tempdir, os.path.basename(builder.timing_file_path))
      _FilterTimingData(builder, output_file)
      if not filecmp.cmp(builder.timing_file_path, output_file):
        errors.append(
            '{timing_data} is not up to date. Please run '
            '`./generate_perf_sharding.py update-timing --filter-only` '
            'to regenerate it.'.format(timing_data=builder.timing_file_path))
  finally:
    shutil.rmtree(tempdir)

  # Check that bot_platforms.py matches the actual shard maps
  for platform in bot_platforms.ALL_PLATFORMS:
    platform_benchmark_names = set(
        b.name for b in platform.benchmark_configs) | set(
            e.name for e in platform.executables)
    shard_map_benchmark_names = _ParseBenchmarks(platform.shards_map_file_path)
    for benchmark in platform_benchmark_names - shard_map_benchmark_names:
      errors.append(
          'Benchmark {benchmark} is supposed to be scheduled on platform '
          '{platform} according to '
          'bot_platforms.py, but it is not yet scheduled. If this is a new '
          'benchmark, please rename it to UNSCHEDULED_{benchmark}, and then '
          'contact '
          'Telemetry and Chrome Client Infra team to schedule the benchmark. '
          'You can email chrome-benchmarking-request@ to get started.'.format(
              benchmark=benchmark, platform=platform.name))
    for benchmark in shard_map_benchmark_names - platform_benchmark_names:
      errors.append(
          'Benchmark {benchmark} is scheduled on shard map {path}, but '
          'bot_platforms.py '
          'says that it should not be on that shard map. This could be because '
          'the benchmark was deleted. If that is the case, you can use '
          '`generate_perf_sharding deschedule` to deschedule the benchmark '
          'from the shard map.'.format(
              benchmark=benchmark, path=platform.shards_map_file_path))

  # Check that every official benchmark is scheduled on some shard map.
  # TODO(crbug.com/963614): Note that this check can be deleted if we
  # find some way other than naming the benchmark with prefix "UNSCHEDULED_"
  # to make it clear that a benchmark is not running.
  scheduled_benchmarks = set()
  for platform in bot_platforms.ALL_PLATFORMS:
    scheduled_benchmarks = scheduled_benchmarks | _ParseBenchmarks(
        platform.shards_map_file_path)
  for benchmark in (
      bot_platforms.OFFICIAL_BENCHMARK_NAMES - scheduled_benchmarks):
    errors.append(
        'Benchmark {benchmark} is an official benchmark, but it is not '
        'scheduled to run anywhere. please rename it to '
        'UNSCHEDULED_{benchmark}'.format(benchmark=benchmark))

  for error in errors:
    print('*', textwrap.fill(error, 70), '\n', file=sys.stderr)
  if errors:
    return 1
  return 0
Validate that the shard maps, csv files, etc. are consistent.
Validate that the shard maps, csv files, etc. are consistent.
Validate that the shard maps csv files etc . are consistent .
def _ValidateShardMaps(args):
  del args
  errors = []
  tempdir = tempfile.mkdtemp()
  try:
    builders = _GetBuilderPlatforms(builders=None, waterfall='all')
    for builder in builders:
      output_file = os.path.join(
          tempdir, os.path.basename(builder.timing_file_path))
      _FilterTimingData(builder, output_file)
      if not filecmp.cmp(builder.timing_file_path, output_file):
        errors.append(
            '{timing_data} is not up to date. Please run '
            '`./generate_perf_sharding.py update-timing --filter-only` '
            'to regenerate it.'.format(timing_data=builder.timing_file_path))
  finally:
    shutil.rmtree(tempdir)
  for platform in bot_platforms.ALL_PLATFORMS:
    platform_benchmark_names = set(
        b.name for b in platform.benchmark_configs) | set(
            e.name for e in platform.executables)
    shard_map_benchmark_names = _ParseBenchmarks(platform.shards_map_file_path)
    for benchmark in platform_benchmark_names - shard_map_benchmark_names:
      errors.append(
          'Benchmark {benchmark} is supposed to be scheduled on platform '
          '{platform} according to '
          'bot_platforms.py, but it is not yet scheduled. If this is a new '
          'benchmark, please rename it to UNSCHEDULED_{benchmark}, and then '
          'contact '
          'Telemetry and Chrome Client Infra team to schedule the benchmark. '
          'You can email chrome-benchmarking-request@ to get started.'.format(
              benchmark=benchmark, platform=platform.name))
    for benchmark in shard_map_benchmark_names - platform_benchmark_names:
      errors.append(
          'Benchmark {benchmark} is scheduled on shard map {path}, but '
          'bot_platforms.py '
          'says that it should not be on that shard map. This could be because '
          'the benchmark was deleted. If that is the case, you can use '
          '`generate_perf_sharding deschedule` to deschedule the benchmark '
          'from the shard map.'.format(
              benchmark=benchmark, path=platform.shards_map_file_path))
  scheduled_benchmarks = set()
  for platform in bot_platforms.ALL_PLATFORMS:
    scheduled_benchmarks = scheduled_benchmarks | _ParseBenchmarks(
        platform.shards_map_file_path)
  for benchmark in (
      bot_platforms.OFFICIAL_BENCHMARK_NAMES - scheduled_benchmarks):
    errors.append(
        'Benchmark {benchmark} is an official benchmark, but it is not '
        'scheduled to run anywhere. please rename it to '
        'UNSCHEDULED_{benchmark}'.format(benchmark=benchmark))
  for error in errors:
    print('*', textwrap.fill(error, 70), '\n', file=sys.stderr)
  if errors:
    return 1
  return 0
def _ValidateShardMaps ( args ) : del args errors = [ ] tempdir = tempfile . mkdtemp ( ) try : builders = _GetBuilderPlatforms ( builders = None , waterfall = 'all' ) for builder in builders : output_file = os . path . join ( tempdir , os . path . basename ( builder . timing_file_path ) ) _FilterTimingData ( builder , output_file ) if not filecmp . cmp ( builder . timing_file_path , output_file ) : errors . append ( '{timing_data} is not up to date. Please run ' '`./generate_perf_sharding.py update-timing --filter-only` ' 'to regenerate it.' . format ( timing_data = builder . timing_file_path ) ) finally : shutil . rmtree ( tempdir ) for platform in bot_platforms . ALL_PLATFORMS : platform_benchmark_names = set ( b . name for b in platform . benchmark_configs ) | set ( e . name for e in platform . executables ) shard_map_benchmark_names = _ParseBenchmarks ( platform . shards_map_file_path ) for benchmark in platform_benchmark_names - shard_map_benchmark_names : errors . append ( 'Benchmark {benchmark} is supposed to be scheduled on platform ' '{platform} according to ' 'bot_platforms.py, but it is not yet scheduled. If this is a new ' 'benchmark, please rename it to UNSCHEDULED_{benchmark}, and then ' 'contact ' 'Telemetry and Chrome Client Infra team to schedule the benchmark. ' 'You can email chrome-benchmarking-request@ to get started.' . format ( benchmark = benchmark , platform = platform . name ) ) for benchmark in shard_map_benchmark_names - platform_benchmark_names : errors . append ( 'Benchmark {benchmark} is scheduled on shard map {path}, but ' 'bot_platforms.py ' 'says that it should not be on that shard map. This could be because ' 'the benchmark was deleted. If that is the case, you can use ' '`generate_perf_sharding deschedule` to deschedule the benchmark ' 'from the shard map.' . format ( benchmark = benchmark , path = platform . shards_map_file_path ) ) scheduled_benchmarks = set ( ) for platform in bot_platforms . ALL_PLATFORMS : scheduled_benchmarks = scheduled_benchmarks | _ParseBenchmarks ( platform . shards_map_file_path ) for benchmark in ( bot_platforms . OFFICIAL_BENCHMARK_NAMES - scheduled_benchmarks ) : errors . append ( 'Benchmark {benchmark} is an official benchmark, but it is not ' 'scheduled to run anywhere. please rename it to ' 'UNSCHEDULED_{benchmark}' . format ( benchmark = benchmark ) ) for error in errors : print ( '*' , textwrap . fill ( error , 70 ) , '\n' , file = sys . stderr ) if errors : return 1 return 0
Validate that the shard maps, csv files, etc.
Validate that the shard maps csv files etc .
"""Validate that the shard maps, csv files, etc. are consistent.""" # Check that bot_platforms.py matches the actual shard maps # Check that every official benchmark is scheduled on some shard map. # TODO(crbug.com/963614): Note that this check can be deleted if we # find some way other than naming the benchmark with prefix "UNSCHEDULED_" # to make it clear that a benchmark is not running.
args