testing/merge_scripts/standard_gtest_merge.py
def merge_shard_results(summary_json, jsons_to_merge):
  """Reads JSON test output from all shards and combines them into one.

  Returns dict with merged test output on success or None on failure. Emits
  annotations.
  """
  # summary.json is produced by swarming client itself. We are mostly interested
  # in the number of shards.
  try:
    with open(summary_json) as f:
      summary = json.load(f)
  except (IOError, ValueError):
    emit_warning(
        'summary.json is missing or can not be read',
        'Something is seriously wrong with swarming client or the bot.')
    return None

  # Merge all JSON files together. Keep track of missing shards.
  merged = {
    'all_tests': set(),
    'disabled_tests': set(),
    'global_tags': set(),
    'missing_shards': [],
    'per_iteration_data': [],
    'swarming_summary': summary,
    'test_locations': {},
  }
  for index, result in enumerate(summary['shards']):
    if result is None:
      merged['missing_shards'].append(index)
      continue

    # Author note: this code path doesn't trigger convert_to_old_format() in
    # client/swarming.py, which means the state enum is saved in its string
    # name form, not in the number form.
    state = result.get('state')
    if state == u'BOT_DIED':
      emit_warning('Shard #%d had a Swarming internal failure' % index)
    elif state == u'EXPIRED':
      emit_warning('There wasn\'t enough capacity to run your test')
    elif state == u'TIMED_OUT':
      emit_warning(
          'Test runtime exceeded allocated time',
          'Either it ran for too long (hard timeout) or it didn\'t produce '
          'I/O for an extended period of time (I/O timeout)')
    elif state != u'COMPLETED':
      emit_warning('Invalid Swarming task state: %s' % state)

    json_data, err_msg = load_shard_json(index, result.get('task_id'),
                                         jsons_to_merge)
    if json_data:
      # Set-like fields.
      for key in ('all_tests', 'disabled_tests', 'global_tags'):
        merged[key].update(json_data.get(key), [])

      # Dict-like fields.
      for key in ('test_locations',):
        merged[key].update(json_data.get(key, {}))

      # 'per_iteration_data' is a list of dicts. Dicts should be merged
      # together, not the 'per_iteration_data' list itself.
      merged['per_iteration_data'] = merge_list_of_dicts(
          merged['per_iteration_data'],
          json_data.get('per_iteration_data', []))
    else:
      merged['missing_shards'].append(index)
      emit_warning('No result was found: %s' % err_msg)

  # If some shards are missing, make it known. Continue parsing anyway. Step
  # should be red anyway, since swarming.py return non-zero exit code in that
  # case.
  if merged['missing_shards']:
    as_str = ', '.join(map(str, merged['missing_shards']))
    emit_warning(
        'some shards did not complete: %s' % as_str,
        MISSING_SHARDS_MSG % as_str)
    # Not all tests run, combined JSON summary can not be trusted.
    merged['global_tags'].add('UNRELIABLE_RESULTS')

  # Convert to jsonish dict.
  for key in ('all_tests', 'disabled_tests', 'global_tags'):
    merged[key] = sorted(merged[key])
  return merged
Reads JSON test output from all shards and combines them into one.

  Returns dict with merged test output on success or None on failure. Emits
  annotations.
  
Reads JSON test output from all shards and combines them into one.
Returns dict with merged test output on success or None on failure. Emits
annotations.
Reads JSON test output from all shards and combines them into one . Returns dict with merged test output on success or None on failure . Emits annotations .
def merge_shard_results(summary_json, jsons_to_merge):
  try:
    with open(summary_json) as f:
      summary = json.load(f)
  except (IOError, ValueError):
    emit_warning(
        'summary.json is missing or can not be read',
        'Something is seriously wrong with swarming client or the bot.')
    return None
  merged = {
    'all_tests': set(),
    'disabled_tests': set(),
    'global_tags': set(),
    'missing_shards': [],
    'per_iteration_data': [],
    'swarming_summary': summary,
    'test_locations': {},
  }
  for index, result in enumerate(summary['shards']):
    if result is None:
      merged['missing_shards'].append(index)
      continue
    state = result.get('state')
    if state == u'BOT_DIED':
      emit_warning('Shard #%d had a Swarming internal failure' % index)
    elif state == u'EXPIRED':
      emit_warning('There wasn\'t enough capacity to run your test')
    elif state == u'TIMED_OUT':
      emit_warning(
          'Test runtime exceeded allocated time',
          'Either it ran for too long (hard timeout) or it didn\'t produce '
          'I/O for an extended period of time (I/O timeout)')
    elif state != u'COMPLETED':
      emit_warning('Invalid Swarming task state: %s' % state)
    json_data, err_msg = load_shard_json(index, result.get('task_id'),
                                         jsons_to_merge)
    if json_data:
      for key in ('all_tests', 'disabled_tests', 'global_tags'):
        merged[key].update(json_data.get(key), [])
      for key in ('test_locations',):
        merged[key].update(json_data.get(key, {}))
      merged['per_iteration_data'] = merge_list_of_dicts(
          merged['per_iteration_data'],
          json_data.get('per_iteration_data', []))
    else:
      merged['missing_shards'].append(index)
      emit_warning('No result was found: %s' % err_msg)
  if merged['missing_shards']:
    as_str = ', '.join(map(str, merged['missing_shards']))
    emit_warning(
        'some shards did not complete: %s' % as_str,
        MISSING_SHARDS_MSG % as_str)
    merged['global_tags'].add('UNRELIABLE_RESULTS')
  for key in ('all_tests', 'disabled_tests', 'global_tags'):
    merged[key] = sorted(merged[key])
  return merged
def merge_shard_results ( summary_json , jsons_to_merge ) : try : with open ( summary_json ) as f : summary = json . load ( f ) except ( IOError , ValueError ) : emit_warning ( 'summary.json is missing or can not be read' , 'Something is seriously wrong with swarming client or the bot.' ) return None merged = { 'all_tests' : set ( ) , 'disabled_tests' : set ( ) , 'global_tags' : set ( ) , 'missing_shards' : [ ] , 'per_iteration_data' : [ ] , 'swarming_summary' : summary , 'test_locations' : { } , } for index , result in enumerate ( summary [ 'shards' ] ) : if result is None : merged [ 'missing_shards' ] . append ( index ) continue state = result . get ( 'state' ) if state == u'BOT_DIED' : emit_warning ( 'Shard #%d had a Swarming internal failure' % index ) elif state == u'EXPIRED' : emit_warning ( 'There wasn\'t enough capacity to run your test' ) elif state == u'TIMED_OUT' : emit_warning ( 'Test runtime exceeded allocated time' , 'Either it ran for too long (hard timeout) or it didn\'t produce ' 'I/O for an extended period of time (I/O timeout)' ) elif state != u'COMPLETED' : emit_warning ( 'Invalid Swarming task state: %s' % state ) json_data , err_msg = load_shard_json ( index , result . get ( 'task_id' ) , jsons_to_merge ) if json_data : for key in ( 'all_tests' , 'disabled_tests' , 'global_tags' ) : merged [ key ] . update ( json_data . get ( key ) , [ ] ) for key in ( 'test_locations' , ) : merged [ key ] . update ( json_data . get ( key , { } ) ) merged [ 'per_iteration_data' ] = merge_list_of_dicts ( merged [ 'per_iteration_data' ] , json_data . get ( 'per_iteration_data' , [ ] ) ) else : merged [ 'missing_shards' ] . append ( index ) emit_warning ( 'No result was found: %s' % err_msg ) if merged [ 'missing_shards' ] : as_str = ', ' . join ( map ( str , merged [ 'missing_shards' ] ) ) emit_warning ( 'some shards did not complete: %s' % as_str , MISSING_SHARDS_MSG % as_str ) merged [ 'global_tags' ] . add ( 'UNRELIABLE_RESULTS' ) for key in ( 'all_tests' , 'disabled_tests' , 'global_tags' ) : merged [ key ] = sorted ( merged [ key ] ) return merged
Reads JSON test output from all shards and combines them into one.
Reads JSON test output from all shards and combines them into one .
"""Reads JSON test output from all shards and combines them into one.

  Returns dict with merged test output on success or None on failure. Emits
  annotations.
  """ # summary.json is produced by swarming client itself. We are mostly interested # in the number of shards. # Merge all JSON files together. Keep track of missing shards. # Author note: this code path doesn't trigger convert_to_old_format() in # client/swarming.py, which means the state enum is saved in its string # name form, not in the number form. # Set-like fields. # Dict-like fields. # 'per_iteration_data' is a list of dicts. Dicts should be merged # together, not the 'per_iteration_data' list itself. # If some shards are missing, make it known. Continue parsing anyway. Step # should be red anyway, since swarming.py return non-zero exit code in that # case. # Not all tests run, combined JSON summary can not be trusted. # Convert to jsonish dict.
summary_json
jsons_to_merge