text: #!/usr/bin/env python
  # Copyright 2017 The Chromium Authors. All rights reserved.
  # Use of this source code is governed by a BSD-style license that can be
  # found in the LICENSE file.
  
  import argparse
  import json
  import os
  import shutil
  import sys
  import tempfile
  import traceback
  
  import merge_api
  
  
  MISSING_SHARDS_MSG = r"""Missing results from the following shard(s): %s
  
  This can happen in following cases:
    * Test failed to start (missing *.dll/*.so dependency for example)
    * Test crashed or hung
    * Task expired because there are not enough bots available and are all used
    * Swarming service experienced problems
  
  Please examine logs to figure out what happened.
  """
  
  
  def emit_warning(title, log=None):
    print '@@@STEP_WARNINGS@@@'
    print title
    if log:
      title = title.rstrip()
      for line in log.splitlines():
        print '@@@STEP_LOG_LINE@%s@%s@@@' % (title, line.rstrip())
      print '@@@STEP_LOG_END@%s@@@' % title
  
  
  def merge_shard_results(summary_json, jsons_to_merge):
    """Reads JSON test output from all shards and combines them into one.
  
    Returns dict with merged test output on success or None on failure. Emits
    annotations.
    """
    # summary.json is produced by swarming client itself. We are mostly interested
    # in the number of shards.
    try:
      with open(summary_json) as f:
        summary = json.load(f)
    except (IOError, ValueError):
      emit_warning(
          'summary.json is missing or can not be read',
          'Something is seriously wrong with swarming client or the bot.')
      return None
  
    # Merge all JSON files together. Keep track of missing shards.
    merged = {
      'all_tests': set(),
      'disabled_tests': set(),
      'global_tags': set(),
      'missing_shards': [],
      'per_iteration_data': [],
      'swarming_summary': summary,
      'test_locations': {},
    }
    for index, result in enumerate(summary['shards']):
      if result is None:
        merged['missing_shards'].append(index)
        continue
  
      # Author note: this code path doesn't trigger convert_to_old_format() in
      # client/swarming.py, which means the state enum is saved in its string
      # name form, not in the number form.
      state = result.get('state')
      if state == u'BOT_DIED':
        emit_warning('Shard #%d had a Swarming internal failure' % index)
      elif state == u'EXPIRED':
        emit_warning('There wasn\'t enough capacity to run your test')
      elif state == u'TIMED_OUT':
        emit_warning(
            'Test runtime exceeded allocated time',
            'Either it ran for too long (hard timeout) or it didn\'t produce '
            'I/O for an extended period of time (I/O timeout)')
      elif state != u'COMPLETED':
        emit_warning('Invalid Swarming task state: %s' % state)
  
      json_data, err_msg = load_shard_json(index, result.get('task_id'),
                                           jsons_to_merge)
      if json_data:
        # Set-like fields.
        for key in ('all_tests', 'disabled_tests', 'global_tags'):
          merged[key].update(json_data.get(key), [])
  
        # Dict-like fields.
        for key in ('test_locations',):
          merged[key].update(json_data.get(key, {}))
  
        # 'per_iteration_data' is a list of dicts. Dicts should be merged
        # together, not the 'per_iteration_data' list itself.
        merged['per_iteration_data'] = merge_list_of_dicts(
            merged['per_iteration_data'],
            json_data.get('per_iteration_data', []))
      else:
        merged['missing_shards'].append(index)
        emit_warning('No result was found: %s' % err_msg)
  
    # If some shards are missing, make it known. Continue parsing anyway. Step
    # should be red anyway, since swarming.py return non-zero exit code in that
    # case.
    if merged['missing_shards']:
      as_str = ', '.join(map(str, merged['missing_shards']))
      emit_warning(
          'some shards did not complete: %s' % as_str,
          MISSING_SHARDS_MSG % as_str)
      # Not all tests run, combined JSON summary can not be trusted.
      merged['global_tags'].add('UNRELIABLE_RESULTS')
  
    # Convert to jsonish dict.
    for key in ('all_tests', 'disabled_tests', 'global_tags'):
      merged[key] = sorted(merged[key])
    return merged
  
  
  OUTPUT_JSON_SIZE_LIMIT = 100 * 1024 * 1024  # 100 MB
  
  
  def load_shard_json(index, task_id, jsons_to_merge):
    """Reads JSON output of the specified shard.
  
    Args:
      output_dir: The directory in which to look for the JSON output to load.
      index: The index of the shard to load data for, this is for old api.
      task_id: The directory of the shard to load data for, this is for new api.
  
    Returns: A tuple containing:
      * The contents of path, deserialized into a python object.
      * An error string.
      (exactly one of the tuple elements will be non-None).
    """
    # 'output.json' is set in swarming/api.py, gtest_task method.
    matching_json_files = [
        j for j in jsons_to_merge
        if (os.path.basename(j) == 'output.json' and
            (os.path.basename(os.path.dirname(j)) == str(index) or
             os.path.basename(os.path.dirname(j)) == task_id))]
  
    if not matching_json_files:
      print >> sys.stderr, 'shard %s test output missing' % index
      return (None, 'shard %s test output was missing' % index)
    elif len(matching_json_files) > 1:
      print >> sys.stderr, 'duplicate test output for shard %s' % index
      return (None, 'shard %s test output was duplicated' % index)
  
    path = matching_json_files[0]
  
    try:
      filesize = os.stat(path).st_size
      if filesize > OUTPUT_JSON_SIZE_LIMIT:
        print >> sys.stderr, 'output.json is %d bytes. Max size is %d' % (
             filesize, OUTPUT_JSON_SIZE_LIMIT)
        return (None, 'shard %s test output exceeded the size limit' % index)
  
      with open(path) as f:
        return (json.load(f), None)
    except (IOError, ValueError, OSError) as e:
      print >> sys.stderr, 'Missing or invalid gtest JSON file: %s' % path
      print >> sys.stderr, '%s: %s' % (type(e).__name__, e)
  
      return (None, 'shard %s test output was missing or invalid' % index)
  
  
  def merge_list_of_dicts(left, right):
    """Merges dicts left[0] with right[0], left[1] with right[1], etc."""
    output = []
    for i in xrange(max(len(left), len(right))):
      left_dict = left[i] if i < len(left) else {}
      right_dict = right[i] if i < len(right) else {}
      merged_dict = left_dict.copy()
      merged_dict.update(right_dict)
      output.append(merged_dict)
    return output
  
  
  def standard_gtest_merge(
      output_json, summary_json, jsons_to_merge):
  
    output = merge_shard_results(summary_json, jsons_to_merge)
    with open(output_json, 'wb') as f:
      json.dump(output, f)
  
    return 0
  
  
  def main(raw_args):
  
    parser = merge_api.ArgumentParser()
    args = parser.parse_args(raw_args)
  
    return standard_gtest_merge(
        args.output_json, args.summary_json, args.jsons_to_merge)
  
  
  if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))
  