text: # Copyright (C) 2010 Google Inc. All rights reserved.
  # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
  #
  # Redistribution and use in source and binary forms, with or without
  # modification, are permitted provided that the following conditions are
  # met:
  #
  #     * Redistributions of source code must retain the above copyright
  # notice, this list of conditions and the following disclaimer.
  #     * Redistributions in binary form must reproduce the above
  # copyright notice, this list of conditions and the following disclaimer
  # in the documentation and/or other materials provided with the
  # distribution.
  #     * Neither the name of Google Inc. nor the names of its
  # contributors may be used to endorse or promote products derived from
  # this software without specific prior written permission.
  #
  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  
  import datetime
  import logging
  import signal
  import sys
  
  from webkitpy.common.iteration_compatibility import iteritems
  from webkitpy.layout_tests.models import test_expectations
  from webkitpy.layout_tests.models import test_failures
  
  
  _log = logging.getLogger(__name__)
  
  INTERRUPTED_EXIT_STATUS = signal.SIGINT + 128
  
  
  class TestRunResults(object):
      def __init__(self, expectations, num_tests):
          self.total = num_tests
          self.remaining = self.total
          self.expectations = expectations
          self.expected = 0
          self.unexpected = 0
          self.unexpected_failures = 0
          self.unexpected_crashes = 0
          self.unexpected_timeouts = 0
          self.tests_by_expectation = {}
          self.tests_by_timeline = {}
          self.results_by_name = {}  # Map of test name to the last result for the test.
          self.all_results = []  # All results from a run, including every iteration of every test.
          self.unexpected_results_by_name = {}
          self.failures_by_name = {}
          self.total_failures = 0
          self.expected_skips = 0
          for expectation in test_expectations.TestExpectations.EXPECTATIONS.values():
              self.tests_by_expectation[expectation] = set()
          for timeline in test_expectations.TestExpectations.TIMELINES.values():
              self.tests_by_timeline[timeline] = expectations.model().get_tests_with_timeline(timeline)
          self.slow_tests = set()
          self.interrupted = False
          self.keyboard_interrupted = False
  
      def add(self, test_result, expected, test_is_slow):
          self.tests_by_expectation[test_result.type].add(test_result.test_name)
          self.results_by_name[test_result.test_name] = test_result
          if test_result.is_other_crash:
              return
          if test_result.type != test_expectations.SKIP:
              self.all_results.append(test_result)
          self.remaining -= 1
          if len(test_result.failures):
              self.total_failures += 1
              self.failures_by_name[test_result.test_name] = test_result.failures
          if expected:
              self.expected += 1
              if test_result.type == test_expectations.SKIP:
                  self.expected_skips += 1
          else:
              self.unexpected_results_by_name[test_result.test_name] = test_result
              self.unexpected += 1
              if len(test_result.failures):
                  self.unexpected_failures += 1
              if test_result.type == test_expectations.CRASH:
                  self.unexpected_crashes += 1
              elif test_result.type == test_expectations.TIMEOUT:
                  self.unexpected_timeouts += 1
          if test_is_slow:
              self.slow_tests.add(test_result.test_name)
  
      def change_result_to_failure(self, existing_result, new_result, existing_expected, new_expected):
          assert existing_result.test_name == new_result.test_name
          if existing_result.type is new_result.type:
              return
  
          self.tests_by_expectation[existing_result.type].remove(existing_result.test_name)
          self.tests_by_expectation[new_result.type].add(new_result.test_name)
  
          had_failures = len(existing_result.failures) > 0
  
          existing_result.convert_to_failure(new_result)
  
          if not had_failures and len(existing_result.failures):
              self.total_failures += 1
  
          if len(existing_result.failures):
              self.failures_by_name[existing_result.test_name] = existing_result.failures
  
          if not existing_expected and new_expected:
              # test changed from unexpected to expected
              self.expected += 1
              self.unexpected_results_by_name.pop(existing_result.test_name, None)
              self.unexpected -= 1
              if had_failures:
                  self.unexpected_failures -= 1
          elif existing_expected and not new_expected:
              # test changed from expected to unexpected
              self.expected -= 1
              self.unexpected_results_by_name[existing_result.test_name] = existing_result
              self.unexpected += 1
              if len(existing_result.failures):
                  self.unexpected_failures += 1
  
      def merge(self, test_run_results):
          if not test_run_results:
              return self
          # self.expectations should be the same for both
          self.total += test_run_results.total
          self.remaining += test_run_results.remaining
          self.expected += test_run_results.expected
          self.unexpected += test_run_results.unexpected
          self.unexpected_failures += test_run_results.unexpected_failures
          self.unexpected_crashes += test_run_results.unexpected_crashes
          self.unexpected_timeouts += test_run_results.unexpected_timeouts
          self.tests_by_expectation.update(test_run_results.tests_by_expectation)
          self.tests_by_timeline.update(test_run_results.tests_by_timeline)
          self.results_by_name.update(test_run_results.results_by_name)
          self.all_results += test_run_results.all_results
          self.unexpected_results_by_name.update(test_run_results.unexpected_results_by_name)
          self.failures_by_name.update(test_run_results.failures_by_name)
          self.total_failures += test_run_results.total_failures
          self.expected_skips += test_run_results.expected_skips
          self.tests_by_expectation.update(test_run_results.tests_by_expectation)
          self.tests_by_timeline.update(test_run_results.tests_by_timeline)
          self.slow_tests.update(test_run_results.slow_tests)
  
          self.interrupted |= test_run_results.interrupted
          self.keyboard_interrupted |= test_run_results.keyboard_interrupted
          return self
  
  
  class RunDetails(object):
      def __init__(self, exit_code, summarized_results=None, initial_results=None, retry_results=None, enabled_pixel_tests_in_retry=False, skipped_all_tests=False):
          self.exit_code = exit_code
          self.summarized_results = summarized_results
          self.initial_results = initial_results
          self.retry_results = retry_results
          self.enabled_pixel_tests_in_retry = enabled_pixel_tests_in_retry
          self.skipped_all_tests = skipped_all_tests
  
  
  def _interpret_test_failures(failures):
      test_dict = {}
  
      failure_types = [type(failure) for failure in failures]
      # FIXME: get rid of all this is_* values once there is a 1:1 map between
      # TestFailure type and test_expectations.EXPECTATION.
      if test_failures.FailureMissingAudio in failure_types:
          test_dict['is_missing_audio'] = True
  
      if test_failures.FailureMissingResult in failure_types:
          test_dict['is_missing_text'] = True
  
      if test_failures.FailureMissingImage in failure_types or test_failures.FailureMissingImageHash in failure_types:
          test_dict['is_missing_image'] = True
  
      if test_failures.FailureDocumentLeak in failure_types:
          leaks = []
          for failure in failures:
              if isinstance(failure, test_failures.FailureDocumentLeak):
                  for url in failure.leaked_document_urls:
                      leaks.append({"document": url})
          test_dict['leaks'] = leaks
  
      if 'image_diff_percent' not in test_dict:
          for failure in failures:
              if isinstance(failure, test_failures.FailureImageHashMismatch) or isinstance(failure, test_failures.FailureReftestMismatch):
                  test_dict['image_diff_percent'] = failure.diff_percent
  
      return test_dict
  
  
  # These results must match ones in print_unexpected_results() in views/buildbot_results.py.
  def summarize_results(port_obj, expectations_by_type, initial_results, retry_results, enabled_pixel_tests_in_retry, include_passes=False, include_time_and_modifiers=False):
      """Returns a dictionary containing a summary of the test runs, with the following fields:
          'version': a version indicator
          'fixable': The number of fixable tests (NOW - PASS)
          'skipped': The number of skipped tests (NOW & SKIPPED)
          'num_regressions': The number of non-flaky failures
          'num_flaky': The number of flaky failures
          'num_missing': The number of tests with missing results
          'num_passes': The number of unexpected passes
          'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
          'date': the current date and time
      """
      results = {}
      results['version'] = 4
  
      tbe = initial_results.tests_by_expectation
      tbt = initial_results.tests_by_timeline
      results['fixable'] = len(tbt[test_expectations.NOW] - tbe[test_expectations.PASS])
      results['skipped'] = len(tbt[test_expectations.NOW] & tbe[test_expectations.SKIP])
  
      num_passes = 0
      num_flaky = 0
      num_missing = 0
      num_regressions = 0
      keywords = {}
      for expectation_string, expectation_enum in test_expectations.TestExpectations.EXPECTATIONS.items():
          keywords[expectation_enum] = expectation_string.upper()
  
      for modifier_string, modifier_enum in test_expectations.TestExpectations.MODIFIERS.items():
          keywords[modifier_enum] = modifier_string.upper()
  
      tests = {}
      other_crashes_dict = {}
  
      for test_name, result in iteritems(initial_results.results_by_name):
          # Note that if a test crashed in the original run, we ignore
          # whether or not it crashed when we retried it (if we retried it),
          # and always consider the result not flaky.
          pixel_tests_enabled = enabled_pixel_tests_in_retry or port_obj._options.pixel_tests or bool(result.reftest_type)
  
          # We're basically trying to find the first non-skip expectation, and use that expectation object for the remainder of the loop.
          # This works because tests are run on the first device type which won't skip them, regardless of other expectations, and never re-run.
          expected = 'SKIP'
          expectations = list(expectations_by_type.values())[0]
          for element in expectations_by_type.values():
              test_expectation = element.filtered_expectations_for_test(test_name, pixel_tests_enabled, port_obj._options.world_leaks)
              expected = element.model().expectations_to_string(test_expectation)
              if expected != 'SKIP':
                  expectations = element
                  continue
  
          result_type = result.type
          actual = [keywords[result_type]]
  
          if result_type == test_expectations.SKIP:
              continue
  
          if result.is_other_crash:
              other_crashes_dict[test_name] = {}
              continue
  
          test_dict = {}
          if result.has_stderr:
              test_dict['has_stderr'] = True
  
          if result.reftest_type:
              test_dict.update(reftest_type=list(result.reftest_type))
  
          if expectations.model().has_modifier(test_name, test_expectations.WONTFIX):
              test_dict['wontfix'] = True
  
          if result_type == test_expectations.PASS:
              num_passes += 1
              # FIXME: include passing tests that have stderr output.
              if expected == 'PASS' and not include_passes:
                  continue
          elif result_type == test_expectations.CRASH:
              if test_name in initial_results.unexpected_results_by_name:
                  num_regressions += 1
                  test_dict['report'] = 'REGRESSION'
          elif result_type == test_expectations.MISSING:
              if test_name in initial_results.unexpected_results_by_name:
                  num_missing += 1
                  test_dict['report'] = 'MISSING'
          elif test_name in initial_results.unexpected_results_by_name:
              if retry_results and test_name not in retry_results.unexpected_results_by_name:
                  actual.extend(expectations.model().get_expectations_string(test_name).split(" "))
                  num_flaky += 1
                  test_dict['report'] = 'FLAKY'
              elif retry_results:
                  retry_result_type = retry_results.unexpected_results_by_name[test_name].type
                  if result_type != retry_result_type:
                      if enabled_pixel_tests_in_retry and result_type == test_expectations.TEXT and (retry_result_type == test_expectations.IMAGE_PLUS_TEXT or retry_result_type == test_expectations.MISSING):
                          if retry_result_type == test_expectations.MISSING:
                              num_missing += 1
                          num_regressions += 1
                          test_dict['report'] = 'REGRESSION'
                      else:
                          num_flaky += 1
                          test_dict['report'] = 'FLAKY'
                      actual.append(keywords[retry_result_type])
                  else:
                      num_regressions += 1
                      test_dict['report'] = 'REGRESSION'
              else:
                  num_regressions += 1
                  test_dict['report'] = 'REGRESSION'
  
          test_dict['expected'] = expected
          test_dict['actual'] = " ".join(actual)
          if include_time_and_modifiers:
              test_dict['time'] = round(1000 * result.test_run_time)
              # FIXME: Fix get_modifiers to return modifiers in new format.
              test_dict['modifiers'] = ' '.join(expectations.model().get_modifiers(test_name)).replace('BUGWK', 'webkit.org/b/')
  
          test_dict.update(_interpret_test_failures(result.failures))
  
          if retry_results:
              retry_result = retry_results.unexpected_results_by_name.get(test_name)
              if retry_result:
                  test_dict.update(_interpret_test_failures(retry_result.failures))
  
          # Store test hierarchically by directory. e.g.
          # foo/bar/baz.html: test_dict
          # foo/bar/baz1.html: test_dict
          #
          # becomes
          # foo: {
          #     bar: {
          #         baz.html: test_dict,
          #         baz1.html: test_dict
          #     }
          # }
          parts = test_name.split('/')
          current_map = tests
          for i, part in enumerate(parts):
              if i == (len(parts) - 1):
                  current_map[part] = test_dict
                  break
              if part not in current_map:
                  current_map[part] = {}
              current_map = current_map[part]
  
      results['tests'] = tests
      results['num_passes'] = num_passes
      results['num_flaky'] = num_flaky
      results['num_missing'] = num_missing
      results['num_regressions'] = num_regressions
      results['uses_expectations_file'] = port_obj.uses_test_expectations_file()
      results['interrupted'] = initial_results.interrupted  # Does results.html have enough information to compute this itself? (by checking total number of results vs. total number of tests?)
      results['layout_tests_dir'] = port_obj.layout_tests_dir()
      results['has_pretty_patch'] = port_obj.pretty_patch.pretty_patch_available()
      results['pixel_tests_enabled'] = port_obj.get_option('pixel_tests')
      results['other_crashes'] = other_crashes_dict
      results['date'] = datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y")
  
      try:
          # We only use the svn revision for using trac links in the results.html file,
          # Don't do this by default since it takes >100ms.
          # FIXME: Do we really need to populate this both here and in the json_results_generator?
          if port_obj.get_option("builder_name"):
              port_obj.host.initialize_scm()
              results['revision'] = port_obj.commits_for_upload()[0]['id']
      except Exception as e:
          _log.warn("Failed to determine svn revision for checkout (cwd: %s, webkit_base: %s), leaving 'revision' key blank in full_results.json.\n%s" % (port_obj._filesystem.getcwd(), port_obj.path_from_webkit_base(), e))
          # Handle cases where we're running outside of version control.
          import traceback
          _log.debug('Failed to learn head svn revision:')
          _log.debug(traceback.format_exc())
          results['revision'] = ""
  
      return results
  