text: # Copyright (C) 2013 Google Inc. All rights reserved.
  #
  # Redistribution and use in source and binary forms, with or without
  # modification, are permitted provided that the following conditions are
  # met:
  #
  #     * Redistributions of source code must retain the above copyright
  # notice, this list of conditions and the following disclaimer.
  #     * Redistributions in binary form must reproduce the above
  # copyright notice, this list of conditions and the following disclaimer
  # in the documentation and/or other materials provided with the
  # distribution.
  #     * Neither the Google name nor the names of its
  # contributors may be used to endorse or promote products derived from
  # this software without specific prior written permission.
  #
  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  
  import unittest
  
  from webkitpy.common.config.builders import BUILDERS
  from webkitpy.layout_tests.layout_package import bot_test_expectations
  from webkitpy.layout_tests.builder_list import BuilderList
  
  
  class BotTestExpectationsFactoryTest(unittest.TestCase):
  
      def fake_builder_list(self):
          return BuilderList({
              "Dummy builder name": {"port_name": "dummy-port", "specifiers": []},
          })
  
      def fake_results_json_for_builder(self, builder):
          return bot_test_expectations.ResultsJSON(builder, 'Dummy content')
  
      def test_expectations_for_builder(self):
          factory = bot_test_expectations.BotTestExpectationsFactory(self.fake_builder_list())
          factory._results_json_for_builder = self.fake_results_json_for_builder
  
          self.assertIsNotNone(factory.expectations_for_builder('Dummy builder name'))
  
      def test_expectations_for_port(self):
          factory = bot_test_expectations.BotTestExpectationsFactory(self.fake_builder_list())
          factory._results_json_for_builder = self.fake_results_json_for_builder
  
          self.assertIsNotNone(factory.expectations_for_port('dummy-port'))
  
  
  class BotTestExpectationsTest(unittest.TestCase):
      # FIXME: Find a way to import this map from Tools/TestResultServer/model/jsonresults.py.
      FAILURE_MAP = {"A": "AUDIO", "C": "CRASH", "F": "TEXT", "I": "IMAGE", "O": "MISSING",
                     "N": "NO DATA", "P": "PASS", "T": "TIMEOUT", "Y": "NOTRUN", "X": "SKIP", "Z": "IMAGE+TEXT", "K": "LEAK"}
  
      # All result_string's in this file represent retries from a single run.
      # The left-most entry is the first try, the right-most is the last.
  
      def _assert_is_flaky(self, results_string, should_be_flaky, only_ignore_very_flaky, expected=None):
          results_json = self._results_json_from_test_data({})
          expectations = bot_test_expectations.BotTestExpectations(results_json, BuilderList(BUILDERS), set('test'))
  
          results_entry = self._results_from_string(results_string)
          if expected:
              results_entry[bot_test_expectations.ResultsJSON.EXPECTATIONS_KEY] = expected
  
          num_actual_results = len(expectations._flaky_types_in_results(results_entry, only_ignore_very_flaky))
          if should_be_flaky:
              self.assertGreater(num_actual_results, 1)
          else:
              self.assertLessEqual(num_actual_results, 1)
  
      def test_basic_flaky(self):
          self._assert_is_flaky('P', should_be_flaky=False, only_ignore_very_flaky=False)
          self._assert_is_flaky('P', should_be_flaky=False, only_ignore_very_flaky=True)
          self._assert_is_flaky('F', should_be_flaky=False, only_ignore_very_flaky=False)
          self._assert_is_flaky('F', should_be_flaky=False, only_ignore_very_flaky=True)
          self._assert_is_flaky('FP', should_be_flaky=True, only_ignore_very_flaky=False)
          self._assert_is_flaky('FP', should_be_flaky=False, only_ignore_very_flaky=True)
          self._assert_is_flaky('FFP', should_be_flaky=True, only_ignore_very_flaky=False)
          self._assert_is_flaky('FFP', should_be_flaky=True, only_ignore_very_flaky=True)
          self._assert_is_flaky('FFT', should_be_flaky=True, only_ignore_very_flaky=False)
          self._assert_is_flaky('FFT', should_be_flaky=True, only_ignore_very_flaky=True)
          self._assert_is_flaky('FFF', should_be_flaky=False, only_ignore_very_flaky=False)
          self._assert_is_flaky('FFF', should_be_flaky=False, only_ignore_very_flaky=True)
  
          self._assert_is_flaky('FT', should_be_flaky=True, only_ignore_very_flaky=False, expected='TIMEOUT')
          self._assert_is_flaky('FT', should_be_flaky=False, only_ignore_very_flaky=True, expected='TIMEOUT')
          self._assert_is_flaky('FFT', should_be_flaky=True, only_ignore_very_flaky=False, expected='TIMEOUT')
          self._assert_is_flaky('FFT', should_be_flaky=True, only_ignore_very_flaky=True, expected='TIMEOUT')
  
      def _results_json_from_test_data(self, test_data):
          test_data[bot_test_expectations.ResultsJSON.FAILURE_MAP_KEY] = self.FAILURE_MAP
          json_dict = {
              'builder': test_data,
          }
          return bot_test_expectations.ResultsJSON('builder', json_dict)
  
      def _results_from_string(self, results_string):
          return {'results': [[1, results_string]]}
  
      def _assert_expectations(self, test_data, expectations_string, only_ignore_very_flaky):
          results_json = self._results_json_from_test_data(test_data)
          expectations = bot_test_expectations.BotTestExpectations(results_json, BuilderList(BUILDERS), set('test'))
          self.assertEqual(expectations.flakes_by_path(only_ignore_very_flaky), expectations_string)
  
      def _assert_unexpected_results(self, test_data, expectations_string):
          results_json = self._results_json_from_test_data(test_data)
          expectations = bot_test_expectations.BotTestExpectations(results_json, BuilderList(BUILDERS), set('test'))
          self.assertEqual(expectations.unexpected_results_by_path(), expectations_string)
  
      def test_basic(self):
          test_data = {
              'tests': {
                  'foo': {
                      'veryflaky.html': self._results_from_string('FFP'),
                      'maybeflaky.html': self._results_from_string('FP'),
                      'notflakypass.html': self._results_from_string('P'),
                      'notflakyfail.html': self._results_from_string('F'),
                      # Even if there are no expected results, it's not very flaky if it didn't do multiple retries.
                      # This accounts for the latest expectations not necessarily matching the expectations
                      # at the time of the given run.
                      'notverflakynoexpected.html': self._results_from_string('FT'),
                      # If the test is flaky, but marked as such, it shouldn't get printed out.
                      'notflakyexpected.html': {'results': [[2, 'FFFP']], 'expected': 'PASS FAIL'},
                  }
              }
          }
          self._assert_expectations(test_data, {
              'foo/veryflaky.html': sorted(["TEXT", "PASS"]),
          }, only_ignore_very_flaky=True)
  
          self._assert_expectations(test_data, {
              'foo/veryflaky.html': sorted(["TEXT", "PASS"]),
              'foo/notverflakynoexpected.html': ['TEXT', 'TIMEOUT'],
              'foo/maybeflaky.html': sorted(["TEXT", "PASS"]),
          }, only_ignore_very_flaky=False)
  
      def test_unexpected_results_no_unexpected(self):
          test_data = {
              'tests': {
                  'foo': {
                      'pass1.html': {'results': [[4, 'P']]},
                      'pass2.html': {'results': [[2, 'Z']], 'expected': 'PASS FAIL'},
                      'fail.html': {'results': [[2, 'P'], [1, 'F']], 'expected': 'PASS FAIL'},
                      'not_run.html': {'results': []},
                      'crash.html': {'results': [[2, 'F'], [1, 'C']], 'expected': 'CRASH FAIL WONTFIX'},
                  }
              }
          }
          self._assert_unexpected_results(test_data, {})
  
      def test_unexpected_results_all_unexpected(self):
          test_data = {
              'tests': {
                  'foo': {
                      'pass1.html': {'results': [[4, 'P']], 'expected': 'FAIL'},
                      'pass2.html': {'results': [[2, 'P']], 'expected': 'IMAGE'},
                      'fail.html': {'results': [[4, 'F']]},
                      'f_p.html': {'results': [[1, 'F'], [2, 'P']]},
                      'crash.html': {'results': [[2, 'F'], [1, 'C']], 'expected': 'WONTFIX'},
                      'image.html': {'results': [[2, 'F'], [1, 'I']], 'expected': 'CRASH TEXT'},
                      'i_f.html': {'results': [[1, 'F'], [5, 'I']], 'expected': 'PASS'},
                      'all.html': self._results_from_string('FPFPCNCNTXTXIZIZOCOCYKYK'),
                  }
              }
          }
          self.maxDiff = None
          self._assert_unexpected_results(test_data, {
              'foo/pass1.html': sorted(["FAIL", "PASS"]),
              'foo/pass2.html': sorted(["IMAGE", "PASS"]),
              'foo/fail.html': sorted(["TEXT", "PASS"]),
              'foo/f_p.html': sorted(["TEXT", "PASS"]),
              'foo/crash.html': sorted(["WONTFIX", "CRASH", "TEXT"]),
              'foo/image.html': sorted(["CRASH", "TEXT", "IMAGE"]),
              'foo/i_f.html': sorted(["PASS", "IMAGE", "TEXT"]),
              'foo/all.html': sorted(["TEXT", "PASS", "IMAGE+TEXT", "TIMEOUT", "CRASH", "IMAGE", "MISSING", "LEAK"]),
          })
  