third_party/blink/tools/blinkpy/web_tests/layout_package/bot_test_expectations.py
def _flaky_types_in_results(self, results_entry, only_consider_very_flaky,
                                ignore_bot_expected_results=False,
                                consider_only_flaky_runs=True):
        """Returns flaky results for a single test using its results entry
        retrieved from the test results server

        args:
          results_entry: Test run results aggregated from the last N builds
          only_consider_very_flaky: Flag for considering only test runs
              with more than one retry
          ignore_bot_expected_results: Flag for ignoring expected results
              retrieved from the test results server
          consider_only_flaky_runs: Flag for only considering test runs with
              more than one result"""
        flaky_results = set()

        if ignore_bot_expected_results:
            latest_expectations = []
        else:
            # Always include pass as an expected result. Passes will never
            # turn the bot red. This fixes cases where the expectations have an
            # implicit Pass, e.g. [ Slow ].
            latest_expectations = [ResultType.Pass]

            if self.results_json.EXPECTATIONS_KEY in results_entry:
                expectations_list = results_entry[self.results_json.
                                                  EXPECTATIONS_KEY].split(' ')
                latest_expectations.extend(expectations_list)

        for result_item in results_entry[self.results_json.RESULTS_KEY]:
            _, result_types_str = (
                self.results_json.occurances_and_type_from_result_item(result_item))

            result_types = []
            for result_type in result_types_str:
                # TODO(ojan): Remove this if-statement once crbug.com/514378 is fixed.
                if (result_type not in self.NON_RESULT_TYPES and
                        result_type not in self.RESULT_TYPES_TO_IGNORE):
                    result_types.append(
                        self.results_json.expectation_for_type(result_type))

            # It didn't flake if it didn't retry.
            if consider_only_flaky_runs and len(result_types) <= 1:
                continue

            # If the test ran as expected after only one retry, it's not very flaky.
            # It's only very flaky if it failed the first run and the first retry
            # and then ran as expected in one of the subsequent retries.
            # If there are only two entries, then that means it failed on the first
            # try and ran as expected on the second because otherwise we'd have
            # a third entry from the next try.
            if only_consider_very_flaky and len(result_types) <= 2:
                continue

            has_unexpected_results = False
            for result_type in result_types:
                # TODO(ojan): We really should be grabbing the expected results from the time
                # of the run instead of looking at the latest expected results. That's a lot
                # more complicated though. So far we've been looking at the aggregated
                # results_small.json off test_results.appspot, which has all the information
                # for the last 100 runs. In order to do this, we'd need to look at the
                # individual runs' full_results.json, which would be slow and more complicated.
                # The only thing we lose by not fixing this is that a test that was flaky
                # and got fixed will still get printed out until 100 runs have passed.
                if result_type not in latest_expectations:
                    has_unexpected_results = True
                    break

            if has_unexpected_results:
                flaky_results = flaky_results.union(set(result_types))

        return flaky_results
Returns flaky results for a single test using its results entry
        retrieved from the test results server

        args:
          results_entry: Test run results aggregated from the last N builds
          only_consider_very_flaky: Flag for considering only test runs
              with more than one retry
          ignore_bot_expected_results: Flag for ignoring expected results
              retrieved from the test results server
          consider_only_flaky_runs: Flag for only considering test runs with
              more than one result
Returns flaky results for a single test using its results entry
retrieved from the test results server

Test run results aggregated from the last N builds
only_consider_very_flaky: Flag for considering only test runs
with more than one retry
ignore_bot_expected_results: Flag for ignoring expected results
retrieved from the test results server
consider_only_flaky_runs: Flag for only considering test runs with
more than one result
Returns flaky results for a single test using its results entry retrieved from the test results server Test run results aggregated from the last N builds only_consider_very_flaky : Flag for considering only test runs with more than one retry ignore_bot_expected_results : Flag for ignoring expected results retrieved from the test results server consider_only_flaky_runs : Flag for only considering test runs with more than one result
def _flaky_types_in_results(self, results_entry, only_consider_very_flaky,
                                ignore_bot_expected_results=False,
                                consider_only_flaky_runs=True):
        flaky_results = set()
        if ignore_bot_expected_results:
            latest_expectations = []
        else:
            latest_expectations = [ResultType.Pass]
            if self.results_json.EXPECTATIONS_KEY in results_entry:
                expectations_list = results_entry[self.results_json.
                                                  EXPECTATIONS_KEY].split(' ')
                latest_expectations.extend(expectations_list)
        for result_item in results_entry[self.results_json.RESULTS_KEY]:
            _, result_types_str = (
                self.results_json.occurances_and_type_from_result_item(result_item))
            result_types = []
            for result_type in result_types_str:
                if (result_type not in self.NON_RESULT_TYPES and
                        result_type not in self.RESULT_TYPES_TO_IGNORE):
                    result_types.append(
                        self.results_json.expectation_for_type(result_type))
            if consider_only_flaky_runs and len(result_types) <= 1:
                continue
            if only_consider_very_flaky and len(result_types) <= 2:
                continue
            has_unexpected_results = False
            for result_type in result_types:
                if result_type not in latest_expectations:
                    has_unexpected_results = True
                    break
            if has_unexpected_results:
                flaky_results = flaky_results.union(set(result_types))
        return flaky_results
def _flaky_types_in_results ( self , results_entry , only_consider_very_flaky , ignore_bot_expected_results = False , consider_only_flaky_runs = True ) : flaky_results = set ( ) if ignore_bot_expected_results : latest_expectations = [ ] else : latest_expectations = [ ResultType . Pass ] if self . results_json . EXPECTATIONS_KEY in results_entry : expectations_list = results_entry [ self . results_json . EXPECTATIONS_KEY ] . split ( ' ' ) latest_expectations . extend ( expectations_list ) for result_item in results_entry [ self . results_json . RESULTS_KEY ] : _ , result_types_str = ( self . results_json . occurances_and_type_from_result_item ( result_item ) ) result_types = [ ] for result_type in result_types_str : if ( result_type not in self . NON_RESULT_TYPES and result_type not in self . RESULT_TYPES_TO_IGNORE ) : result_types . append ( self . results_json . expectation_for_type ( result_type ) ) if consider_only_flaky_runs and len ( result_types ) <= 1 : continue if only_consider_very_flaky and len ( result_types ) <= 2 : continue has_unexpected_results = False for result_type in result_types : if result_type not in latest_expectations : has_unexpected_results = True break if has_unexpected_results : flaky_results = flaky_results . union ( set ( result_types ) ) return flaky_results
Returns flaky results for a single test using its results entry
retrieved from the test results server
Returns flaky results for a single test using its results entry retrieved from the test results server
"""Returns flaky results for a single test using its results entry
        retrieved from the test results server

        args:
          results_entry: Test run results aggregated from the last N builds
          only_consider_very_flaky: Flag for considering only test runs
              with more than one retry
          ignore_bot_expected_results: Flag for ignoring expected results
              retrieved from the test results server
          consider_only_flaky_runs: Flag for only considering test runs with
              more than one result""" # Always include pass as an expected result. Passes will never # turn the bot red. This fixes cases where the expectations have an # implicit Pass, e.g. [ Slow ]. # TODO(ojan): Remove this if-statement once crbug.com/514378 is fixed. # It didn't flake if it didn't retry. # If the test ran as expected after only one retry, it's not very flaky. # It's only very flaky if it failed the first run and the first retry # and then ran as expected in one of the subsequent retries. # If there are only two entries, then that means it failed on the first # try and ran as expected on the second because otherwise we'd have # a third entry from the next try. # TODO(ojan): We really should be grabbing the expected results from the time # of the run instead of looking at the latest expected results. That's a lot # more complicated though. So far we've been looking at the aggregated # results_small.json off test_results.appspot, which has all the information # for the last 100 runs. In order to do this, we'd need to look at the # individual runs' full_results.json, which would be slow and more complicated. # The only thing we lose by not fixing this is that a test that was flaky # and got fixed will still get printed out until 100 runs have passed.
self
results_entry
only_consider_very_flaky
ignore_bot_expected_results
consider_only_flaky_runs