testing/scripts/run_rendering_benchmark_with_gated_performance.py
def compare_values(self, values_per_story, rerun=False):
    """ Parses the raw CSV data
    Compares the values in values_per_story with the upper_limit_data and
    determines if the story passes or fails and updates the ResultRecorder.

    Args:
      values_per_story: An array of rows (dict) descriving the CSV results
      rerun: Is this a rerun or initial run
    """
    for story_name in values_per_story:
      # The experimental stories will not be considered for failing the tests
      if (self.is_experimental_story(story_name)):
        continue
      if len(values_per_story[story_name]['ci_095']) == 0:
        print(('[  FAILED  ] {}/{} has no valid values for {}. Check ' +
          'run_benchmark logs for more information.').format(
            self.benchmark, story_name, METRIC_NAME))
        self.result_recorder[rerun].add_failure(story_name, self.benchmark)
        continue

      upper_limits = self.upper_limit_data
      upper_limit_avg = upper_limits[story_name]['avg']
      upper_limit_ci = upper_limits[story_name]['ci_095']
      lower_limit_cpu_ratio = upper_limits[story_name]['cpu_wall_time_ratio']
      measured_avg = np.mean(np.array(values_per_story[story_name]['averages']))
      measured_ci = np.mean(np.array(values_per_story[story_name]['ci_095']))
      measured_cpu_ratio = np.mean(np.array(
        values_per_story[story_name]['cpu_wall_time_ratio']))

      if (measured_ci > upper_limit_ci * CI_ERROR_MARGIN and
        self.is_control_story(story_name)):
        print(('[  FAILED  ] {}/{} {} has higher noise ({:.3f}) ' +
          'compared to upper limit ({:.3f})').format(
            self.benchmark, story_name, METRIC_NAME, measured_ci,
          upper_limit_ci))
        self.result_recorder[rerun].add_failure(
          story_name, self.benchmark, True)
      elif (measured_avg > upper_limit_avg * AVG_ERROR_MARGIN):
        if (measured_cpu_ratio >= lower_limit_cpu_ratio):
          print(('[  FAILED  ] {}/{} higher average {}({:.3f}) compared' +
            ' to upper limit ({:.3f})').format(self.benchmark, story_name,
            METRIC_NAME, measured_avg, upper_limit_avg))
          self.result_recorder[rerun].add_failure(story_name, self.benchmark)
        else:
          print(('[       OK ] {}/{} higher average {}({:.3f}) compared ' +
            'to upper limit({:.3f}). Invalidated for low cpu_wall_time_ratio'
            ).format(self.benchmark, story_name, METRIC_NAME, measured_avg,
                      upper_limit_avg))
          self.result_recorder[rerun].add_invalidation_reason(
            story_name, self.benchmark, 'Low cpu_wall_time_ratio')
      else:
        print(('[       OK ] {}/{} lower average {}({:.3f}) compared ' +
          'to upper limit({:.3f}).').format(self.benchmark, story_name,
          METRIC_NAME, measured_avg, upper_limit_avg))
 Parses the raw CSV data
    Compares the values in values_per_story with the upper_limit_data and
    determines if the story passes or fails and updates the ResultRecorder.

    Args:
      values_per_story: An array of rows (dict) descriving the CSV results
      rerun: Is this a rerun or initial run
    
Parses the raw CSV data
Compares the values in values_per_story with the upper_limit_data and
determines if the story passes or fails and updates the ResultRecorder.
Parses the raw CSV data Compares the values in values_per_story with the upper_limit_data and determines if the story passes or fails and updates the ResultRecorder .
def compare_values(self, values_per_story, rerun=False):
    for story_name in values_per_story:
      if (self.is_experimental_story(story_name)):
        continue
      if len(values_per_story[story_name]['ci_095']) == 0:
        print(('[  FAILED  ] {}/{} has no valid values for {}. Check ' +
          'run_benchmark logs for more information.').format(
            self.benchmark, story_name, METRIC_NAME))
        self.result_recorder[rerun].add_failure(story_name, self.benchmark)
        continue
      upper_limits = self.upper_limit_data
      upper_limit_avg = upper_limits[story_name]['avg']
      upper_limit_ci = upper_limits[story_name]['ci_095']
      lower_limit_cpu_ratio = upper_limits[story_name]['cpu_wall_time_ratio']
      measured_avg = np.mean(np.array(values_per_story[story_name]['averages']))
      measured_ci = np.mean(np.array(values_per_story[story_name]['ci_095']))
      measured_cpu_ratio = np.mean(np.array(
        values_per_story[story_name]['cpu_wall_time_ratio']))
      if (measured_ci > upper_limit_ci * CI_ERROR_MARGIN and
        self.is_control_story(story_name)):
        print(('[  FAILED  ] {}/{} {} has higher noise ({:.3f}) ' +
          'compared to upper limit ({:.3f})').format(
            self.benchmark, story_name, METRIC_NAME, measured_ci,
          upper_limit_ci))
        self.result_recorder[rerun].add_failure(
          story_name, self.benchmark, True)
      elif (measured_avg > upper_limit_avg * AVG_ERROR_MARGIN):
        if (measured_cpu_ratio >= lower_limit_cpu_ratio):
          print(('[  FAILED  ] {}/{} higher average {}({:.3f}) compared' +
            ' to upper limit ({:.3f})').format(self.benchmark, story_name,
            METRIC_NAME, measured_avg, upper_limit_avg))
          self.result_recorder[rerun].add_failure(story_name, self.benchmark)
        else:
          print(('[       OK ] {}/{} higher average {}({:.3f}) compared ' +
            'to upper limit({:.3f}). Invalidated for low cpu_wall_time_ratio'
            ).format(self.benchmark, story_name, METRIC_NAME, measured_avg,
                      upper_limit_avg))
          self.result_recorder[rerun].add_invalidation_reason(
            story_name, self.benchmark, 'Low cpu_wall_time_ratio')
      else:
        print(('[       OK ] {}/{} lower average {}({:.3f}) compared ' +
          'to upper limit({:.3f}).').format(self.benchmark, story_name,
          METRIC_NAME, measured_avg, upper_limit_avg))
def compare_values ( self , values_per_story , rerun = False ) : for story_name in values_per_story : if ( self . is_experimental_story ( story_name ) ) : continue if len ( values_per_story [ story_name ] [ 'ci_095' ] ) == 0 : print ( ( '[  FAILED  ] {}/{} has no valid values for {}. Check ' + 'run_benchmark logs for more information.' ) . format ( self . benchmark , story_name , METRIC_NAME ) ) self . result_recorder [ rerun ] . add_failure ( story_name , self . benchmark ) continue upper_limits = self . upper_limit_data upper_limit_avg = upper_limits [ story_name ] [ 'avg' ] upper_limit_ci = upper_limits [ story_name ] [ 'ci_095' ] lower_limit_cpu_ratio = upper_limits [ story_name ] [ 'cpu_wall_time_ratio' ] measured_avg = np . mean ( np . array ( values_per_story [ story_name ] [ 'averages' ] ) ) measured_ci = np . mean ( np . array ( values_per_story [ story_name ] [ 'ci_095' ] ) ) measured_cpu_ratio = np . mean ( np . array ( values_per_story [ story_name ] [ 'cpu_wall_time_ratio' ] ) ) if ( measured_ci > upper_limit_ci * CI_ERROR_MARGIN and self . is_control_story ( story_name ) ) : print ( ( '[  FAILED  ] {}/{} {} has higher noise ({:.3f}) ' + 'compared to upper limit ({:.3f})' ) . format ( self . benchmark , story_name , METRIC_NAME , measured_ci , upper_limit_ci ) ) self . result_recorder [ rerun ] . add_failure ( story_name , self . benchmark , True ) elif ( measured_avg > upper_limit_avg * AVG_ERROR_MARGIN ) : if ( measured_cpu_ratio >= lower_limit_cpu_ratio ) : print ( ( '[  FAILED  ] {}/{} higher average {}({:.3f}) compared' + ' to upper limit ({:.3f})' ) . format ( self . benchmark , story_name , METRIC_NAME , measured_avg , upper_limit_avg ) ) self . result_recorder [ rerun ] . add_failure ( story_name , self . benchmark ) else : print ( ( '[       OK ] {}/{} higher average {}({:.3f}) compared ' + 'to upper limit({:.3f}). Invalidated for low cpu_wall_time_ratio' ) . format ( self . benchmark , story_name , METRIC_NAME , measured_avg , upper_limit_avg ) ) self . result_recorder [ rerun ] . add_invalidation_reason ( story_name , self . benchmark , 'Low cpu_wall_time_ratio' ) else : print ( ( '[       OK ] {}/{} lower average {}({:.3f}) compared ' + 'to upper limit({:.3f}).' ) . format ( self . benchmark , story_name , METRIC_NAME , measured_avg , upper_limit_avg ) )
Parses the raw CSV data
Compares the values in values_per_story with the upper_limit_data and
determines if the story passes or fails and updates the ResultRecorder.
Parses the raw CSV data Compares the values in values_per_story with the upper_limit_data and determines if the story passes or fails and updates the ResultRecorder .
""" Parses the raw CSV data
    Compares the values in values_per_story with the upper_limit_data and
    determines if the story passes or fails and updates the ResultRecorder.

    Args:
      values_per_story: An array of rows (dict) descriving the CSV results
      rerun: Is this a rerun or initial run
    """ # The experimental stories will not be considered for failing the tests
self
values_per_story
An array of rows (dict) descriving the CSV results
An array of rows ( dict ) descriving the CSV results
rerun
None