qa/L0_stability_metrics/check_results.py
def check_metrics_stability(self):
        """
        Makes sure that the same configuration
        appears as the best across iterations for
        each model
        """

        # There should be 4 csv files the results path
        pathname = os.path.join(self._result_path, 'result_*.csv')
        csv_contents = []
        for filename in glob.glob(pathname):
            with open(filename, 'r+') as f:
                csv_contents.append(f.read())

        # Now in the first csv, get the metrics
        metric_values = {}
        for csv in csv_contents:
            csv_lines = csv.split('\n')
            for line in csv_lines[1:-2]:
                model, _, _, gpu_memory, gpu_utilization = line.split(',')
                if model in metric_values:
                    metric_values[model]['gpu_memory'].append(float(gpu_memory))
                    metric_values[model]['gpu_utilization'].append(
                        float(gpu_utilization))
                else:
                    metric_values[model] = {
                        'gpu_memory': [float(gpu_memory)],
                        'gpu_utilization': [float(gpu_utilization)]
                    }

        # Compare metrics
        for model in metric_values:
            for metric, values in metric_values[model].items():
                start_value = values[0]
                for value in values[1:]:
                    deviation_percent = abs(
                        (value - start_value) / start_value) * 100
                    if deviation_percent > self._tolerance:
                        print(
                            f"\n***"
                            f"\n***  For model {model}, value for metric {metric}"
                            "\n***  is unstable.\n***\n"
                            f"\n***\n***  Expected: {start_value} +/- {self._tolerance*start_value/100}."
                            f"\n***  Found: {values[1:]}.\n***")
                        return False
        return True

        Makes sure that the same configuration
        appears as the best across iterations for
        each model
        
Makes sure that the same configuration
appears as the best across iterations for
each model
Makes sure that the same configuration appears as the best across iterations for each model
def check_metrics_stability(self):
        pathname = os.path.join(self._result_path, 'result_*.csv')
        csv_contents = []
        for filename in glob.glob(pathname):
            with open(filename, 'r+') as f:
                csv_contents.append(f.read())
        metric_values = {}
        for csv in csv_contents:
            csv_lines = csv.split('\n')
            for line in csv_lines[1:-2]:
                model, _, _, gpu_memory, gpu_utilization = line.split(',')
                if model in metric_values:
                    metric_values[model]['gpu_memory'].append(float(gpu_memory))
                    metric_values[model]['gpu_utilization'].append(
                        float(gpu_utilization))
                else:
                    metric_values[model] = {
                        'gpu_memory': [float(gpu_memory)],
                        'gpu_utilization': [float(gpu_utilization)]
                    }
        for model in metric_values:
            for metric, values in metric_values[model].items():
                start_value = values[0]
                for value in values[1:]:
                    deviation_percent = abs(
                        (value - start_value) / start_value) * 100
                    if deviation_percent > self._tolerance:
                        print(
                            f"\n***"
                            f"\n***  For model {model}, value for metric {metric}"
                            "\n***  is unstable.\n***\n"
                            f"\n***\n***  Expected: {start_value} +/- {self._tolerance*start_value/100}."
                            f"\n***  Found: {values[1:]}.\n***")
                        return False
        return True
def check_metrics_stability ( self ) : pathname = os . path . join ( self . _result_path , 'result_*.csv' ) csv_contents = [ ] for filename in glob . glob ( pathname ) : with open ( filename , 'r+' ) as f : csv_contents . append ( f . read ( ) ) metric_values = { } for csv in csv_contents : csv_lines = csv . split ( '\n' ) for line in csv_lines [ 1 : - 2 ] : model , _ , _ , gpu_memory , gpu_utilization = line . split ( ',' ) if model in metric_values : metric_values [ model ] [ 'gpu_memory' ] . append ( float ( gpu_memory ) ) metric_values [ model ] [ 'gpu_utilization' ] . append ( float ( gpu_utilization ) ) else : metric_values [ model ] = { 'gpu_memory' : [ float ( gpu_memory ) ] , 'gpu_utilization' : [ float ( gpu_utilization ) ] } for model in metric_values : for metric , values in metric_values [ model ] . items ( ) : start_value = values [ 0 ] for value in values [ 1 : ] : deviation_percent = abs ( ( value - start_value ) / start_value ) * 100 if deviation_percent > self . _tolerance : print ( f"\n***" f"\n***  For model {model}, value for metric {metric}" "\n***  is unstable.\n***\n" f"\n***\n***  Expected: {start_value} +/- {self._tolerance*start_value/100}." f"\n***  Found: {values[1:]}.\n***" ) return False return True
Makes sure that the same configuration
appears as the best across iterations for
each model
Makes sure that the same configuration appears as the best across iterations for each model
"""
        Makes sure that the same configuration
        appears as the best across iterations for
        each model
        """ # There should be 4 csv files the results path # Now in the first csv, get the metrics # Compare metrics
self