tools/android/instant_start/stats/analyze.py
def print_report(runs, model, control='control', experiment='experiment'):
    """Print stats of A/B testing"""
    all_df = pandas.DataFrame(runs, dtype=float)
    report = pandas.DataFrame(
        columns=['Median', 'Diff with control', 'p-value'])
    for metric in sorted(set(all_df['metric_name'])):
        mdf = all_df[all_df['metric_name'] == metric]
        df = pandas.DataFrame()
        for variant in sorted(set(all_df['variant_name'])):
            df[variant] = mdf[mdf['variant_name'] == variant]\
                .value.reset_index(drop=True)

        diff_df = pandas.DataFrame()
        diff_df = df[experiment] - df[control]
        n = len(diff_df)

        row = {}
        row['Median'] = '%.1fms' % df[experiment].median()
        row['Diff with control'] = '%.1fms (%.2f%%)' % (
            diff_df.median(), diff_df.median() / df[experiment].median() * 100)
        row['p-value'] = '%f' % (stats.ttest_rel(df[experiment],
                                                 df[control])[1])
        report = report.append(pandas.Series(data=row, name=metric))
    print('%d samples on %s' % (n, model))
    print(report.sort_values(by='p-value'))
Print stats of A/B testing
Print stats of A/B testing
Print stats of A / B testing
def print_report(runs, model, control='control', experiment='experiment'):
    all_df = pandas.DataFrame(runs, dtype=float)
    report = pandas.DataFrame(
        columns=['Median', 'Diff with control', 'p-value'])
    for metric in sorted(set(all_df['metric_name'])):
        mdf = all_df[all_df['metric_name'] == metric]
        df = pandas.DataFrame()
        for variant in sorted(set(all_df['variant_name'])):
            df[variant] = mdf[mdf['variant_name'] == variant]\
                .value.reset_index(drop=True)
        diff_df = pandas.DataFrame()
        diff_df = df[experiment] - df[control]
        n = len(diff_df)
        row = {}
        row['Median'] = '%.1fms' % df[experiment].median()
        row['Diff with control'] = '%.1fms (%.2f%%)' % (
            diff_df.median(), diff_df.median() / df[experiment].median() * 100)
        row['p-value'] = '%f' % (stats.ttest_rel(df[experiment],
                                                 df[control])[1])
        report = report.append(pandas.Series(data=row, name=metric))
    print('%d samples on %s' % (n, model))
    print(report.sort_values(by='p-value'))
def print_report ( runs , model , control = 'control' , experiment = 'experiment' ) : all_df = pandas . DataFrame ( runs , dtype = float ) report = pandas . DataFrame ( columns = [ 'Median' , 'Diff with control' , 'p-value' ] ) for metric in sorted ( set ( all_df [ 'metric_name' ] ) ) : mdf = all_df [ all_df [ 'metric_name' ] == metric ] df = pandas . DataFrame ( ) for variant in sorted ( set ( all_df [ 'variant_name' ] ) ) : df [ variant ] = mdf [ mdf [ 'variant_name' ] == variant ] . value . reset_index ( drop = True ) diff_df = pandas . DataFrame ( ) diff_df = df [ experiment ] - df [ control ] n = len ( diff_df ) row = { } row [ 'Median' ] = '%.1fms' % df [ experiment ] . median ( ) row [ 'Diff with control' ] = '%.1fms (%.2f%%)' % ( diff_df . median ( ) , diff_df . median ( ) / df [ experiment ] . median ( ) * 100 ) row [ 'p-value' ] = '%f' % ( stats . ttest_rel ( df [ experiment ] , df [ control ] ) [ 1 ] ) report = report . append ( pandas . Series ( data = row , name = metric ) ) print ( '%d samples on %s' % ( n , model ) ) print ( report . sort_values ( by = 'p-value' ) )
Print stats of A/B testing
Print stats of A / B testing
"""Print stats of A/B testing"""
runs
model
control
experiment