# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

import pandas as pd
import numpy as np
import json

#in here are automatic intervention counts that may be wrong and do not exclude intersections and traffic
results = """
Camera v1 (May)
{'mae': 0.23088668332126538, 'rmse': 0.30444885103275254, 'max': 1.3838765496927377, 'failure_rate': 0.961781827385472,'distance': 8464.334414890189, 'distance_per_intervention': 2116.083603722547, 'interventions': 2, 'whiteness': 24.62509993931149, 'cmd_whiteness': 65.02619290238128, 'expert_whiteness': 23.387318346549534}
Camera v2 (May)
{'mae': 0.23823662499297601, 'rmse': 0.3130526038207709, 'max': 1.2721541136976486, 'failure_rate': 0.6917558642678205,'distance': 8363.17488650764, 'distance_per_intervention': 1045.396860813455, 'interventions': 4, 'whiteness': 33.46032825076525, 'cmd_whiteness': 107.73348095698013, 'expert_whiteness': 23.387318346549534}
Camera v3 (May)
{'mae': 0.2402807594393857, 'rmse': 0.3086952556841057, 'max': 1.1453249999323865, 'failure_rate': 0.6642254960401941,'distance': 8389.87918022878, 'distance_per_intervention': 932.2087978031977, 'interventions': 7, 'whiteness': 29.698904637469028, 'cmd_whiteness': 67.55371029928838, 'expert_whiteness': 23.387318346549534}
LiDAR v1 (Nov)
{'mae': 0.2164273510701372, 'rmse': 0.28794647831162357, 'max': 1.4814049216935727, 'failure_rate': 0.4200621692010418,'distance': 8442.519560651908, 'distance_per_intervention': 2110.629890162977, 'interventions': 2, 'whiteness': 22.95757235057836, 'cmd_whiteness': 60.71982088365033, 'expert_whiteness': 22.923865138522743}
LiDAR v2 (Nov)
{'mae': 0.23951848356004346, 'rmse': 0.32021653474413686, 'max': 1.410733995029802, 'failure_rate': 0.9813789632611978,'distance': 8465.916553584271, 'distance_per_intervention': 2116.479138396068, 'interventions': 2, 'whiteness': 17.707242631777568, 'cmd_whiteness': 40.788217567179025, 'expert_whiteness': 22.923865138522743}
LiDAR v3 (Nov)
{'mae': 0.2536249345345471, 'rmse': 0.3528786544085774, 'max': 1.6002952563202872, 'failure_rate': 2.1795842800639567, 'distance': 8432.327240726016, 'distance_per_intervention': 1686.465448145203, 'interventions': 3, 'whiteness': 18.841146974682626, 'cmd_whiteness': 56.74452412521819, 'expert_whiteness': 22.923865138522743}
Camera overfit (May)
{'mae': 0.24525640421044248, 'rmse': 0.3291476438504756, 'max': 1.2536699163819658, 'failure_rate': 1.3851891315929674,'distance': 8489.138533056424, 'distance_per_intervention': 2829.7128443521415, 'interventions': 3, 'whiteness': 23.0742793721048, 'cmd_whiteness': 65.94352651549603, 'expert_whiteness': 23.387318346549534}
LiDAR overfit (Nov)
{'mae': 0.2598746390647679, 'rmse': 0.3802804511875996, 'max': 1.6692438325569796, 'failure_rate': 4.375209942895532,'distance': 8436.864018329401, 'distance_per_intervention': 2812.2880061098003, 'interventions': 0, 'whiteness': 19.209361200653984, 'cmd_whiteness': 38.54266349696802, 'expert_whiteness': 22.923865138522743}
LiDAR night
{'mae': 0.23909138446324246, 'rmse': 0.32493206282364406, 'max': 1.659170327718653, 'failure_rate': 1.2672413793103448,'distance': 8216.289369532735, 'distance_per_intervention': 684.6907807943945, 'interventions': 8, 'whiteness': 25.568653921414345, 'cmd_whiteness': 69.0679117344516, 'expert_whiteness': 22.923865138522743}
LiDAR night #2
{'mae': 0.23140690498687244, 'rmse': 0.3148313254633656, 'max': 1.6218812651299235, 'failure_rate': 1.5501905972045744,'distance': 8376.642020810223, 'distance_per_intervention': 1396.1070034683705, 'interventions': 3, 'whiteness': 20.270503405325538, 'cmd_whiteness': 49.14729491131972, 'expert_whiteness': 22.923865138522743}
LiDAR overfit night
{'mae': 0.25314860436451386, 'rmse': 0.3459086938928751, 'max': 1.630496452339, 'failure_rate': 1.5185648727576135,'distance': 8521.477898587687, 'distance_per_intervention': 2840.4926328625625, 'interventions': 1, 'whiteness': 21.52354723505918, 'cmd_whiteness': 67.14565934366986, 'expert_whiteness': 22.923865138522743}
LiDAR Winter v1
{'mae': 0.24077287766041175, 'rmse': 0.32074558957088006, 'max': 1.5369283242890857, 'failure_rate': 0.9426619389431808,'distance': 8080.505297855708, 'distance_per_intervention': 404.0252648927854, 'interventions': 19, 'whiteness': 38.38490904517742, 'cmd_whiteness': 107.12677614094522, 'expert_whiteness': 22.923865138522743}
LiDAR Winter v2
{'mae': 0.2971095664175751, 'rmse': 0.4105259810386456, 'max': 1.5934707554255736, 'failure_rate': 3.536532641591007,'distance': 7980.378562030141, 'distance_per_intervention': 306.93763700115926, 'interventions': 22, 'whiteness': 37.597063418165874, 'cmd_whiteness': 118.53725563410337, 'expert_whiteness': 22.923865138522743}
LiDAR Winter v3
{'mae': 0.256597960271629, 'rmse': 0.3449099860369303, 'max': 1.6163590616877834, 'failure_rate': 1.643884570713839,'distance': 7698.891989550286, 'distance_per_intervention': 197.4074869115458, 'interventions': 34, 'whiteness': 42.15369836835372, 'cmd_whiteness': 145.37316860331046, 'expert_whiteness': 22.923865138522743}
LiDAR v2 day 2 (Nov)
{'mae': 0.21934381266983838, 'rmse': 0.31392314409302075, 'max': 1.6426648453005086, 'failure_rate': 1.968503937007874,'distance': 8491.618679568586, 'distance_per_intervention': 2830.5395598561954, 'interventions': 0, 'whiteness': 19.17217743456574, 'cmd_whiteness': 33.89063165134626, 'expert_whiteness': 22.923865138522743}
LiDAR intensity (Nov)
{'mae': 0.3267694308573402, 'rmse': 0.4732339915443102, 'max': 1.6593815307920154, 'failure_rate': 7.01828292189738,'distance': 8446.16604232979, 'distance_per_intervention': 2111.5415105824477, 'interventions': 2, 'whiteness': 23.961064265768467, 'cmd_whiteness': 53.783987250236294, 'expert_whiteness': 22.923865138522743}
LiDAR range (Nov)
{'mae': 0.6122139347238139, 'rmse': 0.7695435033217324, 'max': 1.6757036288168157, 'failure_rate': 19.954562665656947,'distance': 1679.012717905246, 'distance_per_intervention': 76.31875990478392, 'interventions': 22, 'whiteness': 29.858233482499045, 'cmd_whiteness': 64.02717916493413, 'expert_whiteness': 22.923865138522743}
LiDAR ambience (Nov)
{'mae': 0.728352878106602, 'rmse': 0.8773714729402442, 'max': 1.7017581466320282, 'failure_rate': 17.49271137026239,'distance': 329.4733138441798, 'distance_per_intervention': 17.340700728641043, 'interventions': 19, 'whiteness': 168.18447945397463, 'cmd_whiteness': 555.1688851900144, 'expert_whiteness': 22.923865138522743}

"""

results = results.split('\n')[1:-1]
names = results[::2]
experiment_results = results[1::2]

# +
import warnings
warnings.filterwarnings('ignore')

metrics_df = pd.DataFrame(columns=["model", 'mae', 'rmse', 'max', 'failure_rate', 'distance', 'distance_per_intervention', 'interventions', 'whiteness', 'cmd_whiteness', 'expert_whiteness'])
for name, res in zip(names, experiment_results):
    print(name)
    res = json.loads(res.replace("'", '"'))
    metrics = pd.DataFrame(data=res, index=[0])
    metrics['model'] = name
    metrics_df = metrics_df.append(metrics, ignore_index=True)
metrics_df.set_index("model");
metrics_df = metrics_df[["model", 'mae', 'rmse', 'max', 'failure_rate', 'distance', 'distance_per_intervention', 'interventions', 'whiteness', 'cmd_whiteness', 'expert_whiteness']]
metrics_df

# +
metrics_df.insert(11, 'ol_season_whiteness', [63.201572, 54.70426, 63.663876,
                                              108.563, 97.656, 107.233, 
                                              59.91,81.347, #overfits
                                              107.233, 107.233, 81.3, #night
                                400.582, 462.965, 370.778,
                                97.656, 111.384, 101.421, 204.356])

metrics_df.insert(12, 'ol_mae', [6.2567,6.7220,6.6836,
                                 7.944, 8.223, 8.132, 
                                 5.9565,6.299, #overfits
                                 8.132, 8.132, 6.3, #night
                                 43.140, 58.815, 32.979, #winter
                                 8.223, 7.547, 11.102, 12.570])
# -

metrics_df["distance_per_intervention"] = metrics_df.apply(lambda x: x['distance']/x['interventions'] if x['interventions'] != 0 else x['distance'], axis=1)

metrics_df

metrics_df.rename(columns={'whiteness': 'cl_whiteness'}, inplace=True)
metrics_df.rename(columns={'cmd_whiteness': 'cl_cmd_whiteness'}, inplace=True)
metrics_df.rename(columns={'mae': 'cl_mae'}, inplace=True)

metrics_df = metrics_df.astype({"interventions": float})

# +
#metrics_df.drop([13, 14, 18], inplace=True)
# -

metrics_df.drop(['rmse', 'max', 'expert_whiteness'], axis=1, inplace=True)

cols = metrics_df.columns.tolist()
print(cols)
order=['model', 'distance', 'interventions','distance_per_intervention', 'cl_mae', 'failure_rate', 'cl_whiteness', 'ol_mae','ol_season_whiteness']
metrics_df=metrics_df[order]

metrics_df.style.format({
    'cl_mae': "{:.4f}m",
    'rmse': "{:.4f}m",
    'max': "{:.4f}m",
    'failure_rate': "{:.2f}%",
    'interventions': "{:.0f}",
    'cl_whiteness': "{:.2f}°/s",
    'ol_whiteness': "{:.2f}°/s",
    'cl_cmd_whiteness': "{:.2f}°/s",
    'ol_season_whiteness': "{:.2f}°/s",
    'ol_mae': "{:.2f}°",
    'distance': "{:.2f}m",
    'distance_per_intervention': "{:.2f}m"
})

StarcoderdataJupyterScriptsDedupFiltered