# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/PhilippMatthes/diplom/blob/master/src/shl-deep-learning-timeseries-results.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + colab={"base_uri": "https://localhost:8080/"} id="RlFUudveAowc" outputId="8f4301d7-5a9c-4c39-bfc8-8f012805f458"
# !git clone https://github.com/philippmatthes/diplom

# + colab={"base_uri": "https://localhost:8080/"} id="NURT9pgkAvm0" outputId="2246e7b8-e922-48c5-c068-7ee1cec54492"
# Switch to our src dir for further processing. This command is specific to 
# Google Colab, so it might not work on your local Jupyter Notebook instance

# %cd /content/diplom/src

# + colab={"base_uri": "https://localhost:8080/"} id="DxxKJxfsAC9A" outputId="3a374df5-d984-4ac1-b5e6-9ae2a64e3cef"
# Mount Google Drive and Docs for data analysis

from google.colab import drive
drive.mount('/content/drive')

# + id="jKknmciaSC9a"
# Mount Docs

from google.colab import auth
auth.authenticate_user()

import gspread
from oauth2client.client import GoogleCredentials

docs = gspread.authorize(GoogleCredentials.get_application_default())

# + id="qBJ4YywtAi8T"
# Copy current savestate from gridsearch

datestr = '2021-08-29'

# !mkdir /content/diplom/src/logs/
# !unzip -n -d /content/diplom/src/logs/shl-gridsearch/ /content/drive/MyDrive/logs-{datestr}.zip
# !unzip -n -d /content/diplom/src/models/shl-gridsearch/ /content/drive/MyDrive/gridsearch-{datestr}.zip

# + colab={"base_uri": "https://localhost:8080/"} id="jNEhEkckAy8x" outputId="26b11cf1-cd29-40dd-f56d-0881fb9d3317"
# !ls /content/diplom/src/logs/shl-gridsearch/

# + colab={"base_uri": "https://localhost:8080/"} id="80zgoeaRA2h-" outputId="8081ed91-65d9-487b-c7df-ebafe0d490af"
# !ls /content/diplom/src/models/shl-gridsearch/

# + id="eTMUDEvpakSZ"
import sys

# !{sys.executable} -m pip install pandas==1.3.0 --upgrade -q

# + colab={"base_uri": "https://localhost:8080/"} id="BK0a_8TdA5iz" outputId="8647e6d1-29e7-43da-8097-b263eea58d12"
import os
import json
import tempfile
import shutil
import zipfile

from packaging import version
from pathlib import Path
from collections import defaultdict, OrderedDict

import tensorflow as tf
import pandas as pd
import tensorboard as tb
import numpy as np

from google.colab.files import download
from matplotlib import pyplot as plt
from tensorboard.backend.event_processing import event_accumulator

major_ver, minor_ver, _ = version.parse(tb.__version__).release
assert major_ver >= 2 and minor_ver >= 3, \
    "This notebook requires TensorBoard 2.3 or later."

print("TensorBoard version: ", tb.__version__)
print("Pandas version: ", pd.__version__)
print("Numpy version: ", np.__version__)


# + id="m5kTcoi2Bood"
def parse_gridsearch_tf_logs(
    log_dir: Path, 
    metric_sources=['train', 'validation'],
    metrics=['epoch_loss', 'epoch_acc'],
) -> dict:
    data = defaultdict(dict)

    for trial_dir in [d for d in log_dir.iterdir() if d.is_dir()]:
        trial_id = trial_dir.parts[-1]
        
        for metric in metrics:
            for metric_source in metric_sources:
                ea = event_accumulator.EventAccumulator(
                    str(trial_dir / 'execution0' / metric_source),
                    size_guidance={event_accumulator.TENSORS: 0}
                )
                ea.Reload()

                if metric not in ea.Tags()['tensors']:
                    continue

                metric_tensors = ea.Tensors(metric)

                data[f'{metric_source}_{metric}'][trial_id] = {
                    'epochs': [t.step for t in metric_tensors],
                    'values': [tf.make_ndarray(t.tensor_proto) for t in metric_tensors],
                    'dates': [t.wall_time for t in metric_tensors],
                }
    
    return data

parsed_data = parse_gridsearch_tf_logs(Path('logs/shl-gridsearch'))

# + colab={"base_uri": "https://localhost:8080/", "height": 457} id="_6qDDvU2PU_9" outputId="6cae6441-b697-4917-a799-cbfa37ca5a19"
plt.clf()

fig, axs = plt.subplots(1, len(parsed_data))
fig.set_size_inches(4 * len(parsed_data), 5)

metric_labels = {
    'train_epoch_loss': 'Train Loss (Log-Skala)',
    'validation_epoch_loss': 'Validation Loss (Log-Skala)',
    'train_epoch_acc': 'Train Accuracy (in %)',
    'validation_epoch_acc': 'Validation Accuracy (in %)',
}

metric_scales = {
    'train_epoch_loss': 'log',
    'validation_epoch_loss': 'log',
    'train_epoch_acc': 'linear',
    'validation_epoch_acc': 'linear',
}

for ax, (metric, trials_data) in zip(axs, parsed_data.items()):
    annotated_lines = 0
    for trial_id, data in trials_data.items():
        y = np.array(data['values'])
        ax.plot(y, color=(0, 0, 0, 0.5))

        if metric == 'validation_epoch_loss':
            x = np.array(list(range(len(y))))
            xmax = x[np.argmax(y)]
            ymax = np.array(y).max()

            if ymax > 1.4:
                text= f'{trial_id[:6]}\n{ymax:.3f}'
                if not ax:
                    ax=plt.gca()
                bbox_props = dict(boxstyle="square,pad=0.3", fc="w", ec="k", lw=1)
                arrowprops=dict(arrowstyle="->",connectionstyle="arc3,rad=0.")
                kw = dict(xycoords='data',textcoords="axes fraction",
                          arrowprops=arrowprops, bbox=bbox_props, ha="right", va="top")
                ax.annotate(text, xy=(xmax, ymax), xytext=(0.94 - annotated_lines * 0.24,0.96), **kw)
                annotated_lines += 1

        ax.set_yscale(metric_scales[metric])
        ax.grid(
            b = True, 
            color ='grey',
            linestyle ='-.', 
            linewidth = 0.5,
            alpha = 0.2
        )
    ax.set_ylabel(metric_labels[metric])
    ax.set_xlabel('Epochen')

axs[0].set_ylim(0, 250)
axs[1].set_ylim(0, 250)
axs[2].set_ylim(0.5, 1)
axs[2].set_yticklabels([int(t * 100) for t in axs[2].get_yticks()])
axs[3].set_ylim(0.5, 1)
axs[3].set_yticklabels([int(t * 100) for t in axs[3].get_yticks()])

plt.tight_layout()
plt.savefig(
    f'../images/shl/gridsearch-results.pdf',
    dpi=1200,
    bbox_inches='tight'
)

# + colab={"base_uri": "https://localhost:8080/", "height": 16} id="a9BPPaCfIZvi" outputId="30bb36e6-c5b7-4cb2-c45e-47985b7b6a8e"
download('../images/shl/gridsearch-results.pdf')

# + colab={"base_uri": "https://localhost:8080/", "height": 325} id="qjxv4iM1V86E" outputId="46962c34-f2cd-41b6-9845-e702f690a7e0"
plt.clf()

fig = plt.figure()
ax1 = fig.add_subplot(111)
ax2 = ax1.twiny()

diffs = []

for metric, trials_data in parsed_data.items():
    for trial_id, data in trials_data.items():
        dates = data['dates']
        diffs.append((dates[-1] - dates[0]) / len(dates) / 60)

ax1.set_yticks([])

ax1.hist(diffs, 20, color='grey')

ax1.set_xlabel('Durchschnittliche Trainingszeit pro Epoche in Minuten')
ax1.set_ylabel('Relative Anzahl Modelle')

ax2.set_xlim(ax1.get_xlim())
ax2.set_xticklabels([(t * 15) / 60 for t in ax1.get_xticks()])
ax2.set_xlabel('Geschätzte Finalisierungszeit für 15 Epochen in Stunden')

plt.savefig(
    f'../images/shl/gridsearch-finalization-prediction.pdf',
    dpi=1200,
    bbox_inches='tight'
)

# + colab={"base_uri": "https://localhost:8080/", "height": 16} id="Kn-feBihIk5Z" outputId="cb1f62f7-1108-4b13-cfe9-1c4e57bda713"
download('../images/shl/gridsearch-finalization-prediction.pdf')

# + id="oQs5Fdx09Bta"
oracle_dir = Path('models/shl-gridsearch')

def load_trial_data():
    trial_data = defaultdict(dict)

    for trial_dir in [d for d in oracle_dir.iterdir() if d.is_dir()]:
        trial_json_dir = trial_dir / 'trial.json'
        with open(trial_json_dir) as f:
            trial_json = json.loads(f.read())
        trial_id = trial_json['trial_id']
        trial_hps = trial_json['hyperparameters']['values']
        for hp, hp_value in trial_hps.items():
            trial_data[hp][trial_id] = hp_value
        for metric, trials_data in parsed_data.items():
            if trial_id not in trials_data:
                continue
            trial_data[metric][trial_id] = trials_data[trial_id]['values']
    
    return trial_data


# + colab={"base_uri": "https://localhost:8080/", "height": 591} id="MLMfs1LiMkYf" outputId="5e0af2ca-5f0d-419f-8c32-2065f15f20d2"
df = pd.DataFrame(load_trial_data())

del df['tuner/initial_epoch']
del df['tuner/bracket']
del df['tuner/round']
del df['tuner/trial_id']
del df['train_epoch_loss']
del df['validation_epoch_loss']

df.dropna(inplace=True)

df['n_epochs'] = df['validation_epoch_acc'].apply(lambda x: len(x))
df['best_epoch'] = df['validation_epoch_acc'].apply(lambda x: np.where(x == np.amax(x))[0][0] + 1)
df['max_train_acc_pct'] = df['train_epoch_acc'].apply(lambda x: np.amax(x) * 100)
df['max_validation_acc_pct'] = df['validation_epoch_acc'].apply(lambda x: np.amax(x) * 100)

del df['tuner/epochs']
del df['validation_epoch_acc']
del df['train_epoch_acc']

df.sort_values(by=['max_validation_acc_pct', 'max_train_acc_pct'], inplace=True, ascending=False)

df.reset_index(inplace=True)

df['id'] = df['index'].apply(lambda x: x[:6])
del df['index']

df = df[[
    'id',
    'model_type',
    'version',
    'pooling',
    'optimizer',
    'n_stacks',
    'learning_rate',
    'n_epochs',
    'best_epoch',
    'max_train_acc_pct',
    'max_validation_acc_pct',
]]

df.rename(columns={
    'id': 'Trial-ID',
    'model_type': 'Typ',
    'version': 'Version',
    'n_stacks': 'Tiefe',
    'pooling': 'Pooling',
    'optimizer': 'Optimizer',
    'learning_rate': 'LR',
    'n_epochs': 'Epochen',
    'best_epoch': 'Beste',
    'max_train_acc_pct': 'Train Acc. (in \%)',
    'max_validation_acc_pct': 'Val. Acc. (in \%)',
}, inplace=True)

styled_df = df.style\
    .format({ 
        'LR': '{:.3f}',
        'Train Acc. (in \%)': '{:.2f}',
        'Val. Acc. (in \%)': '{:.2f}',
    }) \
    .hide_index() \
    .background_gradient(cmap='binary')
styled_df

# + id="DVYDHwb4dzRQ"
with open('../sections/evaluation/gridsearch-results.tex', 'w') as f:
    f.write(styled_df.to_latex(convert_css=True))

# + colab={"base_uri": "https://localhost:8080/", "height": 16} id="oKFnXdOKbUNr" outputId="fb2fa3b3-1c01-4250-ecb5-bb8261944ad6"
download('../sections/evaluation/gridsearch-results.tex')

# + id="5gKq9x-y7RGh" colab={"base_uri": "https://localhost:8080/", "height": 235} outputId="d318f3f8-923a-4a0c-f584-99b177af3d1b"
# Finalization and profiling statistics

df = pd.DataFrame(load_trial_data())
df['max_train_acc_pct'] = df['train_epoch_acc'].apply(lambda x: np.amax(x) * 100)
df['max_validation_acc_pct'] = df['validation_epoch_acc'].apply(lambda x: np.amax(x) * 100)
df = pd.DataFrame(df.nlargest(10, ['max_validation_acc_pct']))

def get_model_size(trial_id):
    with tempfile.TemporaryDirectory() as dir:
        shutil.copyfile(f'/content/drive/MyDrive/finalization-tflite-model-{trial_id}.zip', f'{dir}/model.zip')
        with zipfile.ZipFile(f'{dir}/model.zip') as zip_ref:
            zip_ref.extractall(f'{dir}')
        n_bytes = os.path.getsize(f'{dir}/model.tflite')
    n_megabytes = n_bytes / 1_000_000
    return n_megabytes

def get_model_finalization_log(trial_id):
    with tempfile.TemporaryDirectory() as dir:
        shutil.copyfile(f'/content/drive/MyDrive/finalization-log-{trial_id}.zip', f'{dir}/log.zip')
        with zipfile.ZipFile(f'{dir}/log.zip') as zip_ref:
            zip_ref.extractall(f'{dir}')
        log_df = pd.read_csv(f'{dir}/log.csv')
    return log_df

model_sizes = []
model_logs = []
for trial_id in df.index:
    model_sizes.append(get_model_size(trial_id))
    model_logs.append(get_model_finalization_log(trial_id))

df['model_size_mb'] = model_sizes
df['finalization_log'] = model_logs

df['finalization_epochs'] = df['finalization_log'].apply(lambda log_df: max(log_df['epoch']) + 1)
df['finalization_train_acc_pct'] = df['finalization_log'].apply(lambda log_df: max(log_df['acc']) * 100)
df['finalization_val_acc_pct'] = df['finalization_log'].apply(lambda log_df: max(log_df['val_acc']) * 100)

df['trial_rank'] = [i + 1 for i in range(len(df.index))]

df['max_trial_diff_acc_pct'] = df.apply(lambda row: row['max_train_acc_pct'] - row['max_validation_acc_pct'], axis=1)
df['finalization_diff_acc_pct'] = df.apply(lambda row: row['finalization_train_acc_pct'] - row['finalization_val_acc_pct'], axis=1)

finalization_df = df.copy()
finalization_df = finalization_df[[
    'model_type',
    'trial_rank',
    'max_train_acc_pct',
    'max_trial_diff_acc_pct',
    'max_validation_acc_pct',
    'finalization_train_acc_pct',
    'finalization_diff_acc_pct',
    'finalization_val_acc_pct',
    'model_size_mb'
]]

finalization_df.sort_values(by=['finalization_val_acc_pct', 'finalization_train_acc_pct'], inplace=True, ascending=False)

finalization_df.reset_index(inplace=True)

finalization_df['index'] = finalization_df['index'].apply(lambda x: x[:6])

finalization_df.rename(columns={
    'index': 'Trial-ID',
    'trial_rank': 'Trial-Rang',
    'model_type': 'Typ',
    'version': 'Version',
    'pooling': 'Pooling',
    'optimizer': 'Optimizer',
    'max_train_acc_pct': 'Trial Train Acc. (in \%)',
    'max_trial_diff_acc_pct': 'Trial Acc. Diff (in \%)',
    'max_validation_acc_pct': 'Trial Val. Acc. (in \%)',
    'finalization_train_acc_pct': 'Fin. Train Acc. (in \%)',
    'finalization_val_acc_pct': 'Fin. Val. Acc. (in \%)',
    'finalization_diff_acc_pct': 'Fin. Acc. Diff (in \%)',
    'model_size_mb': 'Größe (in MB)',
}, inplace=True)

finalization_df_styled = finalization_df.style \
    .format({ 
        'Trial Train Acc. (in \%)': '{:.2f}',
        'Trial Val. Acc. (in \%)': '{:.2f}',
        'Trial Acc. Diff (in \%)': '{:.2f}',
        'Fin. Train Acc. (in \%)': '{:.2f}',
        'Fin. Val. Acc. (in \%)': '{:.2f}',
        'Fin. Acc. Diff (in \%)': '{:.2f}',
        'Größe (in MB)': '{:.2f}',
    }) \
    .hide_index() \
    .background_gradient(cmap='binary')

finalization_df_styled

# + id="F_8V0i5wHHSO"
with open('../sections/evaluation/finalization-results.tex', 'w') as f:
    f.write(finalization_df_styled.to_latex(convert_css=True))

# + id="vv20LplQHNg2" colab={"base_uri": "https://localhost:8080/", "height": 16} outputId="09bd602f-876e-44e9-b060-0afcb4fe2063"
download('../sections/evaluation/finalization-results.tex')

# + id="abHzC1GrH00y" colab={"base_uri": "https://localhost:8080/", "height": 602} outputId="a19654cf-ba19-4efe-b073-f26c3cfd005f"
plt.clf()
fig, axs = plt.subplots(2, 5, sharey=True)
fig.set_size_inches(3 * 5, 4 * 2)

for ax, trial_id in zip(axs.flatten(), df.index):
    val_acc = df['finalization_log'][trial_id]['val_acc']
    train_acc = df['finalization_log'][trial_id]['acc']
    ax.plot([acc * 100 for acc in val_acc], color='black')
    ax.plot([acc * 100 for acc in train_acc], color='grey')
    ax.fill_between(
        x=[i for i in range(len(val_acc))],
        y1=[acc * 100 for acc in val_acc],
        y2=[acc * 100 for acc in train_acc],
        color=(0, 0, 0, 0.05)
    )
    ax.set_title(f'Rang {list(df.index).index(trial_id) + 1} ({df["model_size_mb"][trial_id]:0.2f} MB)')
    ax.set_xlabel('Epochen')

    ax.grid(
        b = True, 
        color ='grey',
        linestyle ='-.', 
        linewidth = 0.5,
        alpha = 0.2
    )

    ax.set_ylabel('Accuracy (in %)')
    ax.set_ylim(30, 100)
    ax.set_xlim(0, 15)
    ax.axvline(len(val_acc) - 1, color='black')

    ax.axhline(78.94, color='black', linestyle=':')
    ax.axhline(70.42, color='grey', linestyle=':')

    ax.set_xticks([len(val_acc) - 1])
    es = len(val_acc) != 15

    ax.set_xticklabels([f'{len(val_acc)}{" (Early Stopping)" if es else " (Limit erreicht)"}'])

    rax = ax.twinx()
    rax.set_ylim(ax.get_ylim())
    rax.set_yticks([max([acc * 100 for acc in val_acc]), 78.94, 70.42])
    rax.set_yticklabels(['', 'Basis', 'MLP'])

plt.tight_layout()
plt.savefig(
    f'../images/shl/finalization-results-accuracy.pdf',
    dpi=1200,
    bbox_inches='tight'
)
plt.show()

# + id="XiikFObpRHqX" colab={"base_uri": "https://localhost:8080/", "height": 16} outputId="d331c58f-f427-4424-d772-be0116c51ad0"
download('../images/shl/finalization-results-accuracy.pdf')

# + id="uNZPSG-vRO0V" colab={"base_uri": "https://localhost:8080/", "height": 602} outputId="80fbc37f-87e0-403b-a631-182617355777"
plt.clf()
fig, axs = plt.subplots(2, 5, sharey=True)
fig.set_size_inches(3 * 5, 4 * 2)

for ax, trial_id in zip(axs.flatten(), df.index):
    val_loss = df['finalization_log'][trial_id]['val_loss']
    train_loss = df['finalization_log'][trial_id]['loss']
    ax.plot(val_loss, color='black')
    ax.plot(train_loss, color='grey')
    ax.fill_between(
        x=[i for i in range(len(val_loss))],
        y1=val_loss,
        y2=train_loss,
        color=(0, 0, 0, 0.05)
    )
    ax.set_title(f'Rang {list(df.index).index(trial_id) + 1} ({df["model_size_mb"][trial_id]:0.2f} MB)')
    ax.set_xlabel('Epochen')

    ax.grid(
        b = True, 
        color ='grey',
        linestyle ='-.', 
        linewidth = 0.5,
        alpha = 0.2
    )

    ax.set_ylabel('Sparse Categorical Cross Entropy (Log-Skala)')
    ax.set_yscale('log')
    ax.set_xlim(0, 15)
    ax.axvline(len(val_loss) - 1, color='black')

    ax.set_xticks([len(val_loss) - 1])
    es = len(val_loss) != 15

    ax.set_xticklabels([f'{len(val_loss)}{" (Early Stopping)" if es else " (Limit erreicht)"}'])

plt.tight_layout()
plt.savefig(
    f'../images/shl/finalization-results-loss.pdf',
    dpi=1200,
    bbox_inches='tight'
)
plt.show()

# + id="xwkMn4LrSIfW" colab={"base_uri": "https://localhost:8080/", "height": 16} outputId="96434fba-2199-491a-940f-b48543fe1445"
download('../images/shl/finalization-results-loss.pdf')

# + id="gFuw_zcnMAik" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="4d064ba7-a8ab-49e8-c44e-87d1984f133d"
finalization_order = finalization_df['Trial-ID']
finalization_order[0] # Best model

# + colab={"base_uri": "https://localhost:8080/", "height": 472} id="ZUiXY3oBw4bI" outputId="1b744687-d443-4c23-8a99-6c61faf4f92e"
# Tradeoff

sheet = docs.open('Profiling').sheet1
tradeoff_params = sheet.get_all_values()

tradeoff_param_names = [params[0] for params in tradeoff_params]
tradeoff_param_values = [params[1:] for params in tradeoff_params]

tradeoff_dict = OrderedDict(zip(tradeoff_param_names, tradeoff_param_values))
tradeoff_df = pd.DataFrame.from_records(tradeoff_dict)
tradeoff_df['Trial-ID'] = tradeoff_df['Trial-ID'].apply(lambda id: id[:6])
tradeoff_df.set_index('Trial-ID', inplace=True)

for column in tradeoff_df.columns:
    tradeoff_df[column] = pd.to_numeric(tradeoff_df[column], errors='coerce')

tradeoff_df['Operationen'] = tradeoff_df['Operationen'].astype(int)
tradeoff_df['Operationen auf GPU'] = tradeoff_df['Operationen auf GPU'].astype(int)
tradeoff_df['Operationen auf ANE'] = tradeoff_df['Operationen auf ANE'].astype(int)

tradeoff_df = tradeoff_df[[
    n for n in tradeoff_param_names if n != 'Trial-ID'
]]

tradeoff_df = tradeoff_df.T[reversed(finalization_order)]

tradeoff_df_to_print = tradeoff_df.style.format('{:.2f}', na_rep='k.A.').background_gradient(cmap='binary', axis=1)
tradeoff_df_to_print

# + id="A162yW5sWwY4"
with open('../sections/evaluation/tradeoff-results.tex', 'w') as f:
    f.write(tradeoff_df_to_print.to_latex(convert_css=True))

# + id="oE3U69KrXAOz" colab={"base_uri": "https://localhost:8080/", "height": 16} outputId="b3e16744-cf09-4eb5-d6ee-43b5bad19d91"
download('../sections/evaluation/tradeoff-results.tex')

# + colab={"base_uri": "https://localhost:8080/", "height": 472} id="95QJEIvRJ6EA" outputId="e51a9dfc-9063-4e78-d07a-40c5027a5ad3"
# Optimization Tradeoff

sheet = docs.open('Profiling').worksheet('Optimierung')
tradeoff_params = sheet.get_all_values()

tradeoff_param_names = [params[0] for params in tradeoff_params]
tradeoff_param_values = [params[1:] for params in tradeoff_params]
tradeoff_dict = OrderedDict(zip(tradeoff_param_names, tradeoff_param_values))
tradeoff_df = pd.DataFrame.from_records(tradeoff_dict)
tradeoff_df.set_index('Optimierung', inplace=True)

for column in tradeoff_df.columns:
    tradeoff_df[column] = pd.to_numeric(tradeoff_df[column], errors='coerce')

tradeoff_df['Operationen'] = tradeoff_df['Operationen'].astype(int)
tradeoff_df['Operationen auf GPU'] = tradeoff_df['Operationen auf GPU'].astype(int)
tradeoff_df['Operationen auf ANE'] = tradeoff_df['Operationen auf ANE'].astype(int)

tradeoff_df = tradeoff_df[[
    n for n in tradeoff_param_names if n != 'Optimierung'
]]

tradeoff_df = tradeoff_df.T

tradeoff_df_to_print = tradeoff_df.style.format('{:.2f}', na_rep='k.A.').background_gradient(cmap='binary', axis=1)
tradeoff_df_to_print

# + id="OzCYJgjtMYwe"
with open('../sections/evaluation/optimization-tradeoff-results.tex', 'w') as f:
    f.write(tradeoff_df_to_print.to_latex(convert_css=True))

# + colab={"base_uri": "https://localhost:8080/", "height": 16} id="CczDxxEaMrc0" outputId="01f0a8d2-63f8-4712-ba50-e25392d7de11"
download('../sections/evaluation/optimization-tradeoff-results.tex')

# + id="eYuL507Xw2Ge" colab={"base_uri": "https://localhost:8080/", "height": 313} outputId="90256ddf-870c-41b6-a2b5-1c44e51934cc"
# Results from optimization

from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix

f, axs = plt.subplots(1, 3, figsize=(14, 4))

conf_mat = np.array([ [5738,  55,     0,   19,   33,   39,   89,   63,],
                      [ 219, 5652,   12,   48,    8,   12,   15,   38,],
                      [   4,   10, 1743,    0,    1,    0,    0,    0,],
                      [ 132,   43,    3, 4945,   25,   26,    7,   10,],
                      [ 115,    8,    0,   18, 6546,  413,   73,   56,],
                      [ 211,   24,    0,   31,  558, 5061,  184,   50,],
                      [ 605,   22,    0,   10,  152,  183, 5107, 1064,],
                      [ 383,   18,    1,    5,  111,   71, 1260, 3663,], ])

order = [
    'Still',
    'Walking',
    'Run',
    'Bike',
    'Car',
    'Bus',
    'Train',
    'Subway',
]

conf_mat_disp = ConfusionMatrixDisplay(
    confusion_matrix=conf_mat,
    display_labels=order
)
conf_mat_disp.plot(cmap=plt.cm.binary, ax=axs[1], xticks_rotation=45, values_format='')
axs[1].set_xlabel('Vorhergesagte Klasse')
axs[1].set_ylabel('Tatsächliche Klasse')


axs[0].bar(order, class_accuracies, color='grey')
axs[0].grid(
    b = True, 
    color ='grey',
    linestyle ='-.', 
    linewidth = 0.5,
    alpha = 0.2
)
axs[0].set_xlabel('Klasse')
axs[0].set_ylabel('Accuracy (in %)')
axs[0].set_xticklabels(order, rotation=45)
axs[0].set_ylim(0, 110)
rects = axs[0].patches
labels = [f"{a:0.2f}" for a in class_accuracies]
for rect, label in zip(rects, labels):
    height = rect.get_height()
    axs[0].text(
        rect.get_x() + rect.get_width() / 2, height + 1, label, ha="center", va="bottom"
    )


classes_to_merge = {
    'Foot': ['Walking', 'Run'], 
    'Road': ['Car', 'Bus'], 
    'Rail': ['Subway', 'Train'],
}

merged_order = [
    'Still',
    'Foot',
    'Bike',
    'Road',
    'Rail',
]

merge_buckets = []
for label in merged_order:
    if label in classes_to_merge:
        merge_buckets.append([order.index(l) for l in classes_to_merge[label]])
    else:
        merge_buckets.append([order.index(label)])

merged_conf_mat = np.vstack([np.sum(conf_mat[merge_buckets[n],:],0) for n in range(len(merged_order))])
merged_conf_mat = np.vstack([np.sum(merged_conf_mat[:,merge_buckets[n]],1) for n in range(len(merged_order))]).T

conf_mat_disp = ConfusionMatrixDisplay(
    confusion_matrix=merged_conf_mat,
    display_labels=merged_order
)
conf_mat_disp.plot(cmap=plt.cm.binary, ax=axs[2], xticks_rotation=45, values_format='')
axs[2].set_xlabel('Vorhergesagte Klasse')
axs[2].set_ylabel('Tatsächliche Klasse')
merged_class_accuracies = merged_conf_mat.diagonal() / merged_conf_mat.sum(axis=1) * 100
axs[2].set_title(f'Durchschnittliche Accuracy: {np.average(merged_class_accuracies):.2f}%')

plt.tight_layout()
plt.savefig(
    f'../images/shl/optimization-class-merge.pdf',
    dpi=1200,
    bbox_inches='tight'
)
plt.show()

# + id="ldIq7ENKAXe5" colab={"base_uri": "https://localhost:8080/", "height": 16} outputId="807cdeb4-149f-4f88-ec1a-136a86a66e67"
download('../images/shl/optimization-class-merge.pdf')

# + id="q7u69tz3rBH6" colab={"base_uri": "https://localhost:8080/", "height": 278} outputId="0be0e352-7784-4da4-a886-a6e0185ae3fe"
# Analysis of measurement distortion

cpu_cpu_measurements = [5.0, 5.3999996, 7.2, 4.7, 4.6, 15.8, 4.2000003, 5.2, 5.4, 4.1, 5.0, 5.4, 8.099999, 5.1, 5.5, 13.6, 4.1000004, 5.1000004, 5.2999997, 6.3, 4.9, 5.5, 12.4, 4.6, 4.9, 15.6, 3.9, 5.3, 5.2, 6.2000003, 4.7999997, 5.2999997, 10.3, 5.1, 5.1000004, 14.9, 4.1000004, 5.3, 5.9, 9.7, 4.3, 5.3, 5.5, 8.3, 4.6, 5.3, 10.700001, 4.1, 5.1, 5.4, 5.1, 5.2, 5.1, 12.800001, 5.3999996, 5.0, 5.1000004, 8.5, 4.9, 5.3999996, 8.9, 4.6, 5.3999996, 15.2, 3.8000002, 4.8999996, 5.3, 9.4, 4.7, 5.1, 14.6, 3.8999999, 4.9, 5.0, 9.4, 4.4, 5.4, 11.7, 4.6, 5.2, 14.599999, 4.0, 5.1000004, 5.3999996, 4.1, 5.0, 5.1000004, 6.6000004, 4.6, 5.0, 10.1, 4.2999997, 5.3, 15.4, 4.0, 5.2999997, 5.3, 7.0999994, 3.8999999, 5.3999996]
cpu_gpu_measurements = [5.2, 7.4, 4.9, 5.8, 10.599999, 4.2999997, 4.8999996, 13.1, 3.6, 5.2, 5.1, 7.1, 5.6, 6.1000004, 7.9, 5.5000005, 6.0, 7.9, 5.9, 5.9, 7.7, 5.0, 5.7, 10.0, 4.7000003, 5.4, 13.300001, 4.4, 5.3, 15.499999, 4.2999997, 5.2, 5.3, 4.6, 5.2999997, 5.6, 4.0, 5.2, 5.2, 7.5, 5.6, 6.1000004, 7.9, 5.6000004, 6.1000004, 9.0, 5.4, 5.8, 10.6, 5.1, 5.9, 10.599999, 5.2000003, 5.9, 11.6, 5.1000004, 4.9, 5.9, 7.6, 5.5, 5.4, 10.1, 4.7, 5.4, 14.4, 4.9, 5.8, 5.2, 3.8, 4.6, 5.7, 7.3, 5.2999997, 5.8, 7.9, 5.4, 6.3, 10.8, 5.3, 6.0, 16.300001, 4.6, 5.7, 16.400002, 4.2999997, 5.4, 5.4, 6.8999996, 4.6, 4.9, 9.8, 4.4, 5.0, 4.9, 3.8000002, 4.7, 5.0, 9.900001, 4.4, 4.7]
cpu_ane_measurements = [5.2999997, 11.900001, 4.6, 5.2, 5.5, 5.3, 5.5, 5.2, 9.400001, 4.7999997, 5.2000003, 11.900001, 4.3, 4.7, 5.1000004, 8.3, 3.8999999, 4.8, 5.2999997, 3.9, 4.8999996, 5.3999996, 9.700001, 4.7, 5.0, 14.4, 4.1, 4.8, 5.3, 8.2, 4.7, 5.7, 12.0, 4.2999997, 5.1, 4.9, 10.799999, 4.0, 4.9, 5.1, 6.8, 4.5000005, 5.2000003, 7.7000003, 4.6, 4.9, 14.200001, 4.2000003, 5.1, 5.3, 3.8000002, 4.9, 4.6, 9.3, 4.5, 4.9, 10.400001, 4.2, 5.2999997, 10.6, 4.6, 5.2000003, 5.3999996, 6.8, 4.5, 4.7, 5.1000004, 4.6, 4.9, 5.0, 9.8, 4.3, 5.5, 10.5, 4.2999997, 5.0, 4.7999997, 7.0, 4.7000003, 5.2, 11.500001, 4.2000003, 5.2, 5.5, 6.9000006, 4.6, 5.5, 12.3, 7.2000003, 4.5, 5.2, 9.700001, 4.7000003, 5.2, 5.3999996, 3.8, 4.8, 5.3, 12.0, 4.6]
inf_measurements = [0.005532083, 0.005589125, 0.00558625, 0.005574292, 0.005655167, 0.005617375, 0.005563125, 0.005608042, 0.0056005, 0.005538167, 0.005603458, 0.005517417, 0.005517083, 0.005498083, 0.005537125, 0.005521041, 0.005548417, 0.005532875, 0.005542084, 0.00556725, 0.005539916, 0.005561333, 0.005618583, 0.005585708, 0.005583833, 0.005554167, 0.005556125, 0.005531375, 0.00558825, 0.005575291, 0.005588875, 0.0054875, 0.005561083, 0.0055885, 0.005566417, 0.005603209, 0.0055985, 0.0055835, 0.005586375, 0.005564875, 0.005587666, 0.005586958, 0.00560275, 0.005552625, 0.005557458, 0.005612291, 0.005627208, 0.00557375, 0.005652333, 0.00553025, 0.005495583, 0.005505542, 0.00553625, 0.005502042, 0.005558792, 0.005571542, 0.005528209, 0.005566459, 0.005652375, 0.005577375, 0.005595625, 0.005635125, 0.005617792, 0.005572042, 0.0056695, 0.005593458, 0.005646417, 0.005664125, 0.005572917, 0.005607375, 0.005625, 0.0056545, 0.00560175, 0.005577042, 0.005531708, 0.005524375, 0.005611959, 0.005641583, 0.005615708, 0.005576459, 0.00566225, 0.005680458, 0.005636709, 0.005615708, 0.005590416, 0.00569225, 0.005549208, 0.005531375, 0.005583958, 0.005616167, 0.005597917, 0.005629459, 0.005659, 0.005553083, 0.0056735, 0.005710917, 0.005531708, 0.005517958, 0.005514041, 0.005517709]
inf_measurements = [m * 1000 for m in inf_measurements]

f, axs = plt.subplots(1, 4, figsize=(16, 4))

axs[0].hist(cpu_cpu_measurements, 10, color='grey')
axs[0].axvline(np.mean(cpu_cpu_measurements), color='black')
axs[0].set_xlabel('CPU-Usage in %')
axs[0].set_yticks([])

axs[1].hist(cpu_gpu_measurements, 10, color='grey')
axs[1].axvline(np.mean(cpu_gpu_measurements), color='black')
axs[1].set_xlabel('CPU-Usage (GPU-Beschl.) in %')
axs[1].set_yticks([])

axs[2].hist(cpu_ane_measurements, 10, color='grey')
axs[2].axvline(np.mean(cpu_ane_measurements), color='black')
axs[2].set_xlabel('CPU-Usage (ANE-Beschl.) in %')
axs[2].set_yticks([])

axs[3].hist(inf_measurements, 10, color='grey')
axs[3].axvline(np.mean(inf_measurements), color='black')
axs[3].set_xlabel('Inferenzzeit in ms')
axs[3].set_yticks([])

plt.savefig(
    f'../images/shl/measurement-skew.pdf',
    dpi=1200,
    bbox_inches='tight'
)
plt.show()

# + id="0mvJo_AqxLDR" colab={"base_uri": "https://localhost:8080/", "height": 16} outputId="10567f18-0051-4de4-ee64-abadefe81e12"
download('../images/shl/measurement-skew.pdf')

# + id="dZj8LbP55MPZ" colab={"base_uri": "https://localhost:8080/", "height": 275} outputId="42232cdc-26ec-4b1c-afcd-e39ac39e4d2f"

f, axs = plt.subplots(1, 2, figsize=(12, 4))

colors=['darkgray','gray','dimgray','lightgray']
models = ['4ad2e7 ohne Opt.', 'Pruning', 'Separierbare Schichten', 'Pruning + Sep. Schichten']
threads = [1, 2, 3, 4, 5, 6]

cpu_loads = [
    [6.7600007, 7.3199997, 6.0699997, 6.85],
    [7.8, 7.1800003, 6.5700006, 7.9],
    [7.9, 8.299999, 7.66, 6.8799996],
    [7.4899993, 7.2799997, 7.7599998, 9.079999],
    [14.07, 10.42, 9.7, 11.79],
    [17.8, 17.42, 14.63, 18.89],
]

inference_times = [
    [5.54325167, 5.598065, 5.29341375, 5.59045708],
    [4.464085420000001, 4.47067292, 4.42956667, 4.47041125],
    [5.29119, 5.392499170000001, 5.76115666, 5.27598834],
    [5.81410541, 5.44891041, 5.80792583, 6.26313209],
    [5.934325, 5.38066916, 6.345491250000001, 5.8097875],
    [17.79326875, 18.04454333, 16.02248458, 15.03513167],
]

colors=['lightgray', 'darkgray','gray','dimgray']

cpu_loads_df = pd.DataFrame(cpu_loads)
cpu_loads_df.columns = models
cpu_loads_df['Threads'] = threads
cpu_loads_df.set_index('Threads', inplace=True)
cpu_loads_df.plot(kind='bar', color=colors, ax=axs[0])
axs[0].set_ylabel('CPU-Usage (in %)')
axs[0].set_xlabel('Threads')

inference_times_df = pd.DataFrame(inference_times)
inference_times_df.columns = models
inference_times_df['Threads'] = threads
inference_times_df.set_index('Threads', inplace=True)
inference_times_df.plot(kind='bar', color=colors, ax=axs[1])
axs[1].set_ylabel('Inferenzzeit (in ms)')
axs[1].set_xlabel('Threads')

plt.savefig(
    f'../images/shl/threads.pdf',
    dpi=1200,
    bbox_inches='tight'
)
plt.show()

# + id="GUnIysMO88je" colab={"base_uri": "https://localhost:8080/", "height": 16} outputId="75b27eaf-420e-4634-d2d2-a82b097e6af8"
download('../images/shl/threads.pdf')

# + colab={"base_uri": "https://localhost:8080/", "height": 298} id="FMepwnrEvTC3" outputId="a5cc1e77-7c78-4aa1-d84d-96d18c6c4de0"

f, axs = plt.subplots(1, 3, figsize=(16, 4), sharey=True)

colors = ['lightgray','darkgray','gray','dimgray']
linestyles = ['-', ':', '-.', '--']
models = ['4ad2e7 ohne Opt.', 'Pruning', 'Sep. Schichten', 'Pruning + Sep.']
titles = ['Ausführung auf CPU', 'Ausführung auf GPU', 'Ausführung auf ANE']
rates_hz = [1, 2, 5, 20, 50]

cpu_cpu_inference_times = [5.54325167, 5.598065, 5.29341375, 5.59045708]
cpu_cpu_loads = [
    [1.4599999, 1.01, 1.4699999, 1.4800001],
    [4.6600003, 5.0899997, 4.35, 4.9],
    [14.55, 12.92, 12.429999, 15.029999],
    [34.87, 34.719997, 35.57, 35.34],
    [43.210003, 43.51, 38.689995, 40.6],
]

gpu_cpu_loads = [
    [1.61, 1.1299999, 0.99000007, 0.9200001],
    [4.8300004, 5.3, 4.7499995, 5.1499996],
    [14.309999, 12.75, 11.97, 14.789999],
    [36.57, 33.26, 36.909996, 34.800003],
    [39.480003, 42.339996, 41.0, 41.379997],
]

ane_cpu_loads = [
    [4.04, 2.2899997, 15.17, 2.1499999],
    [11.42, 4.7400002, 10.960001, 5.0999994],
    [20.39, 14.240001, 30.940002, 14.56],
    [43.2, 36.57, 49.280003, 37.649994],
    [35.769997, 42.44, 43.529995, 48.000004],
]

for ax, loads, title in zip(axs, [cpu_cpu_loads, gpu_cpu_loads, ane_cpu_loads], titles):
    data_df = pd.DataFrame(loads)
    data_df.columns = models
    data_df['Hz'] = rates_hz
    data_df.set_index('Hz', inplace=True)
    data_df.plot(kind='line', style=['k+-','ko-','k.--','ks:'], ax=ax)

    for rate in rates_hz:
        ax.axvline(rate, color=(0, 0, 0, 0.05))
    ax.set_xscale('log')
    ax.set_ylabel('CPU-Usage (in %)')
    ax.set_xlabel('Inferenzrate (in Hz, Log-Skala)')
    ax.set_title(title)

plt.savefig(
    f'../images/shl/inference-time.pdf',
    dpi=1200,
    bbox_inches='tight'
)
plt.show()

# + colab={"base_uri": "https://localhost:8080/", "height": 16} id="9WyarI5C1ZWa" outputId="bd48910b-6c39-4d61-d761-ebab16a44195"
download('../images/shl/inference-time.pdf')

# + id="VX0TFgm3FwMg" outputId="0bcdbb73-9bcc-4203-e790-21d4da3add0b" colab={"base_uri": "https://localhost:8080/", "height": 334}
f, ax = plt.subplots(1, 1, figsize=(8, 4))

def decolorize_boxplot(boxplot):
    for element in ['boxes', 'whiskers', 'fliers', 'caps']:
        plt.setp(boxplot[element], color='grey')
    for element in ['means', 'medians']:
        plt.setp(boxplot[element], color='black')

energy_usages = OrderedDict({
    'Youtube (Video-Wiedergabe)': [14, 14, 14, 13, 14, 13, 14, 13, 13, 13, 13, 13, 13, 14, 14, 13, 13, 14, 14, 13, 14],
    'App Store (Aktive Suche)': [19, 19, 19, 19, 19, 19, 18, 17, 14, 18, 19, 19, 19, 19, 19, 19, 18, 19],
    'Twitch (Stream-Wiedergabe)': [15, 18, 17, 15, 15, 14, 13, 15, 15, 13, 15, 14, 14],
    'Twitch (Hintergrund)': [15, 15, 14, 15, 16],
    'Nachrichten': [11, 16, 13, 11, 12, 14, 15, 14, 16, 16, 14, 15, 17, 16, 14, 16, 14],
    'Safari (Suche auf tu-dresden.de)': [10, 15, 16, 16, 17, 15, 18, 16, 12, 15, 18, 16],
    'Maps (Aktive Routensuche)': [13, 18, 19, 18, 19, 17, 18, 17, 17, 15, 15, 15, 17, 17, 18, 18, 18, 19],
    'Maps (Routenverfolgung, Hintergrund)': [13, 14, 15, 15, 16, 15, 13, 15, 13, 13, 13, 13],
    'Kamera': [16, 16, 16, 16, 16, 16, 16, 18, 17, 17, 16, 16, 16, 16, 16, 16, 17],
    'Prototyp': [13, 13, 13, 13, 13, 12, 12, 12, 13, 12, 13],
    'Prototyp (Visualisierung)': [13, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13],
    'Prototyp (GNSS + Visualisierung)': [15, 14, 15, 15, 14, 14, 15, 15, 14, 14, 15, 15, 14, 14, 14],
})

values = [v for v in energy_usages.values()]
keys = [k for k in energy_usages.keys()]

bp = ax.boxplot(values, showfliers=True, vert=False)
ax.set_xlim(0, 20)
ax.set_xticks(range(1, 21))
ax.set_ylabel('Applikation')
ax.set_xlabel('Gemessene AEU')
ax.set_yticklabels(keys, rotation=0)
decolorize_boxplot(bp)
ax.grid(
    b = True, 
    color ='grey',
    linestyle ='-.', 
    linewidth = 0.5,
    alpha = 0.2
)

plt.savefig(
    f'../images/shl/energy-apps.pdf',
    dpi=1200,
    bbox_inches='tight'
)
plt.show()

# + id="JHjvprUFS-L4" outputId="60610a19-039f-4d6c-9817-bb42f799e6a1" colab={"base_uri": "https://localhost:8080/", "height": 16}
download('../images/shl/energy-apps.pdf')

StarcoderdataJupyterScriptsDedupFiltered