# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + deletable=true editable=true
import csv
import datetime
import h5py
import itertools
import numpy as np
import os
import pandas as pd
import sklearn.metrics
import sys
import time

sys.path.append("../src")
import localmodule


for aug_kind_str in ["none", "all"]:

    # Define constants.
    data_dir = localmodule.get_data_dir()
    dataset_name = localmodule.get_dataset_name()
    folds = localmodule.fold_units()
    models_dir = localmodule.get_models_dir()
    units = localmodule.get_units()
    n_units = len(units)
    model_name = "icassp-convnet"
    if not aug_kind_str == "none":
        model_name = "_".join(
            [model_name, "aug-" + aug_kind_str])
    model_dir = os.path.join(models_dir, model_name)

    # Initialize dict of best trials.
    adhoc_best_trials_dict = {}
    
    # Initialize "best-5" matrices.
    adhoc_val_best5_tps = np.zeros((n_units, 5))
    adhoc_val_best5_fps = np.zeros((n_units, 5))
    adhoc_val_best5_tns = np.zeros((n_units, 5))
    adhoc_val_best5_fns = np.zeros((n_units, 5))

    adhoc_test_best5_tps = np.zeros((n_units, 5))
    adhoc_test_best5_fps = np.zeros((n_units, 5))
    adhoc_test_best5_tns = np.zeros((n_units, 5))
    adhoc_test_best5_fns = np.zeros((n_units, 5))


    # Define list of trial combinations.
    list_of_trials = [list(range(5))] * n_units
    combinations = list(itertools.product(*list_of_trials))
    n_combinations = len(combinations)


    # Loop over recording units.
    for test_unit_id, test_unit_str in enumerate(units):

        # Define directory for test unit.
        unit_dir = os.path.join(model_dir, test_unit_str)

        # Create CSV file for metrics.
        metrics_name = "_".join([
            dataset_name,
            model_name,
            test_unit_str,
            "clip-metrics"
        ])
        metrics_path = os.path.join(unit_dir, metrics_name + ".csv")
        metrics_df = pd.read_csv(metrics_path)

        # Find 5 trials maximizing validation accuracy.
        adhoc_val_accs = np.array( \
            metrics_df["Ad hoc validation accuracy (%)"])
        adhoc_val_best5_trials = np.argsort(adhoc_val_accs)[::-1][:5]
        
        
        # Store best trial in dictionary.
        adhoc_val_best1_trial = adhoc_val_best5_trials[0]
        adhoc_best_trials_dict[test_unit_str] = adhoc_val_best1_trial

        # Read best 5 validation TP, FP, TN, and FN.
        adhoc_val_tps = np.array(metrics_df["Ad hoc validation TP"])
        adhoc_val_best5_tps[test_unit_id, :] = \
            adhoc_val_tps[adhoc_val_best5_trials]
        adhoc_val_fps = np.array(metrics_df["Ad hoc validation FP"])
        adhoc_val_best5_fps[test_unit_id, :] = \
            adhoc_val_fps[adhoc_val_best5_trials]
        adhoc_val_tns = np.array(metrics_df["Ad hoc validation TN"])
        adhoc_val_best5_tns[test_unit_id, :] = \
            adhoc_val_tns[adhoc_val_best5_trials]
        adhoc_val_fns = np.array(metrics_df["Ad hoc validation FN"])
        adhoc_val_best5_fns[test_unit_id, :] = \
            adhoc_val_fns[adhoc_val_best5_trials]

        # Read cv-best 5 test TP, FP, TN, and FN.
        adhoc_test_tps = np.array(metrics_df["Ad hoc test TP"])
        adhoc_test_best5_tps[test_unit_id, :] = \
            adhoc_test_tps[adhoc_val_best5_trials]
        adhoc_test_fps = np.array(metrics_df["Ad hoc test FP"])
        adhoc_test_best5_fps[test_unit_id, :] = \
            adhoc_test_fps[adhoc_val_best5_trials]
        adhoc_test_tns = np.array(metrics_df["Ad hoc test TN"])
        adhoc_test_best5_tns[test_unit_id, :] = \
            adhoc_test_tns[adhoc_val_best5_trials]
        adhoc_test_fns = np.array(metrics_df["Ad hoc test FN"])
        adhoc_test_best5_fns[test_unit_id, :] = \
            adhoc_test_fns[adhoc_val_best5_trials]


    # Get mean and standard deviation of validation accuracies.
    comb_best5_adhoc_val_tps = []
    comb_best5_adhoc_val_fps = []
    comb_best5_adhoc_val_tns = []
    comb_best5_adhoc_val_fns = []
    for comb_id in range(n_combinations):
        comb = list(combinations[comb_id])
        comb_best5_adhoc_val_tps.append(np.sum(np.array(
            [adhoc_val_best5_tps[t, comb[t]] for t in range(n_units)])))
        comb_best5_adhoc_val_fps.append(np.sum(np.array(
            [adhoc_val_best5_fps[t, comb[t]] for t in range(n_units)])))
        comb_best5_adhoc_val_tns.append(np.sum(np.array(
            [adhoc_val_best5_tns[t, comb[t]] for t in range(n_units)])))
        comb_best5_adhoc_val_fns.append(np.sum(np.array(
            [adhoc_val_best5_fns[t, comb[t]] for t in range(n_units)])))
    comb_best5_adhoc_val_tps = np.array(comb_best5_adhoc_val_tps)
    comb_best5_adhoc_val_fps = np.array(comb_best5_adhoc_val_fps)
    comb_best5_adhoc_val_tns = np.array(comb_best5_adhoc_val_tns)
    comb_best5_adhoc_val_fns = np.array(comb_best5_adhoc_val_fns)
    comb_best5_adhoc_val_accs =\
        (comb_best5_adhoc_val_tps+comb_best5_adhoc_val_tns)/\
        (comb_best5_adhoc_val_tps+comb_best5_adhoc_val_tns+\
         comb_best5_adhoc_val_fps+comb_best5_adhoc_val_fns)
    mean_best5_adhoc_val_accs = np.mean(comb_best5_adhoc_val_accs)
    std_best5_adhoc_val_accs = np.std(comb_best5_adhoc_val_accs)
    oracle_adhoc_val_accs = np.max(comb_best5_adhoc_val_accs)


    # Get mean and standard deviation of test accuracies.
    comb_best5_adhoc_test_tps = []
    comb_best5_adhoc_test_fps = []
    comb_best5_adhoc_test_tns = []
    comb_best5_adhoc_test_fns = []
    for comb_id in range(n_combinations):
        comb = list(combinations[comb_id])
        comb_best5_adhoc_test_tps.append(np.sum(np.array(
            [adhoc_test_best5_tps[t, comb[t]] for t in range(n_units)])))
        comb_best5_adhoc_test_fps.append(np.sum(np.array(
            [adhoc_test_best5_fps[t, comb[t]] for t in range(n_units)])))
        comb_best5_adhoc_test_tns.append(np.sum(np.array(
            [adhoc_test_best5_tns[t, comb[t]] for t in range(n_units)])))
        comb_best5_adhoc_test_fns.append(np.sum(np.array(
            [adhoc_test_best5_fns[t, comb[t]] for t in range(n_units)])))
    comb_best5_adhoc_test_tps = np.array(comb_best5_adhoc_test_tps)
    comb_best5_adhoc_test_fps = np.array(comb_best5_adhoc_test_fps)
    comb_best5_adhoc_test_tns = np.array(comb_best5_adhoc_test_tns)
    comb_best5_adhoc_test_fns = np.array(comb_best5_adhoc_test_fns)
    comb_best5_adhoc_test_accs =\
        (comb_best5_adhoc_test_tps+comb_best5_adhoc_test_tns)/\
        (comb_best5_adhoc_test_tps+comb_best5_adhoc_test_tns+\
         comb_best5_adhoc_test_fps+comb_best5_adhoc_test_fns)
    mean_best5_adhoc_test_accs = np.mean(comb_best5_adhoc_test_accs)
    std_best5_adhoc_test_accs = np.std(comb_best5_adhoc_test_accs)
    oracle_adhoc_test_accs = np.max(comb_best5_adhoc_test_accs)
    
    print("Augmentation: {}".format(aug_kind_str))
    print(adhoc_best_trials_dict)
    print("Validation accuracy: {:5.3f}% ± {:5.3f} (best 5)".format(
        mean_best5_adhoc_val_accs * 100, std_best5_adhoc_val_accs * 100))
    print("Validation accuracy: {:5.3f}% (oracle)".format(
        oracle_adhoc_val_accs * 100))
    print("Test accuracy: {:5.3f}% ± {:5.3f} (best 5)".format(
        mean_best5_adhoc_test_accs * 100, std_best5_adhoc_test_accs * 100))
    print("Test accuracy: {:5.3f}% (oracle)".format(
        oracle_adhoc_test_accs * 100))
    print("")

# + deletable=true editable=true
metrics_df

StarcoderdataJupyterScriptsDedupFiltered