### Load necessary libraries ###

import numpy as np
from sklearn.model_selection import KFold
from sklearn.metrics import accuracy_score

import tensorflow as tf
from tensorflow import keras

from sklearn.metrics import ConfusionMatrixDisplay

model = get_network()
model.summary()

### Train and evaluate via 10-Folds cross-validation ###
accuracies = []
folds = np.array(['fold1','fold2','fold3','fold4',
                  'fold5','fold6','fold7','fold8',
                  'fold9','fold10'])
load_dir = "UrbanSounds8K/processed/"
kf = KFold(n_splits=10)
for train_index, test_index in kf.split(folds):
    x_train, y_train = [], []
    for ind in train_index:
        # read features or segments of an audio file
        train_data = np.load("{0}/{1}.npz".format(load_dir,folds[ind]), 
                       allow_pickle=True)
        # for training stack all the segments so that they are treated as an example/instance
        features = np.concatenate(train_data["features"], axis=0) 
        labels = np.concatenate(train_data["labels"], axis=0)
        x_train.append(features)
        y_train.append(labels)

    # stack x,y pairs of all training folds 
    x_train = np.concatenate(x_train, axis = 0).astype(np.float32)
    y_train = np.concatenate(y_train, axis = 0).astype(np.float32)
    
    # for testing we will make predictions on each segment and average them to 
    # produce single label for an entire sound clip.
    test_data = np.load("{0}/{1}.npz".format(load_dir,
                   folds[test_index][0]), allow_pickle=True)
    x_test = test_data["features"]
    y_test = test_data["labels"]

    log_dir="logs/fit/" + folds[test_index][0]
    tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)

    model = get_network()
    model.fit(x_train, y_train, epochs = 20, batch_size = 64, verbose = 1, validation_split=0.2,
                use_multiprocessing=True, workers=8, callbacks=[tensorboard_callback])
    
    # evaluate on test set/fold
    y_true, y_pred = [], []
    for x, y in zip(x_test, y_test):
        # average predictions over segments of a sound clip
        avg_p = np.argmax(np.mean(model.predict(x), axis = 0))
        y_pred.append(avg_p) 
        # pick single label via np.unique for a sound clip
        y_true.append(np.unique(y)[0]) 
    accuracies.append(accuracy_score(y_true, y_pred))    
    print("Fold n accuracy: {0}".format(accuracy_score(y_true, y_pred)))

    cm = ConfusionMatrixDisplay.from_predictions(y_true, y_pred)
    cm.figure_.savefig('conf_mat_' + str(test_index) + '_acc_' + str(accuracy_score(y_true, y_pred)) + '.png',dpi=1000)    

print("Average 10 Folds Accuracy: {0}".format(np.mean(accuracies)))

[((365, 470), 'numpy.array', 'np.array', (["['fold1', 'fold2', 'fold3', 'fold4', 'fold5', 'fold6', 'fold7', 'fold8',\n    'fold9', 'fold10']"], {}), "(['fold1', 'fold2', 'fold3', 'fold4', 'fold5', 'fold6', 'fold7',\n    'fold8', 'fold9', 'fold10'])\n", (373, 470), True, 'import numpy as np\n'), ((539, 557), 'sklearn.model_selection.KFold', 'KFold', ([], {'n_splits': '(10)'}), '(n_splits=10)\n', (544, 557), False, 'from sklearn.model_selection import KFold\n'), ((1696, 1761), 'tensorflow.keras.callbacks.TensorBoard', 'tf.keras.callbacks.TensorBoard', ([], {'log_dir': 'log_dir', 'histogram_freq': '(1)'}), '(log_dir=log_dir, histogram_freq=1)\n', (1726, 1761), True, 'import tensorflow as tf\n'), ((2468, 2523), 'sklearn.metrics.ConfusionMatrixDisplay.from_predictions', 'ConfusionMatrixDisplay.from_predictions', (['y_true', 'y_pred'], {}), '(y_true, y_pred)\n', (2507, 2523), False, 'from sklearn.metrics import ConfusionMatrixDisplay\n'), ((945, 991), 'numpy.concatenate', 'np.concatenate', (["train_data['features']"], {'axis': '(0)'}), "(train_data['features'], axis=0)\n", (959, 991), True, 'import numpy as np\n'), ((1010, 1054), 'numpy.concatenate', 'np.concatenate', (["train_data['labels']"], {'axis': '(0)'}), "(train_data['labels'], axis=0)\n", (1024, 1054), True, 'import numpy as np\n'), ((2349, 2379), 'sklearn.metrics.accuracy_score', 'accuracy_score', (['y_true', 'y_pred'], {}), '(y_true, y_pred)\n', (2363, 2379), False, 'from sklearn.metrics import accuracy_score\n'), ((2695, 2714), 'numpy.mean', 'np.mean', (['accuracies'], {}), '(accuracies)\n', (2702, 2714), True, 'import numpy as np\n'), ((1179, 1210), 'numpy.concatenate', 'np.concatenate', (['x_train'], {'axis': '(0)'}), '(x_train, axis=0)\n', (1193, 1210), True, 'import numpy as np\n'), ((1246, 1277), 'numpy.concatenate', 'np.concatenate', (['y_train'], {'axis': '(0)'}), '(y_train, axis=0)\n', (1260, 1277), True, 'import numpy as np\n'), ((2425, 2455), 'sklearn.metrics.accuracy_score', 'accuracy_score', (['y_true', 'y_pred'], {}), '(y_true, y_pred)\n', (2439, 2455), False, 'from sklearn.metrics import accuracy_score\n'), ((2309, 2321), 'numpy.unique', 'np.unique', (['y'], {}), '(y)\n', (2318, 2321), True, 'import numpy as np\n'), ((2593, 2623), 'sklearn.metrics.accuracy_score', 'accuracy_score', (['y_true', 'y_pred'], {}), '(y_true, y_pred)\n', (2607, 2623), False, 'from sklearn.metrics import accuracy_score\n')]