# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# +
import csv
import re
import math
import matplotlib.pyplot as plt
import numpy as np
import sys
import pickle
sys.path.append("..") # adds higher directory to python modules path

from LoaderPACK.Loader import load_whole_data, load_shuffle_5_min
import torch
# -

# # Section 1: Visualize the eeg-recording and targets

# In the following a whole eeg-recording will be loaded:

# +
device = "cpu"

# the load_whole_data laods a whole eeg-recording and targets into memory
trainload = load_whole_data(path = "C:/Users/Marc/Desktop/model_data/train_model_data", 
                            series_dict = 'train_series_length.pickle', 
                            ind = [i for i in range(1, 195 + 1)])

trainloader = torch.utils.data.DataLoader(trainload,
                                          batch_size=1,
                                          shuffle=False,
                                          num_workers=0)
it = iter(trainloader)
model_input, model_target, model_data = next(it)

torch.set_printoptions(edgeitems=2)

print(model_input[0])

print()

print(model_target[0])

print()

print(model_input[0].shape)
# -

# Thus 4 channels from this individuals recording will be used in the training process.

# The network will be trained on only 5 minutes of data. Thus the above recorded data is passed into another dataloader. This dataloader cuts the recording into 5 min. intervals:

# +
device = "cpu"

# the load_whole_data laods a whole eeg-recording and targets into memory
trainload = load_whole_data(path = "C:/Users/Marc/Desktop/model_data/train_model_data", 
                            series_dict = 'train_series_length.pickle', 
                            ind = [i for i in range(1, 195 + 1)])

trainloader = torch.utils.data.DataLoader(trainload,
                                          batch_size=1,
                                          shuffle=False,
                                          num_workers=0)
it = iter(trainloader)

batch_size = 1

# the load_shuffle_5_min cuts the recording and targets loaded from load_whole_data
# into random 5 minutes intervals.
loader2 = load_shuffle_5_min(next(it), device)
loader2 = torch.utils.data.DataLoader(loader2,
                                      batch_size=batch_size,
                                      shuffle=True,
                                      num_workers=0)

it2 = iter(loader2)
model_input, model_target, model_data = next(it2)

torch.set_printoptions(edgeitems=2)

print(model_input[0][0])

print(model_target[0][0])


# -

# # Section 2: Equalize dataloading

# The Lorenz curve will be used to visualize the equality of the data.

def lorenz_curve(ls: list, name: str):
    """
    This function return a lorenz curve for the lenght of the patients
    eeg recordings.

    Args:
        ls (list): list with the length of each eeg recording.
        name (str): the name - or sting displayed in the title of the 
                    plot.

    Return:
        matplotlib plot
    """
    
    tot_sum = sum(ls) # get the total sum of intervals
    nr_patients = len(ls) # get the number of patients
    sorted_ls = sorted(ls) # sort the list
    
    res = []
    rang = [i/100 for i in range(0, 100 + 5, 5)] # get the percentage range
    for per in rang:
        res.append(sum(sorted_ls[:math.floor(nr_patients * per)])/tot_sum)
    
    plt.title(f"Lorenz curve of {name}")
    plt.xlabel("Cummulative % of patients")
    plt.ylabel("Cummulative % of intervals")
    plt.plot(rang, res)
    plt.plot(rang, rang)
    return plt


# For the train encoding file, each patient can be linked with his recordings:

# +
path = "C:/Users/Marc/Desktop/model_data/train_model_data"

patient_samples = dict()

with open(path + '/train_encoding.csv', 'r') as file:
    ls = csv.reader(file)
    for rows in ls:
        m = re.match(r".+/\d+/(\d+)/+.", rows[0])
        val_to_dict = patient_samples.get(m.group(1), [])
        val_to_dict.append(rows[2])
        patient_samples[m.group(1)] = val_to_dict
# -

print(patient_samples)

# ## Show the data with no correction

# +
import time


device = "cpu"
print(device)

# the load_whole_data laods a whole eeg-recording and targets into memory
trainload = load_whole_data(path = "C:/Users/Marc/Desktop/model_data/train_model_data", 
                            series_dict = 'train_series_length.pickle', 
                            ind = [i for i in range(1, 195 + 1)])


trainloader = torch.utils.data.DataLoader(trainload,
                                          batch_size=1,
                                          shuffle=False,
                                          num_workers=0)

data_before = [] # list to save the amount of 5 min. cuts per recording

start = time.time()

for file in trainloader:
    size = (file[0][0].shape[0], file[0][0].shape[1])
    length = math.floor(((size[1]-200*30)/(200*60*5)))*size[0] 
    # the amount of total possible cuts for the recording
    
    data_before.append(length)

print("time:", time.time()-start)
    

# -

# The amount of 5 minute intervals per recording can be displayed with a histogram

# +
plt.hist(data_before, bins = 30)
plt.title("Histogram over intervals per recording before correction")
plt.savefig(f"C:/Users/Marc/Desktop/BP photos/hist_exp_before.jpg")
plt.show()
plt.close()


print(np.mean(data_before))
print(np.median(data_before))
# -

# It can be seen, that there is a few but very large samples (way larger than the mean).

# The above plot is for the amount of 5 min. intervals per recording. A patient might have multiple recordings. Thus in the following the amount of 5 min. intervals per patient will be found:

# +
patient_before = []

for value in patient_samples.values(): # go though the patient indicies
    vals = 0
    for i in value:
        vals += data_before[int(i)-1]
    
    patient_before.append(vals)

# +
plt.hist(patient_before, bins = 30)
plt.title("Histogram over intervals per patient before correction")
plt.savefig(f"C:/Users/Marc/Desktop/BP photos/hist_patient_before.jpg")
plt.show()
plt.close()

print(np.mean(patient_before))
print(np.median(patient_before))
# -

# It can be seen from the histogram, that a few patients constitute with a lot of the 5 min. intervals in the data. Let's investigate how much data the top 25% of patients constitute with.

sorted_patient_before = sorted(patient_before, reverse=True)
print("Total number of patients:", len(patient_before))
print("25% of patients amounts to:", int(0.25*len(patient_before)))
print("The top 25% of patients constitute with:",
      sum(sorted_patient_before[:int(0.25*len(patient_before))])/sum(patient_before),
      "of the data")

# This can also be seen in the plot of the Lorenz curve:

# +
m = lorenz_curve(patient_before, "intervals before correction")

plt.savefig(f"C:/Users/Marc/Desktop/BP photos/Lorenz_before.jpg")
plt.show()
plt.close()

# -

# Thus it can be seen that a few patients constitute with a large amount of the data.

# ## Show the data with correction

# To try and eliminate this inequality a few steps has to be made in the dataset:

# +
import time


device = "cpu"
print(device)

# the load_whole_data laods a whole eeg-recording and targets into memory
trainload = load_whole_data(path = "C:/Users/Marc/Desktop/model_data/train_model_data", 
                            series_dict = 'train_series_length.pickle', 
                            ind = [i for i in range(1, 195 + 1)])


trainloader = torch.utils.data.DataLoader(trainload,
                                          batch_size=1,
                                          shuffle=False,
                                          num_workers=0)

data_after = []

nr_of_files_loaded = 0

batch_size = 1

start = time.time()

for file in trainloader:
    # instead of loading the total amount of 5 min. intervals possible
    # this dataloader tries to weight the recordings
    # thus short recordings will be represented as much as longer recordings
    loader2 = load_shuffle_5_min(file, device)
    loader2 = torch.utils.data.DataLoader(loader2,
                                          batch_size=batch_size,
                                          shuffle=True,
                                          num_workers=0)
    nr_of_files_loaded += 1
    j = 0
    for i in loader2:
        j += 1
    data_after.append(j) # append the amount of series loaded 

print("time:", time.time()-start)
    
print("Number of files loaded in total:", nr_of_files_loaded)

# -

# This data can now be plotted with a histogram:

# +
plt.title("Histogram over intervals per recording after correction")
plt.hist(data_after, bins = 30)
plt.savefig(f"C:/Users/Marc/Desktop/BP photos/hist_exp_after.jpg")
plt.show()
plt.close()

print(np.mean(data_after))
print(np.median(data_after))
# -

# Now it can be seen, that there are no longer any extreme cases.

# The amount of 5 min. intevals per patient will be found:

# +
patient_after = []

for value in patient_samples.values():
    vals = 0
    for i in value:
        vals += data_after[int(i)-1]
    
    patient_after.append(vals)

# +
plt.title("Histogram over intervals per patient after correction")
plt.hist(patient_after, bins = 30)
plt.savefig(f"C:/Users/Marc/Desktop/BP photos/hist_patient_after.jpg")
plt.show()
plt.close()



print(np.mean(patient_after))
print(np.median(patient_after))
# -

# This again show, that a more fair distribution has been achived.

# After the correction the amount of data that the top 25% of patients constitute with is:

sorted_patient_after = sorted(patient_after, reverse=True)
print("Total number of patients:", len(patient_after))
print("25% of patients amounts to:", int(0.25*len(patient_after)))
print("The top 25% of patients constitute with:",
      sum(sorted_patient_after[:int(0.25*len(patient_after))])/sum(patient_after),
      "of the data")

# This more fair distribution can also be seen in the following Lorenz curve:

# +
m = lorenz_curve(patient_after, "intervals after correction")

plt.savefig(f"C:/Users/Marc/Desktop/BP photos/Lorenz_after.jpg")
plt.show()
plt.close()
# -

# ## Show the data with extreme correction

# +
import time


device = "cpu"
print(device)

# the load_whole_data laods a whole eeg-recording and targets into memory
trainload = load_whole_data(path = "C:/Users/Marc/Desktop/model_data/train_model_data", 
                            series_dict = 'train_series_length.pickle', 
                            ind = [i for i in range(1, 195 + 1)])


trainloader = torch.utils.data.DataLoader(trainload,
                                          batch_size=1,
                                          shuffle=False,
                                          num_workers=0)

data_ex = []

nr_of_files_loaded = 0

batch_size = 1

start = time.time()

for file in trainloader:
    # the load_shuffle_5_min will now load the data with extreme correction
    loader2 = load_shuffle_5_min(file, device, ex = True)
    loader2 = torch.utils.data.DataLoader(loader2,
                                          batch_size=batch_size,
                                          shuffle=True,
                                          num_workers=0)
    nr_of_files_loaded += 1
    j = 0
    for i in loader2:
        j += 1
    
    data_ex.append(j)

print("time:", time.time()-start)
    
print("Number of files loaded in total:", nr_of_files_loaded)



# -

# Find how many intervals each patient consitute with:

# +
patient_ex = []

for value in patient_samples.values():
    vals = 0
    for i in value:
        vals += data_ex[int(i)-1]
    
    patient_ex.append(vals)
# -

# The following shows that extreme equality has been achived:

sorted_patient_ex = sorted(patient_ex, reverse=True)
print("Total number of patients:", len(patient_ex))
print("25% of patients amounts to:", int(0.25*len(patient_ex)))
print("The top 25% of patients constitute with:",
      sum(sorted_patient_ex[:int(0.25*len(patient_ex))])/sum(patient_ex),
      "of the data")

# Now the Lorenz plot can be created:

# +
m = lorenz_curve(patient_ex, "intervals with extreme correction")

plt.savefig(f"C:/Users/Marc/Desktop/BP photos/Lorenz_low.jpg")
plt.show()
plt.close()
# -

# # Section 3: How the dataloading is equalized:

# To make sure that data is distributed in the way that is intented, a pickle file with meta data of each recording is used. In the following code, this file will be looked at.
# Files of this type is created by the script: "2 series_dict.py" found in the preprocess folder on git.

# +
import pickle

# load the file:
with open('C:/Users/Marc/Desktop/model_data/train_model_data/train_series_length.pickle', 'rb') as handle:
    s_dict = pickle.load(handle)
# -

# Print the content for the first recording:

print(s_dict[1])

# The first number is how many samples the patient delivered (in all of the recorded eeg-sessions).  
# The second number is how many different times the patient has been recorded.  
# The third number is how many intervals should be sampled from the recording in total. If a patient has a short recording, this will not be over sampled.  
# The fourth value return the size of the given recording: (number of usefull channels, the length of the channels).

# +
new_p = list(i for i in np.array(list(s_dict.values()))[...,2])

patient_after = []

for value in patient_samples.values():
    vals = 0
    for i in value:
        vals += new_p[int(i)-1]
    
    patient_after.append(vals)
# -

# The following is the histogram and lorenz curve achived earlier

# +
plt.title("Histogram over intervals per experiment after correction")
plt.hist(patient_after, bins = 30)
plt.show()

print(np.mean(patient_after))
print(np.median(patient_after))

# +
m = lorenz_curve(patient_after, "intervals after correction")

#plt.savefig(f"C:/Users/Marc/Desktop/BP photos/Lorenz_after.jpg")
plt.show()
#plt.close()

StarcoderdataJupyterScriptsDedupFiltered