import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import os
import matplotlib.pyplot as plt
import CurveFit
import shutil

#find all DIRECTORIES containing non-hidden files ending in FILENAME
def getDataDirectories(DIRECTORY, FILENAME="valLoss.txt"):
    directories=[]
    for directory in os.scandir(DIRECTORY):
        for item in os.scandir(directory):
            if item.name.endswith(FILENAME) and not item.name.startswith("."):
                directories.append(directory.path)
    return directories

#get all non-hidden data files in DIRECTORY with extension EXT
def getDataFiles(DIRECTORY, EXT='txt'):
    datafiles=[]
    for item in os.scandir(DIRECTORY):
        if item.name.endswith("."+EXT) and not item.name.startswith("."):
            datafiles.append(item.path)
    return datafiles

#checking if loss ever doesn't decrease for numEpochs epochs in a row.
def stopsDecreasing(loss, epoch, numEpochs):
    minLoss=np.inf
    epochMin=0
    for i in range(0,loss.size):
        if loss[i] < minLoss:
            minLoss=loss[i]
            epochMin=epoch[i]
        elif (epoch[i]-epochMin) >= numEpochs:
            return i, minLoss
        
    return i, minLoss

#dirpath is where the accuracy and loss files are stored. want to move the files into the same format expected by grabNNData.
def createFolders(SEARCHDIR, SAVEDIR):
    for item in os.scandir(SEARCHDIR):
        name=str(item.name)
        files=name.split('-')
        SAVEFULLDIR=SAVEDIR+str(files[0])
        if not os.path.exists(SAVEFULLDIR):
            try:
                os.makedirs(SAVEFULLDIR)
            except FileExistsError:
                #directory already exists--must have been created between the if statement & our attempt at making directory
                pass
        shutil.move(item.path, SAVEFULLDIR+"/"+str(files[1]))
    

#a function to read in information (e.g. accuracy, loss) stored at FILENAME
def grabNNData(FILENAME, header='infer', sep=' '):
    data = pd.read_csv(FILENAME, sep, header=header)

    if ('epochs' in data.columns) and ('trainLoss' in data.columns) and ('valLoss' in data.columns) and ('valAcc' in data.columns) and ('batch_size' in data.columns) and ('learning_rate' in data.columns):

        sortedData=data.sort_values(by="epochs", axis=0, ascending=True)

        epoch=np.array(sortedData['epochs'])
        trainLoss=np.array(sortedData['trainLoss'])
        valLoss=np.array(sortedData['valLoss'])
        valAcc=np.array(sortedData['valAcc'])

        batch_size=np.array(sortedData['batch_size'])
        learning_rate=np.array(sortedData['learning_rate'])

        convKers=np.array(sortedData['convKernels'])
        
        return(epoch, trainLoss, valLoss, valAcc, batch_size, learning_rate, convKers)

    elif ('epochs' in data.columns) and ('trainLoss' in data.columns) and ('valLoss' in data.columns) and ('valAcc' in data.columns):

        sortedData=data.sort_values(by="epochs", axis=0, ascending=True)
            
        epoch=np.array(sortedData['epochs'])
        trainLoss=np.array(sortedData['trainLoss'])
        valLoss=np.array(sortedData['valLoss'])
        valAcc=np.array(sortedData['valAcc'])

    else:
        print("Missing a column in NN datafile")
        raise Exception('NN datafile is missing one of the expected columns: epochs trainLoss valLoss valAcc [optional extra columns: batch_size, learning_rate]')
        

#slice data could be used to test values of E other than E=0.5, which we use by default
def sliceData(xsize, x, y, z=None, w=None):
    #we can slice the data to sample less often, but not more often. We verify that we're not being asked for a granularity that is smaller than the frequency of datapoints in the vectors.
    if x[0] > xsize:
        return x,y,z,w
    else:
        result=(1.0/x[0])*xsize
        #result is how often we should take datapoints if we wish to consider values every xsize

        x=x[int(result-1)::int(result)] 
        y=y[int(result-1)::int(result)]

        if z is not None:
            z=z[int(result-1)::int(result)]
            if w is None:
                return x,y,z
        else:
            return x,y

        #if we get to this point in function, it means z and w are both not None.
        w=w[int(result-1)::int(result)]
        return x,y,z,w


[((12, 21, 12, 42), 'os.scandir', 'os.scandir', ({(12, 32, 12, 41): 'DIRECTORY'}, {}), '(DIRECTORY)', False, 'import os\n'), ((21, 16, 21, 37), 'os.scandir', 'os.scandir', ({(21, 27, 21, 36): 'DIRECTORY'}, {}), '(DIRECTORY)', False, 'import os\n'), ((41, 16, 41, 37), 'os.scandir', 'os.scandir', ({(41, 27, 41, 36): 'SEARCHDIR'}, {}), '(SEARCHDIR)', False, 'import os\n'), ((56, 11, 56, 52), 'pandas.read_csv', 'pd.read_csv', (), '', True, 'import pandas as pd\n'), ((13, 20, 13, 41), 'os.scandir', 'os.scandir', ({(13, 31, 13, 40): 'directory'}, {}), '(directory)', False, 'import os\n'), ((62, 14, 62, 44), 'numpy.array', 'np.array', ({(62, 23, 62, 43): "sortedData['epochs']"}, {}), "(sortedData['epochs'])", True, 'import numpy as np\n'), ((63, 18, 63, 51), 'numpy.array', 'np.array', ({(63, 27, 63, 50): "sortedData['trainLoss']"}, {}), "(sortedData['trainLoss'])", True, 'import numpy as np\n'), ((64, 16, 64, 47), 'numpy.array', 'np.array', ({(64, 25, 64, 46): "sortedData['valLoss']"}, {}), "(sortedData['valLoss'])", True, 'import numpy as np\n'), ((65, 15, 65, 45), 'numpy.array', 'np.array', ({(65, 24, 65, 44): "sortedData['valAcc']"}, {}), "(sortedData['valAcc'])", True, 'import numpy as np\n'), ((67, 19, 67, 53), 'numpy.array', 'np.array', ({(67, 28, 67, 52): "sortedData['batch_size']"}, {}), "(sortedData['batch_size'])", True, 'import numpy as np\n'), ((68, 22, 68, 59), 'numpy.array', 'np.array', ({(68, 31, 68, 58): "sortedData['learning_rate']"}, {}), "(sortedData['learning_rate'])", True, 'import numpy as np\n'), ((70, 17, 70, 52), 'numpy.array', 'np.array', ({(70, 26, 70, 51): "sortedData['convKernels']"}, {}), "(sortedData['convKernels'])", True, 'import numpy as np\n'), ((45, 15, 45, 42), 'os.path.exists', 'os.path.exists', ({(45, 30, 45, 41): 'SAVEFULLDIR'}, {}), '(SAVEFULLDIR)', False, 'import os\n'), ((78, 14, 78, 44), 'numpy.array', 'np.array', ({(78, 23, 78, 43): "sortedData['epochs']"}, {}), "(sortedData['epochs'])", True, 'import numpy as np\n'), ((79, 18, 79, 51), 'numpy.array', 'np.array', ({(79, 27, 79, 50): "sortedData['trainLoss']"}, {}), "(sortedData['trainLoss'])", True, 'import numpy as np\n'), ((80, 16, 80, 47), 'numpy.array', 'np.array', ({(80, 25, 80, 46): "sortedData['valLoss']"}, {}), "(sortedData['valLoss'])", True, 'import numpy as np\n'), ((81, 15, 81, 45), 'numpy.array', 'np.array', ({(81, 24, 81, 44): "sortedData['valAcc']"}, {}), "(sortedData['valAcc'])", True, 'import numpy as np\n'), ((47, 16, 47, 40), 'os.makedirs', 'os.makedirs', ({(47, 28, 47, 39): 'SAVEFULLDIR'}, {}), '(SAVEFULLDIR)', False, 'import os\n')]