I want to sort values in a csv file based on timestamp and print it to another file, but for files with many lines python runs out of memory (when the file is being read). 
Is there something I can do to make this more efficient or should I use something else then csv.DictReader?

```
import csv, sys
import datetime
from pathlib import Path

localPath = "C:/MyPath"

    # data variables 
dataDir = localPath + "data/" dataExtension = ".dat" 

    pathlistData = Path(dataDir).glob('**/*'+ dataExtension)

    # Generated filename as date, Format: YYYY-DDDTHH
    generatedDataDir = localPath + "result/"
    #generatedExtension = ".dat"
    errorlog = 'errorlog.csv'

    fieldnames = ['TimeStamp', 'A', 'B', 'C', 'C', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L','M', 'N', 'O', 'P', 'Q', 'R'] 

    for dataPath in pathlistData:
        #stores our data in a dictionary
        dataDictionary = {}

        dataFileName = str(dataPath).replace('\\', '/')
        newFilePathString = dataFileName.replace(dataDir,generatedDataDir)

        with open(dataPath, 'r') as readFile:
            print(str("Reading data from " + dataFileName))
            keysAsDate = []#[datetime.datetime.strptime(ts, "%Y-%m-%d") for ts in timestamps]
            reader = csv.DictReader(readFile, fieldnames=fieldnames) 

            for row in reader:

                try:
                    timestamp = row['TimeStamp']
                    #create a key based on the timestamp
                    timestampKey = datetime.datetime.strptime(timestamp[0:16], "%Y-%jT%H:%M:%S")
                    #save this key as a date, used later for sorting
                    keysAsDate.append(timestampKey)
                    #save the row data in a dictionary
                    dataDictionary[timestampKey] = row

                except csv.Error as e:
                    sys.exit('file %s, line %d: %s' % (errorlog, reader.line_num, e))

            #sort the keys
            keysAsDate.sort()
        readFile.close()

        with open(newFilePathString, 'w') as writeFile:
            writer = csv.DictWriter(writeFile, fieldnames=fieldnames, lineterminator='\n')
            print(str("Writing data to " + newFilePathString))
            #loop over the sorted keys
            for idx in range(0, len(keysAsDate)):

                #get the row from our data dictionary 
                writeRow = dataDictionary[keysAsDate[idx]]
                #print(dataDictionary[keysAsDate[key]])
                writer.writerow(writeRow)
                if idx%30000 == 0:
                    print("Writing to new file: " + str(int(idx/len(keysAsDate) * 100)) + "%")

        print(str("Finished writing to file: " + newFilePathString))

        writeFile.close()

```

**UPDATE:** I have used pandas and divided the large file into smaller chunks that I could sort individually.
This currently doesn't solve the issue for wildly misplaced values if I append the files after each other.

```
for dataPath in pathlistData:

dataFileName = str(dataPath).replace('\\', '/')
#newFilePathString = dataFileName.replace(dataDir,generatedDataDir)

print(str("Reading data from " + dataFileName))
#divide our large data frame into smaller data frame chunks
#so we can sort the content in memory
for df_chunk in pd.read_csv(dataFileName, header = None, chunksize = chunk_size, names = fieldnames):
    dataDictionary = {}
    dataDictionary.clear()

    for idx in range(0, chunk_size):
        #print(df_chunk[idx:idx+1])
        row = df_chunk[idx:idx+1]
        dataDictionary = df_chunk.sort_values(['TimeStamp'], ascending=True)
    firstTimeStampInChunk = dataDictionary[0:1]['TimeStamp']
    #print("first: " + firstTimeStampInChunk)
    lastTimeStampInChunk = dataDictionary[chunk_size-1:chunk_size]['TimeStamp']
    #print("last: " + lastTimeStampInChunk)

    timestampStr = str(firstTimeStampInChunk)[chunk_shift:timestamp_size+chunk_shift] + str(lastTimeStampInChunk)[chunk_shift:timestamp_size+chunk_shift]
    tempFilePathString = str(timestampStr + dataExtension).replace(':', '_').replace('\\', '/')
    dataDictionary.to_csv('temp/'+tempFilePathString, header = None, index=False)

# data variables
tempDataDir = localPath + "temp/"
tempPathlistData = Path(tempDataDir).glob('**/*'+ dataExtension)

tempPathList = list(tempPathlistData)

```

My algorithm theory (no code) to solve the issue with random values is: 

**Step 1** - divide into smaller chunks where "chunk\_size = max lines to handle in memory divided by two"

**Step 2** - Loop through files in order, merge two files at a time and sort them together, then split them up again so no file is larger than chunk\_size. 

**Step 3** - Loop through backwards, merging two files at a time and sorting them, then split up again so no file is larger than chunk\_size. 

**Step 4** - Now all wildly misplaced low values should have travelled to the lowest part, and all the wildly misplaced high values should have travelled to the highest part. Append the files in order!

Cons; The time complexity for this is not preferable at all, basically O(N^2) if I'm not mistaken