code: def mergeResults():
      allResults = getDeduplicatedResults()
  
      # with open('final/allResults.csv', 'rt') as file:
      #     results = [dict(result) for result in csv.DictReader(file)]
      #     for result in results:
      #         del result['blank1']
      #         del result['blank2']
      #     allResults = results
  
      # Convert everything to floats where possible
      for result in allResults:
          for key in result.keys():
              if result[key] is not None and result[key] != "":
                  try:
                      result[key] = float(result[key])
                  except ValueError:
                      pass
  
      # Filter out all results with a loss of 1.0. These results are meaningless for our dataset, since they are caused our random-failure noise, and not by actually searching the dataset
      allResults = [
          result for result in allResults if float(result["loss"]) < 1.0
      ]
  
      for result in allResults:
          if "algorithm" not in result:
              result["algorithm"] = ""
          if "time" not in result:
              result["time"] = ""
  
      if not os.path.exists("final"):
          os.mkdir("final")
      if not os.path.exists("final/algos"):
          os.mkdir("final/algos")
  
      savedAlgorithms = {}
  
      for result in allResults:
          result["blank1"] = ""
          result["blank2"] = ""
  
          if (
              result["algorithm"]
              and (result["run"] + result["algorithm"])
              not in savedAlgorithms
          ):
              algoFileName = os.path.join(
                  result["run"], "hypermax", result["algorithm"]
              )
              if os.path.exists(algoFileName):
                  data = pickle.load(open(algoFileName, "rb"))["algo"]
  
                  scriptName = (
                      "algorithm-" + str(len(savedAlgorithms)) + ".py"
                  )
                  dataName = (
                      "algorithm-"
                      + str(len(savedAlgorithms))
                      + "-pickle.bin"
                  )
                  with open(
                      os.path.join("final", "algos", scriptName), "wt"
                  ) as file:
                      file.write(data.computeScript)
                  with open(
                      os.path.join("final", "algos", dataName), "wb"
                  ) as file:
                      pickle.dump(data, file)
                  savedAlgorithms[
                      result["run"] + result["algorithm"]
                  ] = scriptName
                  result["algorithm"] = scriptName
              else:
                  result["algorithm"] = ""
          elif result["algorithm"]:
              result["algorithm"] = savedAlgorithms[
                  result["run"] + result["algorithm"]
              ]
  
      prettyResults = []
      for result in allResults:
          prettyResult = {}
          for key in result.keys():
              prettyResult[key] = result[key]
              if prettyResult[key]:
                  try:
                      number = roundPrecision(float(prettyResult[key]))
                      prettyResult[key] = number
                  except ValueError:
                      pass
                  except TypeError:
                      pass
          prettyResults.append(prettyResult)
  
      with open("final/allResults.csv", "wt") as file:
          writer = csv.DictWriter(
              file,
              fieldnames=(
                  atpeParameterKeys
                  + ["blank1"]
                  + featureKeys
                  + ["blank2"]
                  + nonFeatureKeys
              ),
          )
          writer.writeheader()
          writer.writerows(prettyResults)
  
      # Compute the normalization region for each predictor
      scalers = {}
      for feature in featureKeys:
          vectors = []
          for result in allResults:
              if float(result[feature]) != 0:
                  vectors.append([result[feature]])
  
          vectors = numpy.array(vectors)
          # Use percentiles to focus our scaler on the most common values, making it more immune to the weird outliers in our dataset
          percentile20 = numpy.percentile(vectors[:, 0], q=20)
          percentile80 = numpy.percentile(vectors[:, 0], q=80)
          vectors = [
              vector
              for vector in vectors
              if vector[0] > percentile20 and vector[0] < percentile80
          ]
  
          scaler = sklearn.preprocessing.StandardScaler(vectors)
          scaler.fit(vectors)
  
          scalers[feature] = {
              "scales": scaler.scale_.tolist(),
              "means": scaler.mean_.tolist(),
              "variances": scaler.var_.tolist(),
          }
      with open("scaling_model.json", "wt") as file:
          json.dump(scalers, file)
  
      for keyIndex, key in enumerate(featureKeys):
          featureScalingModel = sklearn.preprocessing.StandardScaler()
          featureScalingModel.scale_ = numpy.array(
              scalers[key]["scales"]
          )
          featureScalingModel.mean_ = numpy.array(scalers[key]["means"])
          featureScalingModel.var_ = numpy.array(
              scalers[key]["variances"]
          )
  
          vectors = []
          for resultIndex, result in enumerate(allResults):
              vectors.append([float(result[key])])
  
          vectors = featureScalingModel.transform(vectors)
          for resultIndex, result in enumerate(allResults):
              result[key] = roundPrecision(vectors[resultIndex][0])
  
      # Output the normalized results
      prettyResults = []
      for result in allResults:
          prettyResult = {}
          for key in result.keys():
              prettyResult[key] = result[key]
              if prettyResult[key]:
                  try:
                      number = roundPrecision(float(prettyResult[key]))
                      prettyResult[key] = number
                  except ValueError:
                      pass
                  except TypeError:
                      pass
          prettyResults.append(prettyResult)
  
      with open("final/allResults_normalized.csv", "wt") as file:
          writer = csv.DictWriter(
              file,
              fieldnames=atpeParameterKeys
              + ["blank1"]
              + featureKeys
              + ["blank2"]
              + nonFeatureKeys,
          )
          writer.writeheader()
          writer.writerows(prettyResults)
  
      # Now put together the dataset for each predictor feature, so its more convenient to build models on them
      shuffled = list(allResults)
      random.shuffle(shuffled)
      cutoff = int(len(shuffled) * 0.2)
      testing = shuffled[:cutoff]
      training = shuffled[cutoff:]
  
      def writeDataset(key, filename, dataset):
          with open(filename, "wt") as file:
              keyResults = []
              for result in dataset:
                  if result[key]:
                      data = {}
                      data[key] = result[key]
                      for feature in featureKeys:
                          data[feature] = result[feature]
                      keyResults.append(data)
  
              writer = csv.DictWriter(
                  file, fieldnames=[key] + featureKeys
              )
              writer.writeheader()
              writer.writerows(keyResults)
  
      for key in atpeParameterKeys:
          writeDataset(key, "final/" + key + "_testing.csv", testing)
          writeDataset(key, "final/" + key + "_training.csv", training)
  
      return training, testing
  
file_path: hypermax/research/atpe_research_2/process_results.py
imports: import os.path
  import json
  import traceback
  import csv
  import copy
  import sklearn.preprocessing
  import sklearn.neighbors
  import random
  import pickle
  import lightgbm
  import numpy
  from pprint import pprint
  from simulation import AlgorithmSimulation
label_desc: Use image node_id 6 for calling a global function with example usage: mergeResults() and returns: training,  testing