code: def trainATPEModels():
      with open("final/allResults_normalized.csv", "rt") as file:
          allResults = [dict(result) for result in csv.DictReader(file)]
  
      # Now put together the dataset for each predictor feature, so its more convenient to build models on them
      shuffled = list(allResults)
      random.shuffle(shuffled)
      cutoff = int(len(shuffled) * 0.2)
      testing = shuffled[:cutoff]
      training = shuffled[cutoff:]
  
      def createDataset(key, dataset, atpeParamFeatures):
          vectors = []
          targets = []
          allTargets = set()
          for result in dataset:
              if result[key] not in allTargets:
                  allTargets.add(result[key])
  
          allTargets = sorted(list(allTargets))
  
          names = copy.copy(featureKeys)
          for atpeParamFeature in atpeParamFeatures:
              if atpeParamFeature in atpeParameterValues:
                  for value in atpeParameterValues[atpeParamFeature]:
                      names.append(atpeParamFeature + "_" + value)
              else:
                  names.append(atpeParamFeature)
  
          for result in dataset:
              if result[key]:
                  vector = []
                  for feature in featureKeys:
                      vector.append(float(result[feature]))
                  for atpeParamFeature in atpeParamFeatures:
                      if (
                          atpeParamFeature in result
                          and result[atpeParamFeature] is not None
                          and result[atpeParamFeature] != ""
                      ):
                          if atpeParamFeature in atpeParameterValues:
                              for value in atpeParameterValues[
                                  atpeParamFeature
                              ]:
                                  vector.append(
                                      1.0
                                      if result[atpeParamFeature]
                                      == value
                                      else 0
                                  )
                          else:
                              vector.append(
                                  float(result[atpeParamFeature])
                              )
                      else:
                          vector.append(
                              -3
                          )  # We use -3 because none of our atpe parameters ever take this value
                  vectors.append(vector)
  
                  if key in classPredictorKeys:
                      targets.append(allTargets.index(result[key]))
                  else:
                      targets.append(float(result[key]))
          return lightgbm.Dataset(
              numpy.array(vectors),
              label=numpy.array(targets),
              feature_name=names,
          )
  
      allModels = []
      for keyIndex, key in enumerate(predictorKeyCascadeOrdering):
          atpeParamFeatures = predictorKeyCascadeOrdering[:keyIndex]
          trainingData = createDataset(
              key, training, atpeParamFeatures=atpeParamFeatures
          )
          testingData = createDataset(
              key, testing, atpeParamFeatures=atpeParamFeatures
          )
  
          allFeatureNames = trainingData.feature_name
  
          params = {
              "num_iterations": 100,
              "is_provide_training_metric": True,
              "early_stopping_round": 5,
              "feature_fraction": 0.85,
              "learning_rate": 0.05,
          }
  
          if key in customLightGBMParams:
              for param in customLightGBMParams[key]:
                  params[param] = customLightGBMParams[key][param]
  
          if key in classPredictorKeys:
              params["num_class"] = numPredictorClasses[key]
              params["objective"] = "multiclass"
              params["metric"] = "multi_error"
          else:
              params["objective"] = "regression_l2"
              params["metric"] = "l2"
  
          model = lightgbm.train(
              params,
              trainingData,
              valid_sets=[testingData],
              verbose_eval=False,
          )
  
          model.save_model("model-" + key + ".txt")
  
          if key not in classPredictorKeys:
              # Now we determine the "adjustment factor". Because these models are trained on an extremely noisy data set,
              # We have to eliminate the central tendency that results from training on it, so that the outputs of our model
              # Take up the full range of possible ATPE parameter values
              print(key)
              origStddev = numpy.std(
                  [
                      float(result[key])
                      for result in training
                      if result[key] is not None and result[key] != ""
                  ]
              )
              origMean = numpy.mean(
                  [
                      float(result[key])
                      for result in training
                      if result[key] is not None and result[key] != ""
                  ]
              )
  
              vectors = []
              for result in training:
                  vector = []
                  for feature in featureKeys:
                      vector.append(result[feature])
                  for atpeParamFeature in atpeParamFeatures:
                      if (
                          atpeParamFeature in result
                          and result[atpeParamFeature] is not None
                          and result[atpeParamFeature] != ""
                      ):
                          if atpeParamFeature in atpeParameterValues:
                              for value in atpeParameterValues[
                                  atpeParamFeature
                              ]:
                                  vector.append(
                                      1.0
                                      if result[atpeParamFeature]
                                      == value
                                      else 0
                                  )
                          else:
                              vector.append(
                                  float(result[atpeParamFeature])
                              )
                      else:
                          vector.append(
                              -3
                          )  # We use -3 because none of our atpe parameters ever take this value, so it acts as our signal that this parameter is unfilled
                  vectors.append(vector)
  
              trainingPredicted = model.predict(numpy.array(vectors))
  
              predStddev = numpy.std(trainingPredicted)
              predMean = numpy.mean(trainingPredicted)
  
              origStddev = (
                  origStddev
                  * atpeParameterPredictionStandardDeviationRatio[key]
              )
  
              with open(
                  "model-" + key + "-configuration.json", "wt"
              ) as file:
                  json.dump(
                      {
                          "origStddev": origStddev,
                          "origMean": origMean,
                          "predStddev": predStddev,
                          "predMean": predMean,
                      },
                      file,
                  )
  
              def renormalize(value):
                  return (
                      ((value - predMean) / predStddev) * origStddev
                  ) + origMean
  
              totalL1Error = 0
              totalL1NormalizedError = 0
              totalCount = 0
              with open("predictions-" + key + ".csv", "wt") as file:
                  writer = csv.DictWriter(
                      file,
                      fieldnames=[
                          key,
                          key + "_predicted",
                          key + "_predicted_normalized",
                          key + "_l1_error",
                          key + "_l1_normalized_error",
                      ],
                  )
                  writer.writeheader()
                  for result in testing:
                      if result[key] is not None and result[key] != "":
                          vector = []
                          for feature in featureKeys:
                              vector.append(float(result[feature]))
                          for atpeParamFeature in atpeParamFeatures:
                              if (
                                  atpeParamFeature in result
                                  and result[atpeParamFeature]
                                  is not None
                                  and result[atpeParamFeature] != ""
                              ):
                                  if (
                                      atpeParamFeature
                                      in atpeParameterValues
                                  ):
                                      for value in atpeParameterValues[
                                          atpeParamFeature
                                      ]:
                                          vector.append(
                                              1.0
                                              if result[
                                                  atpeParamFeature
                                              ]
                                              == value
                                              else 0
                                          )
                                  else:
                                      vector.append(
                                          float(
                                              result[atpeParamFeature]
                                          )
                                      )
                              else:
                                  vector.append(
                                      -3
                                  )  # We use -3 because none of our atpe parameters ever take this value
                          value = roundPrecision(
                              float(model.predict([vector])[0])
                          )
                          predicted = roundPrecision(
                              float(renormalize(value))
                          )
                          l1_error = roundPrecision(
                              float(abs(value - float(result[key])))
                          )
                          l1_normalized_error = roundPrecision(
                              float(abs(predicted - float(result[key])))
                          )
                          totalL1Error += l1_error
                          totalL1NormalizedError += l1_normalized_error
                          totalCount += 1
                          writer.writerow(
                              {
                                  key: result[key],
                                  key + "_predicted": value,
                                  key
                                  + "_predicted_normalized": predicted,
                                  key + "_l1_error": l1_error,
                                  key
                                  + "_l1_normalized_error": l1_normalized_error,
                              }
                          )
              print("Average L1 Error:", totalL1Error / totalCount)
              print(
                  "Average Normalized L1 Error:",
                  totalL1NormalizedError / totalCount,
              )
          else:
              # Now we have to determine the adjustment factor for each of the class decisions. Because the dataset is very noisy,
              # our output probabilities tend to be very close to 0.25. Lets renormalize them so we can use them for stochastic
              # selection.
              values = atpeParameterValues[key]
              origMeans = {}
              origStddevs = {}
              for value in values:
                  origMean = numpy.mean(
                      [
                          (1.0 if result[key] == value else 0)
                          for result in training
                          if result[key] is not None
                          and result[key] != ""
                      ]
                  )
                  origStddev = numpy.std(
                      [
                          (1.0 if result[key] == value else 0)
                          for result in training
                          if result[key] is not None
                          and result[key] != ""
                      ]
                  )
                  origMeans[value] = origMean
                  origStddevs[value] = (
                      origStddev
                      * atpeParameterPredictionStandardDeviationRatio[
                          key
                      ]
                  )
  
              vectors = []
              for result in training:
                  vector = []
                  for feature in featureKeys:
                      vector.append(result[feature])
                  for atpeParamFeature in atpeParamFeatures:
                      if (
                          atpeParamFeature in result
                          and result[atpeParamFeature] is not None
                          and result[atpeParamFeature] != ""
                      ):
                          if atpeParamFeature in atpeParameterValues:
                              for value in atpeParameterValues[
                                  atpeParamFeature
                              ]:
                                  vector.append(
                                      1.0
                                      if result[atpeParamFeature]
                                      == value
                                      else 0
                                  )
                          else:
                              vector.append(
                                  float(result[atpeParamFeature])
                              )
                      else:
                          vector.append(
                              -3
                          )  # We use -3 because none of our atpe parameters ever take this value, so it acts as our signal that this parameter is unfilled
                  vectors.append(vector)
  
              trainingPredicted = model.predict(numpy.array(vectors))
              trainingPredicted = numpy.array(trainingPredicted)
  
              predMeans = {}
              predStddevs = {}
              for valueIndex, value in enumerate(values):
                  predMean = numpy.mean(
                      trainingPredicted[:, valueIndex]
                  )
                  predStddev = numpy.std(
                      trainingPredicted[:, valueIndex]
                  )
                  predMeans[value] = predMean
                  predStddevs[value] = predStddev
  
              with open(
                  "model-" + key + "-configuration.json", "wt"
              ) as file:
                  json.dump(
                      {
                          "origStddevs": origStddevs,
                          "origMeans": origMeans,
                          "predStddevs": predStddevs,
                          "predMeans": predMeans,
                      },
                      file,
                  )
  
              def renormalize(predictedClasses):
                  predictedClasses = copy.copy(predictedClasses)
                  for valueIndex, value in enumerate(values):
                      predictedClasses[valueIndex] = (
                          (
                              (
                                  predictedClasses[valueIndex]
                                  - predMeans[value]
                              )
                              / predStddevs[value]
                          )
                          * origStddevs[value]
                      ) + origMeans[value]
                      predictedClasses[valueIndex] = max(
                          0, min(1.0, predictedClasses[valueIndex])
                      )
                  return predictedClasses
  
              totalCorrect = 0
              totalCount = 0
              with open("predictions-" + key + ".csv", "wt") as file:
                  writer = csv.DictWriter(
                      file,
                      fieldnames=[
                          key,
                          key + "_predicted",
                          key + "_correct",
                      ],
                  )
                  writer.writeheader()
                  for resultIndex, result in enumerate(testing):
                      vector = []
                      for feature in featureKeys:
                          vector.append(float(result[feature]))
                      for atpeParamFeature in atpeParamFeatures:
                          if (
                              atpeParamFeature in result
                              and result[atpeParamFeature] is not None
                              and result[atpeParamFeature] != ""
                          ):
                              if (
                                  atpeParamFeature
                                  in atpeParameterValues
                              ):
                                  for value in atpeParameterValues[
                                      atpeParamFeature
                                  ]:
                                      vector.append(
                                          1.0
                                          if result[atpeParamFeature]
                                          == value
                                          else 0
                                      )
                              else:
                                  vector.append(
                                      float(result[atpeParamFeature])
                                  )
                          else:
                              vector.append(
                                  -3
                              )  # We use -3 because none of our atpe parameters ever take this value, so it acts as our signal that this parameter is unfilled
  
                      predictedClasses = model.predict([vector])[0]
                      predicted = atpeParameterValues[key][
                          int(numpy.argmax(predictedClasses))
                      ]
                      correct = predicted == result[key]
  
                      totalCount += 1
                      if correct:
                          totalCorrect += 1
  
                      writer.writerow(
                          {
                              key: result[key],
                              key + "_predicted": predicted,
                              key + "_correct": correct,
                          }
                      )
              print(
                  "Accuracy:", str(totalCorrect * 100 / totalCount), "%"
              )
  
          importances = zip(allFeatureNames, model.feature_importance())
          importances = sorted(importances, key=lambda r: -r[1])
          print(key)
          for importance in importances:
              print("    ", importance[0], importance[1])
  
file_path: hypermax/research/atpe_research_2/process_results.py
imports: import os.path
  import json
  import traceback
  import csv
  import copy
  import sklearn.preprocessing
  import sklearn.neighbors
  import random
  import pickle
  import lightgbm
  import numpy
  from pprint import pprint
  from simulation import AlgorithmSimulation
label_desc: Use image node_id 7 for calling a global function with example usage: trainATPEModels() and returns: lightgbm, unknown, predictedClasses