code: def getDeduplicatedResults():
      logResults = extractResultsFromLogs()
      csvResults = extractResultsFromCSVs()
  
      logResultsByRun = {}
      csvResultsByRun = {}
      for result in logResults:
          # Make sure its a valid result. A very small number of results are missing all the requisite features (for an unknown reason, perhaps because they were killed part way into output
          valid = True
          for key in featureKeys:
              if key not in result:
                  valid = False
          if not valid:
              continue
          if result["run"] in logResultsByRun:
              logResultsByRun[result["run"]].append(result)
          else:
              logResultsByRun[result["run"]] = [result]
      for result in csvResults:
          if result["run"] in csvResultsByRun:
              csvResultsByRun[result["run"]].append(result)
          else:
              csvResultsByRun[result["run"]] = [result]
  
      duplicates = []
      additionals = []
      csvResultsByRunCloned = copy.deepcopy(csvResultsByRun)
      for run in logResultsByRun.keys():
          runDuplicates = []
          runAdditionals = []
  
          model = None
          if run in csvResultsByRun:
              # Build a nearest neighbor model to help us find the duplicates
              model = sklearn.neighbors.NearestNeighbors(n_neighbors=10)
              vectors = []
              for result in csvResultsByRunCloned[run]:
                  vectors.append([result[key] for key in featureKeys])
              model.fit(vectors)
  
          def testCSVResult(result2):
              same = True
              for key in result.keys():
                  if (
                      result[key] is not None
                      and result2[key] is not None
                  ):
                      try:
                          same = not (
                              abs(
                                  float(result[key])
                                  - float(result2[key])
                              )
                              > 0.01
                          )
                      except ValueError:
                          same = result[key] == result2[key]
  
                      if not same:
                          break
              return same
  
          for result in logResultsByRun[run]:
              found = False
              if run in csvResultsByRunCloned:
                  vector = [result[key] for key in featureKeys]
                  nearest = model.kneighbors(
                      [vector], n_neighbors=10
                  )  # The k nearest neighbors is good enough to always find the duplicates.
                  for neighbor in nearest[1][0]:
                      found = testCSVResult(
                          csvResultsByRun[run][neighbor]
                      )
                      if found:
                          break
              if found:
                  runDuplicates.append(result)
              if not found:
                  runAdditionals.append(result)
  
          print(run, "dupes", len(runDuplicates))
          print(run, "adds", len(runAdditionals))
          duplicates = duplicates + runDuplicates
          additionals = additionals + runAdditionals
  
      allResults = csvResults + additionals
      return allResults
  
file_path: hypermax/research/atpe_research_2/process_results.py
imports: import os.path
  import json
  import traceback
  import csv
  import copy
  import sklearn.preprocessing
  import sklearn.neighbors
  import random
  import pickle
  import lightgbm
  import numpy
  from pprint import pprint
  from simulation import AlgorithmSimulation
label_desc: Use image node_id 5 for calling a global function with example usage: getDeduplicatedResults() and returns: allResults, same