message: Multiple Runs
diff: @@ -24,33 +24,48 @@ def run_tpch_query(duckdb_conn,query_number):
  query_result.append(total_time)
  return statistics.median(query_result)
  -def run_tpch():
  +def run_tpch(repetitions):
  import duckdb
  duckdb_conn = duckdb.connect()
  duckdb_conn.execute("CALL dbgen(sf=0.01);")
  result_list = []
  for i in range (1,23):
  - result_list.append(run_tpch_query(duckdb_conn,i))
  + query_result_list = []
  + for j in range(repetitions):
  + query_result_list.append(run_tpch_query(duckdb_conn,i))
  + result_list.append(query_result_list)
  return result_list
  -def run_benchmark(install_function):
  +def run_benchmark(install_function,repetitions):
  install_function()
  - result_list = run_tpch()
  + result_list = run_tpch(repetitions)
  uninstall_duck()
  return result_list
  +# We want to run the regression tests 3x if a query fails (i.e., is slower than the one in the master these 3 times then it fails)
  def regression_test(threshold):
  - master_time = run_benchmark(install_duck_master)
  - current_time = run_benchmark(install_duck_current)
  + repetitions = 3
  + master_time = run_benchmark(install_duck_master,repetitions)
  + current_time = run_benchmark(install_duck_current,repetitions)
  # Print Query Results for Logging
  is_it_faster = True
  for i in range(len(master_time)):
  - if current_time[i] > master_time[i] * (1+threshold):
  + query_ok = False
  + query_faster = False
  + for j in range (repetitions):
  + if current_time[i][j] <= master_time[i][j] * (1+threshold):
  + query_ok = True
  +
  + if current_time[i][j] <= master_time[i][j] * (1-threshold):
  + query_faster = True
  +
  + if not query_ok:
  is_it_faster = False
  - print ("Query "+ str(i+1) + " (SLOW) " + "Master: " + str(master_time[i]) + " Current: " + str(master_time[i]))
  - else:
  - print ("Query "+ str(i+1) + " (FAST) " + "Master: " + str(master_time[i]) + " Current: " + str(master_time[i]))
  + print ("Query "+ str(i+1) + " (SLOWER) ")
  + if query_faster:
  + print ("Query "+ str(i+1) + " (FASTER) ")
  +
  assert (is_it_faster)
  regression_test(0.1)
  \ No newline at end of file
  