update apps_metric to provide outputs

testing_util.py

@@ -54,7 +54,8 @@ def run_test(sample, test=None, debug=False):
     otherwise it'll just return an input and output pair.
     """
     # Disable functionalities that can make destructive changes to the test.
-    reliability_guard()
+    # TODO: disable for now as it interferes with GPT-4 generation through gateway
+    # reliability_guard()
     
     if debug:
         print(f"start = {datetime.now().time()}")
@@ -99,7 +100,7 @@ def run_test(sample, test=None, debug=False):
                 if debug:
                      print(f"type 0 compilation error = {e}")
                 results.append(-2)
-                return results
+                return results, {}
             signal.alarm(0)
 
         elif which_type == CODE_TYPE.standard_input:
@@ -156,6 +157,7 @@ def run_test(sample, test=None, debug=False):
             results.append(-2)
             return results
 
+        program_outputs = {}
         for index, inputs in enumerate(in_outs["inputs"]):
             # JSON forces dictionaries to have string keys; this undoes this (assuming a singleton list)
             try:
@@ -200,6 +202,15 @@ def run_test(sample, test=None, debug=False):
 
                     # reset the alarm
                     signal.alarm(0)
+
+                    program_outputs[index] = {
+                        "pass": tmp_result,
+                        "pass_pct": int(tmp_result),
+                        "pass_res": [int(tmp_result)],
+                        "output": output,
+                        "input": inputs,
+                        "ground_truth": in_outs["outputs"][index]
+                    }
                 except Exception as e:
                     signal.alarm(0)
                     faulthandler.disable()
@@ -234,6 +245,10 @@ def run_test(sample, test=None, debug=False):
                         results.append(-1)
                     signal.alarm(0)
 
+                program_outputs[index] = {"output": output}
+                program_outputs[index]["ground_truth"] = in_outs['outputs'][index]
+                program_outputs[index]["input"] = in_outs['inputs'][index]
+
                 if not passed:
                     if debug:
                         nl = "\n"
@@ -246,7 +261,12 @@ def run_test(sample, test=None, debug=False):
                 if passed and debug:
                     print(f"==> output = {output}, test outputs = {in_outs['outputs'][index]}")
 
-                if custom_compare_(output, in_outs['outputs'][index]):
+                all_pass, pass_pct, pass_res = custom_compare_(output, in_outs['outputs'][index])
+                program_outputs[index]["pass"] = all_pass
+                program_outputs[index]["pass_pct"] = pass_pct
+                program_outputs[index]["pass_res"] = pass_res
+                
+                if all_pass:
                     tmp_result = True
                     results.append(tmp_result)
                     continue
@@ -391,26 +411,42 @@ def run_test(sample, test=None, debug=False):
                     if not isinstance(inputs, list):
                         print(f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}")
                     else:
-                        print(f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}") 
-
+                        print(f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}")
 
-    return results
+    return results, program_outputs
 
 
 def custom_compare_(output, ground_truth):
+
+    # TODO: split ground_truth and compare one by one
+    ground_truth_list = ground_truth.strip().split("\n")
+    correct = 0
+    res = []
     
     if isinstance(output, list):
-        output_1 = "\n".join(output)
-        if stripped_string_compare(output_1, ground_truth):
-            return True
+        for out, g_t in zip(output, ground_truth_list):
+            if out.strip() == g_t.strip():
+                correct += 1
+                res.append(1)
+            else:
+                res.append(0)
 
-    if isinstance(output, list):
-        output_2 = [o.lstrip().rstrip() for o in output]
-        output_2 = "\n".join(output_2)
-        if stripped_string_compare(output_2, ground_truth):
-            return True
+        return correct == len(ground_truth_list), correct / len(ground_truth_list), res
+
+    return False, 0.0, []
+            
+    # if isinstance(output, list):
+    #     output_1 = "\n".join(output)
+    #     if stripped_string_compare(output_1, ground_truth):
+    #         return True
+
+    # if isinstance(output, list):
+    #     output_2 = [o.lstrip().rstrip() for o in output]
+    #     output_2 = "\n".join(output_2)
+    #     if stripped_string_compare(output_2, ground_truth):
+    #         return True
 
-    return False
+    # return False
 
 def stripped_string_compare(s1, s2):
     s1 = s1.lstrip().rstrip()
@@ -427,6 +463,7 @@ def call_method(method, inputs):
     # sys.setrecursionlimit(10000)
 
     # @patch('builtins.input', side_effect=inputs.split("\n"))
+    @patch('builtins.input', lambda *args: next(inputs_line_iterator))
     @patch('builtins.open', mock_open(read_data=inputs))
     @patch('sys.stdin', StringIO(inputs))
     @patch('sys.stdin.readline', lambda *args: next(inputs_line_iterator))
@@ -522,4 +559,4 @@ def reliability_guard(maximum_memory_bytes=None):
     sys.modules["joblib"] = None
     sys.modules["resource"] = None
     sys.modules["psutil"] = None
-    sys.modules["tkinter"] = None
+    sys.modules["tkinter"] = None

utils.py

@@ -48,7 +48,7 @@ def evaluate_generations(generations: list, indices: list = [], level: str = "al
      """
 
     # generations are code generations in the same order of the dataset
-    apps_eval = load_dataset(DATASET, split="test", difficulties=[level])
+    apps_eval = load_dataset(DATASET, level, split="train")
 
     if indices is None:
         indices = range(len(generations))