reduce timer back to 5 seconds

VerifiableRewardsForScalableLogicalReasoning.py

@@ -135,7 +135,8 @@ def _evaluate_with_prolog(prediction, validation_program, eval_config, timeout=5
     # extract predicate from rule_to_evaluate
     rule_to_evaluate = extract_ilp_from_text_v2(prediction, positive_pred, allow_multiple_rules)
     if positive_pred not in rule_to_evaluate:
-        logger.warning(f"Rule does not contain positive predicate '{positive_pred}'")
+        p = prediction.replace('\n', ' ')
+        logger.warning(f"Rule does not contain predicate '{positive_pred}': {p}")
         return {
             "is_correct": False,
             "partial_score": 0.0,
@@ -212,7 +213,8 @@ check_all :- forall((pos({vars});neg({vars})), check({vars})).
         }
 
     except subprocess.TimeoutExpired:
-        logger.warning(f"Evaluation timed out after {timeout} seconds for rule: '{rule_to_evaluate.replace('\n', ' ')}'")
+        r = rule_to_evaluate.replace('\n', ' ')
+        logger.warning(f"Evaluation timed out after {timeout} seconds for rule: '{r}'")
         return {"is_correct": False, "partial_score": 0.0, "syntax_valid": False,
                 "error": f"Evaluation timed out after {timeout} seconds"}
     except Exception as e:
@@ -323,7 +325,7 @@ class VerifiableRewardsForScalableLogicalReasoning(evaluate.Metric):
             raise ValueError(
                 f"Number of predictions ({len(predictions)}) and references {len(references)}) don't match")
 
-        TIMEOUT = 15 if len(predictions) > 500 else 5
+        TIMEOUT = 5 if len(predictions) > 500 else 5
         # Prepare evaluation inputs
         eval_inputs = []
         for i, (prediction, reference) in enumerate(zip(predictions, references)):