prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Implement function to calculate F1 metric
  As specified in proposal #5794 we need to implement a function to calculate the F1 metric.
  
  Ideally the function should be part of the private interface and called only through the `calculate_metrics` function (see #6063). `_calculate_f1()` could be a nice name.
  
  For more detailed information check out the original proposal.
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `haystack/evaluation/eval.py`
  Content:
  ```
  1 from typing import Any, Callable, Dict, List, Union
  2 
  3 import numpy as np
  4 
  5 from haystack import Pipeline
  6 from haystack.core.component import Component
  7 from haystack.evaluation.eval_utils import get_answers_from_output, preprocess_text
  8 from haystack.evaluation.metrics import Metric, MetricsResult
  9 
  10 
  11 class EvaluationResult:
  12     """
  13     EvaluationResult keeps track of all the information related to evaluation, namely the runnable (Pipeline or
  14     component), inputs, outputs, and expected outputs.
  15     The EvaluationResult keeps track of all the information stored by eval.
  16 
  17     :param runnable: The runnable (Pipeline or component) used for evaluation.
  18     :param inputs: List of inputs used for evaluation.
  19     :param outputs: List of outputs generated by the runnable.
  20     :param expected_outputs: List of expected outputs used for evaluation.
  21     """
  22 
  23     def __init__(
  24         self,
  25         runnable: Union[Pipeline, Component],
  26         inputs: List[Dict[str, Any]],
  27         outputs: List[Dict[str, Any]],
  28         expected_outputs: List[Dict[str, Any]],
  29     ) -> None:
  30         self.runnable = runnable
  31         self.inputs = inputs
  32         self.outputs = outputs
  33         self.expected_outputs = expected_outputs
  34 
  35         # Determine the type of the runnable
  36         if str(type(runnable).__name__) == "Pipeline":
  37             self.runnable_type = "pipeline"
  38         else:
  39             self.runnable_type = "component"
  40 
  41         # Mapping of metrics to their corresponding functions.
  42         # This should be kept in sync with the Metric enum
  43         self._supported_metrics: Dict[Metric, Callable[..., MetricsResult]] = {
  44             Metric.RECALL: self._calculate_recall,
  45             Metric.MRR: self._calculate_mrr,
  46             Metric.MAP: self._calculate_map,
  47             Metric.F1: self._calculate_f1,
  48             Metric.EM: self._calculate_em,
  49             Metric.SAS: self._calculate_sas,
  50         }
  51 
  52     def calculate_metrics(self, metric: Union[Metric, Callable[..., MetricsResult]], **kwargs) -> MetricsResult:
  53         """
  54         Calculate evaluation metrics based on the provided Metric or using the custom metric function.
  55 
  56         :param metric: The Metric indicating the type of metric to calculate or custom function to compute.
  57         :return: MetricsResult containing the calculated metric.
  58         """
  59 
  60         if isinstance(metric, Metric):
  61             return self._supported_metrics[metric](**kwargs)
  62 
  63         return metric(self, **kwargs)
  64 
  65     def _calculate_recall(self):
  66         return MetricsResult({"recall": None})
  67 
  68     def _calculate_map(self):
  69         return MetricsResult({"mean_average_precision": None})
  70 
  71     def _calculate_mrr(self):
  72         return MetricsResult({"mean_reciprocal_rank": None})
  73 
  74     def _calculate_f1(self):
  75         return MetricsResult({"f1": None})
  76 
  77     def _calculate_em(
  78         self, output_key: str, regexes_to_ignore=None, ignore_case=False, ignore_punctuation=False, ignore_numbers=False
  79     ) -> MetricsResult:
  80         """
  81         Calculates the Exact Match (EM) score between two lists of predictions and labels.
  82         Exact Match (EM) score measures the percentage of samples where the predicted text exactly matches the
  83           corresponding ground truth label.
  84 
  85         :param output_key: The key of the output to use for comparison.
  86         :param regexes_to_ignore (list, optional): A list of regular expressions. If provided, it removes substrings
  87             matching these regular expressions from both predictions and labels before comparison. Defaults to None.
  88         :param ignore_case (bool, optional): If True, performs case-insensitive comparison. Defaults to False.
  89         :param ignore_punctuation (bool, optional): If True, removes punctuation from both predictions and labels before
  90             comparison. Defaults to False.
  91         :param ignore_numbers (bool, optional): If True, removes numerical digits from both predictions and labels
  92             before comparison. Defaults to False.
  93         :return: A MetricsResult object containing the calculated Exact Match (EM) score.
  94         """
  95 
  96         predictions = get_answers_from_output(
  97             outputs=self.outputs, output_key=output_key, runnable_type=self.runnable_type
  98         )
  99         labels = get_answers_from_output(
  100             outputs=self.expected_outputs, output_key=output_key, runnable_type=self.runnable_type
  101         )
  102 
  103         if len(predictions) != len(labels):
  104             raise ValueError("The number of predictions and labels must be the same.")
  105         if len(predictions) == len(labels) == 0:
  106             # Return Exact Match as 0 for no inputs
  107             return MetricsResult({"exact_match": 0.0})
  108 
  109         predictions = preprocess_text(predictions, regexes_to_ignore, ignore_case, ignore_punctuation, ignore_numbers)
  110         labels = preprocess_text(labels, regexes_to_ignore, ignore_case, ignore_punctuation, ignore_numbers)
  111 
  112         score_list = np.array(predictions) == np.array(labels)
  113         exact_match_score = np.mean(score_list)
  114 
  115         return MetricsResult({"exact_match": exact_match_score})
  116 
  117     def _calculate_sas(self):
  118         return MetricsResult({"exact_match": None})
  119 
  120 
  121 def eval(
  122     runnable: Union[Pipeline, Component], inputs: List[Dict[str, Any]], expected_outputs: List[Dict[str, Any]]
  123 ) -> EvaluationResult:
  124     """
  125     Evaluates the provided Pipeline or component based on the given inputs and expected outputs.
  126 
  127     This function facilitates the evaluation of a given runnable (either a Pipeline or a component) using the provided
  128     inputs and corresponding expected outputs.
  129 
  130     :param runnable: The runnable (Pipeline or component) used for evaluation.
  131     :param inputs: List of inputs used for evaluation.
  132     :param expected_outputs: List of expected outputs used for evaluation.
  133 
  134     :return: An instance of EvaluationResult containing information about the evaluation, including the runnable,
  135     inputs, outputs, and expected outputs.
  136     """
  137 
  138     outputs = []
  139 
  140     # Check that expected outputs has the correct shape
  141     if len(inputs) != len(expected_outputs):
  142         raise ValueError(
  143             f"The number of inputs ({len(inputs)}) does not match the number of expected outputs "
  144             f"({len(expected_outputs)}). Please ensure that each input has a corresponding expected output."
  145         )
  146 
  147     for input_ in inputs:
  148         output = runnable.run(input_)
  149         outputs.append(output)
  150 
  151     return EvaluationResult(runnable, inputs, outputs, expected_outputs)
  152
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/haystack/evaluation/eval.py b/haystack/evaluation/eval.py
  --- a/haystack/evaluation/eval.py
  +++ b/haystack/evaluation/eval.py
  @@ -1,3 +1,4 @@
  +import collections
   from typing import Any, Callable, Dict, List, Union
   
   import numpy as np
  @@ -71,8 +72,68 @@
       def _calculate_mrr(self):
           return MetricsResult({"mean_reciprocal_rank": None})
   
  -    def _calculate_f1(self):
  -        return MetricsResult({"f1": None})
  +    def _compute_f1_single(self, label_toks: List[str], pred_toks: List[str]) -> float:
  +        """
  +        Compute F1 score for a single sample.
  +        """
  +        common: collections.Counter = collections.Counter(label_toks) & collections.Counter(pred_toks)
  +        num_same = sum(common.values())
  +        if len(label_toks) == 0 or len(pred_toks) == 0:
  +            # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
  +            return int(label_toks == pred_toks)
  +        if num_same == 0:
  +            return 0
  +        precision = 1.0 * num_same / len(pred_toks)
  +        recall = 1.0 * num_same / len(label_toks)
  +        f1 = (2 * precision * recall) / (precision + recall)
  +        return f1
  +
  +    def _calculate_f1(
  +        self, output_key: str, regexes_to_ignore=None, ignore_case=False, ignore_punctuation=False, ignore_numbers=False
  +    ) -> MetricsResult:
  +        """
  +        Calculates the F1 score between two lists of predictions and labels.
  +        F1 score measures the word overlap between the predicted text and the corresponding ground truth label.
  +
  +        :param output_key: The key of the output to use for comparison.
  +        :param regexes_to_ignore (list, optional): A list of regular expressions. If provided, it removes substrings
  +            matching these regular expressions from both predictions and labels before comparison. Defaults to None.
  +        :param ignore_case (bool, optional): If True, performs case-insensitive comparison. Defaults to False.
  +        :param ignore_punctuation (bool, optional): If True, removes punctuation from both predictions and labels before
  +            comparison. Defaults to False.
  +        :param ignore_numbers (bool, optional): If True, removes numerical digits from both predictions and labels
  +            before comparison. Defaults to False.
  +        :return: A MetricsResult object containing the calculated Exact Match (EM) score.
  +        """
  +
  +        predictions = get_answers_from_output(
  +            outputs=self.outputs, output_key=output_key, runnable_type=self.runnable_type
  +        )
  +        labels = get_answers_from_output(
  +            outputs=self.expected_outputs, output_key=output_key, runnable_type=self.runnable_type
  +        )
  +
  +        if len(predictions) != len(labels):
  +            raise ValueError("The number of predictions and labels must be the same.")
  +        if len(predictions) == len(labels) == 0:
  +            # Return F1 as 0 for no inputs
  +            return MetricsResult({"f1": 0.0})
  +
  +        predictions = preprocess_text(predictions, regexes_to_ignore, ignore_case, ignore_punctuation, ignore_numbers)
  +        labels = preprocess_text(labels, regexes_to_ignore, ignore_case, ignore_punctuation, ignore_numbers)
  +
  +        # Tokenize by splitting on spaces
  +        tokenized_predictions = [pred.split() for pred in predictions]
  +        tokenized_labels = [label.split() for label in labels]
  +
  +        f1_scores = [
  +            self._compute_f1_single(label_toks, pred_toks)
  +            for label_toks, pred_toks in zip(tokenized_labels, tokenized_predictions)
  +        ]
  +
  +        f1 = np.mean(f1_scores)
  +
  +        return MetricsResult({"f1": f1})
   
       def _calculate_em(
           self, output_key: str, regexes_to_ignore=None, ignore_case=False, ignore_punctuation=False, ignore_numbers=False
  
verification_info: {"golden_diff": "diff --git a/haystack/evaluation/eval.py b/haystack/evaluation/eval.py\n--- a/haystack/evaluation/eval.py\n+++ b/haystack/evaluation/eval.py\n@@ -1,3 +1,4 @@\n+import collections\n from typing import Any, Callable, Dict, List, Union\n \n import numpy as np\n@@ -71,8 +72,68 @@\n     def _calculate_mrr(self):\n         return MetricsResult({\"mean_reciprocal_rank\": None})\n \n-    def _calculate_f1(self):\n-        return MetricsResult({\"f1\": None})\n+    def _compute_f1_single(self, label_toks: List[str], pred_toks: List[str]) -> float:\n+        \"\"\"\n+        Compute F1 score for a single sample.\n+        \"\"\"\n+        common: collections.Counter = collections.Counter(label_toks) & collections.Counter(pred_toks)\n+        num_same = sum(common.values())\n+        if len(label_toks) == 0 or len(pred_toks) == 0:\n+            # If either is no-answer, then F1 is 1 if they agree, 0 otherwise\n+            return int(label_toks == pred_toks)\n+        if num_same == 0:\n+            return 0\n+        precision = 1.0 * num_same / len(pred_toks)\n+        recall = 1.0 * num_same / len(label_toks)\n+        f1 = (2 * precision * recall) / (precision + recall)\n+        return f1\n+\n+    def _calculate_f1(\n+        self, output_key: str, regexes_to_ignore=None, ignore_case=False, ignore_punctuation=False, ignore_numbers=False\n+    ) -> MetricsResult:\n+        \"\"\"\n+        Calculates the F1 score between two lists of predictions and labels.\n+        F1 score measures the word overlap between the predicted text and the corresponding ground truth label.\n+\n+        :param output_key: The key of the output to use for comparison.\n+        :param regexes_to_ignore (list, optional): A list of regular expressions. If provided, it removes substrings\n+            matching these regular expressions from both predictions and labels before comparison. Defaults to None.\n+        :param ignore_case (bool, optional): If True, performs case-insensitive comparison. Defaults to False.\n+        :param ignore_punctuation (bool, optional): If True, removes punctuation from both predictions and labels before\n+            comparison. Defaults to False.\n+        :param ignore_numbers (bool, optional): If True, removes numerical digits from both predictions and labels\n+            before comparison. Defaults to False.\n+        :return: A MetricsResult object containing the calculated Exact Match (EM) score.\n+        \"\"\"\n+\n+        predictions = get_answers_from_output(\n+            outputs=self.outputs, output_key=output_key, runnable_type=self.runnable_type\n+        )\n+        labels = get_answers_from_output(\n+            outputs=self.expected_outputs, output_key=output_key, runnable_type=self.runnable_type\n+        )\n+\n+        if len(predictions) != len(labels):\n+            raise ValueError(\"The number of predictions and labels must be the same.\")\n+        if len(predictions) == len(labels) == 0:\n+            # Return F1 as 0 for no inputs\n+            return MetricsResult({\"f1\": 0.0})\n+\n+        predictions = preprocess_text(predictions, regexes_to_ignore, ignore_case, ignore_punctuation, ignore_numbers)\n+        labels = preprocess_text(labels, regexes_to_ignore, ignore_case, ignore_punctuation, ignore_numbers)\n+\n+        # Tokenize by splitting on spaces\n+        tokenized_predictions = [pred.split() for pred in predictions]\n+        tokenized_labels = [label.split() for label in labels]\n+\n+        f1_scores = [\n+            self._compute_f1_single(label_toks, pred_toks)\n+            for label_toks, pred_toks in zip(tokenized_labels, tokenized_predictions)\n+        ]\n+\n+        f1 = np.mean(f1_scores)\n+\n+        return MetricsResult({\"f1\": f1})\n \n     def _calculate_em(\n         self, output_key: str, regexes_to_ignore=None, ignore_case=False, ignore_punctuation=False, ignore_numbers=False\n", "issue": "Implement function to calculate F1 metric\nAs specified in proposal #5794 we need to implement a function to calculate the F1 metric.\r\n\r\nIdeally the function should be part of the private interface and called only through the `calculate_metrics` function (see #6063). `_calculate_f1()` could be a nice name.\r\n\r\nFor more detailed information check out the original proposal.\n", "before_files": [{"content": "from typing import Any, Callable, Dict, List, Union\n\nimport numpy as np\n\nfrom haystack import Pipeline\nfrom haystack.core.component import Component\nfrom haystack.evaluation.eval_utils import get_answers_from_output, preprocess_text\nfrom haystack.evaluation.metrics import Metric, MetricsResult\n\n\nclass EvaluationResult:\n    \"\"\"\n    EvaluationResult keeps track of all the information related to evaluation, namely the runnable (Pipeline or\n    component), inputs, outputs, and expected outputs.\n    The EvaluationResult keeps track of all the information stored by eval.\n\n    :param runnable: The runnable (Pipeline or component) used for evaluation.\n    :param inputs: List of inputs used for evaluation.\n    :param outputs: List of outputs generated by the runnable.\n    :param expected_outputs: List of expected outputs used for evaluation.\n    \"\"\"\n\n    def __init__(\n        self,\n        runnable: Union[Pipeline, Component],\n        inputs: List[Dict[str, Any]],\n        outputs: List[Dict[str, Any]],\n        expected_outputs: List[Dict[str, Any]],\n    ) -> None:\n        self.runnable = runnable\n        self.inputs = inputs\n        self.outputs = outputs\n        self.expected_outputs = expected_outputs\n\n        # Determine the type of the runnable\n        if str(type(runnable).__name__) == \"Pipeline\":\n            self.runnable_type = \"pipeline\"\n        else:\n            self.runnable_type = \"component\"\n\n        # Mapping of metrics to their corresponding functions.\n        # This should be kept in sync with the Metric enum\n        self._supported_metrics: Dict[Metric, Callable[..., MetricsResult]] = {\n            Metric.RECALL: self._calculate_recall,\n            Metric.MRR: self._calculate_mrr,\n            Metric.MAP: self._calculate_map,\n            Metric.F1: self._calculate_f1,\n            Metric.EM: self._calculate_em,\n            Metric.SAS: self._calculate_sas,\n        }\n\n    def calculate_metrics(self, metric: Union[Metric, Callable[..., MetricsResult]], **kwargs) -> MetricsResult:\n        \"\"\"\n        Calculate evaluation metrics based on the provided Metric or using the custom metric function.\n\n        :param metric: The Metric indicating the type of metric to calculate or custom function to compute.\n        :return: MetricsResult containing the calculated metric.\n        \"\"\"\n\n        if isinstance(metric, Metric):\n            return self._supported_metrics[metric](**kwargs)\n\n        return metric(self, **kwargs)\n\n    def _calculate_recall(self):\n        return MetricsResult({\"recall\": None})\n\n    def _calculate_map(self):\n        return MetricsResult({\"mean_average_precision\": None})\n\n    def _calculate_mrr(self):\n        return MetricsResult({\"mean_reciprocal_rank\": None})\n\n    def _calculate_f1(self):\n        return MetricsResult({\"f1\": None})\n\n    def _calculate_em(\n        self, output_key: str, regexes_to_ignore=None, ignore_case=False, ignore_punctuation=False, ignore_numbers=False\n    ) -> MetricsResult:\n        \"\"\"\n        Calculates the Exact Match (EM) score between two lists of predictions and labels.\n        Exact Match (EM) score measures the percentage of samples where the predicted text exactly matches the\n          corresponding ground truth label.\n\n        :param output_key: The key of the output to use for comparison.\n        :param regexes_to_ignore (list, optional): A list of regular expressions. If provided, it removes substrings\n            matching these regular expressions from both predictions and labels before comparison. Defaults to None.\n        :param ignore_case (bool, optional): If True, performs case-insensitive comparison. Defaults to False.\n        :param ignore_punctuation (bool, optional): If True, removes punctuation from both predictions and labels before\n            comparison. Defaults to False.\n        :param ignore_numbers (bool, optional): If True, removes numerical digits from both predictions and labels\n            before comparison. Defaults to False.\n        :return: A MetricsResult object containing the calculated Exact Match (EM) score.\n        \"\"\"\n\n        predictions = get_answers_from_output(\n            outputs=self.outputs, output_key=output_key, runnable_type=self.runnable_type\n        )\n        labels = get_answers_from_output(\n            outputs=self.expected_outputs, output_key=output_key, runnable_type=self.runnable_type\n        )\n\n        if len(predictions) != len(labels):\n            raise ValueError(\"The number of predictions and labels must be the same.\")\n        if len(predictions) == len(labels) == 0:\n            # Return Exact Match as 0 for no inputs\n            return MetricsResult({\"exact_match\": 0.0})\n\n        predictions = preprocess_text(predictions, regexes_to_ignore, ignore_case, ignore_punctuation, ignore_numbers)\n        labels = preprocess_text(labels, regexes_to_ignore, ignore_case, ignore_punctuation, ignore_numbers)\n\n        score_list = np.array(predictions) == np.array(labels)\n        exact_match_score = np.mean(score_list)\n\n        return MetricsResult({\"exact_match\": exact_match_score})\n\n    def _calculate_sas(self):\n        return MetricsResult({\"exact_match\": None})\n\n\ndef eval(\n    runnable: Union[Pipeline, Component], inputs: List[Dict[str, Any]], expected_outputs: List[Dict[str, Any]]\n) -> EvaluationResult:\n    \"\"\"\n    Evaluates the provided Pipeline or component based on the given inputs and expected outputs.\n\n    This function facilitates the evaluation of a given runnable (either a Pipeline or a component) using the provided\n    inputs and corresponding expected outputs.\n\n    :param runnable: The runnable (Pipeline or component) used for evaluation.\n    :param inputs: List of inputs used for evaluation.\n    :param expected_outputs: List of expected outputs used for evaluation.\n\n    :return: An instance of EvaluationResult containing information about the evaluation, including the runnable,\n    inputs, outputs, and expected outputs.\n    \"\"\"\n\n    outputs = []\n\n    # Check that expected outputs has the correct shape\n    if len(inputs) != len(expected_outputs):\n        raise ValueError(\n            f\"The number of inputs ({len(inputs)}) does not match the number of expected outputs \"\n            f\"({len(expected_outputs)}). Please ensure that each input has a corresponding expected output.\"\n        )\n\n    for input_ in inputs:\n        output = runnable.run(input_)\n        outputs.append(output)\n\n    return EvaluationResult(runnable, inputs, outputs, expected_outputs)\n", "path": "haystack/evaluation/eval.py"}], "after_files": [{"content": "import collections\nfrom typing import Any, Callable, Dict, List, Union\n\nimport numpy as np\n\nfrom haystack import Pipeline\nfrom haystack.core.component import Component\nfrom haystack.evaluation.eval_utils import get_answers_from_output, preprocess_text\nfrom haystack.evaluation.metrics import Metric, MetricsResult\n\n\nclass EvaluationResult:\n    \"\"\"\n    EvaluationResult keeps track of all the information related to evaluation, namely the runnable (Pipeline or\n    component), inputs, outputs, and expected outputs.\n    The EvaluationResult keeps track of all the information stored by eval.\n\n    :param runnable: The runnable (Pipeline or component) used for evaluation.\n    :param inputs: List of inputs used for evaluation.\n    :param outputs: List of outputs generated by the runnable.\n    :param expected_outputs: List of expected outputs used for evaluation.\n    \"\"\"\n\n    def __init__(\n        self,\n        runnable: Union[Pipeline, Component],\n        inputs: List[Dict[str, Any]],\n        outputs: List[Dict[str, Any]],\n        expected_outputs: List[Dict[str, Any]],\n    ) -> None:\n        self.runnable = runnable\n        self.inputs = inputs\n        self.outputs = outputs\n        self.expected_outputs = expected_outputs\n\n        # Determine the type of the runnable\n        if str(type(runnable).__name__) == \"Pipeline\":\n            self.runnable_type = \"pipeline\"\n        else:\n            self.runnable_type = \"component\"\n\n        # Mapping of metrics to their corresponding functions.\n        # This should be kept in sync with the Metric enum\n        self._supported_metrics: Dict[Metric, Callable[..., MetricsResult]] = {\n            Metric.RECALL: self._calculate_recall,\n            Metric.MRR: self._calculate_mrr,\n            Metric.MAP: self._calculate_map,\n            Metric.F1: self._calculate_f1,\n            Metric.EM: self._calculate_em,\n            Metric.SAS: self._calculate_sas,\n        }\n\n    def calculate_metrics(self, metric: Union[Metric, Callable[..., MetricsResult]], **kwargs) -> MetricsResult:\n        \"\"\"\n        Calculate evaluation metrics based on the provided Metric or using the custom metric function.\n\n        :param metric: The Metric indicating the type of metric to calculate or custom function to compute.\n        :return: MetricsResult containing the calculated metric.\n        \"\"\"\n\n        if isinstance(metric, Metric):\n            return self._supported_metrics[metric](**kwargs)\n\n        return metric(self, **kwargs)\n\n    def _calculate_recall(self):\n        return MetricsResult({\"recall\": None})\n\n    def _calculate_map(self):\n        return MetricsResult({\"mean_average_precision\": None})\n\n    def _calculate_mrr(self):\n        return MetricsResult({\"mean_reciprocal_rank\": None})\n\n    def _compute_f1_single(self, label_toks: List[str], pred_toks: List[str]) -> float:\n        \"\"\"\n        Compute F1 score for a single sample.\n        \"\"\"\n        common: collections.Counter = collections.Counter(label_toks) & collections.Counter(pred_toks)\n        num_same = sum(common.values())\n        if len(label_toks) == 0 or len(pred_toks) == 0:\n            # If either is no-answer, then F1 is 1 if they agree, 0 otherwise\n            return int(label_toks == pred_toks)\n        if num_same == 0:\n            return 0\n        precision = 1.0 * num_same / len(pred_toks)\n        recall = 1.0 * num_same / len(label_toks)\n        f1 = (2 * precision * recall) / (precision + recall)\n        return f1\n\n    def _calculate_f1(\n        self, output_key: str, regexes_to_ignore=None, ignore_case=False, ignore_punctuation=False, ignore_numbers=False\n    ) -> MetricsResult:\n        \"\"\"\n        Calculates the F1 score between two lists of predictions and labels.\n        F1 score measures the word overlap between the predicted text and the corresponding ground truth label.\n\n        :param output_key: The key of the output to use for comparison.\n        :param regexes_to_ignore (list, optional): A list of regular expressions. If provided, it removes substrings\n            matching these regular expressions from both predictions and labels before comparison. Defaults to None.\n        :param ignore_case (bool, optional): If True, performs case-insensitive comparison. Defaults to False.\n        :param ignore_punctuation (bool, optional): If True, removes punctuation from both predictions and labels before\n            comparison. Defaults to False.\n        :param ignore_numbers (bool, optional): If True, removes numerical digits from both predictions and labels\n            before comparison. Defaults to False.\n        :return: A MetricsResult object containing the calculated Exact Match (EM) score.\n        \"\"\"\n\n        predictions = get_answers_from_output(\n            outputs=self.outputs, output_key=output_key, runnable_type=self.runnable_type\n        )\n        labels = get_answers_from_output(\n            outputs=self.expected_outputs, output_key=output_key, runnable_type=self.runnable_type\n        )\n\n        if len(predictions) != len(labels):\n            raise ValueError(\"The number of predictions and labels must be the same.\")\n        if len(predictions) == len(labels) == 0:\n            # Return F1 as 0 for no inputs\n            return MetricsResult({\"f1\": 0.0})\n\n        predictions = preprocess_text(predictions, regexes_to_ignore, ignore_case, ignore_punctuation, ignore_numbers)\n        labels = preprocess_text(labels, regexes_to_ignore, ignore_case, ignore_punctuation, ignore_numbers)\n\n        # Tokenize by splitting on spaces\n        tokenized_predictions = [pred.split() for pred in predictions]\n        tokenized_labels = [label.split() for label in labels]\n\n        f1_scores = [\n            self._compute_f1_single(label_toks, pred_toks)\n            for label_toks, pred_toks in zip(tokenized_labels, tokenized_predictions)\n        ]\n\n        f1 = np.mean(f1_scores)\n\n        return MetricsResult({\"f1\": f1})\n\n    def _calculate_em(\n        self, output_key: str, regexes_to_ignore=None, ignore_case=False, ignore_punctuation=False, ignore_numbers=False\n    ) -> MetricsResult:\n        \"\"\"\n        Calculates the Exact Match (EM) score between two lists of predictions and labels.\n        Exact Match (EM) score measures the percentage of samples where the predicted text exactly matches the\n          corresponding ground truth label.\n\n        :param output_key: The key of the output to use for comparison.\n        :param regexes_to_ignore (list, optional): A list of regular expressions. If provided, it removes substrings\n            matching these regular expressions from both predictions and labels before comparison. Defaults to None.\n        :param ignore_case (bool, optional): If True, performs case-insensitive comparison. Defaults to False.\n        :param ignore_punctuation (bool, optional): If True, removes punctuation from both predictions and labels before\n            comparison. Defaults to False.\n        :param ignore_numbers (bool, optional): If True, removes numerical digits from both predictions and labels\n            before comparison. Defaults to False.\n        :return: A MetricsResult object containing the calculated Exact Match (EM) score.\n        \"\"\"\n\n        predictions = get_answers_from_output(\n            outputs=self.outputs, output_key=output_key, runnable_type=self.runnable_type\n        )\n        labels = get_answers_from_output(\n            outputs=self.expected_outputs, output_key=output_key, runnable_type=self.runnable_type\n        )\n\n        if len(predictions) != len(labels):\n            raise ValueError(\"The number of predictions and labels must be the same.\")\n        if len(predictions) == len(labels) == 0:\n            # Return Exact Match as 0 for no inputs\n            return MetricsResult({\"exact_match\": 0.0})\n\n        predictions = preprocess_text(predictions, regexes_to_ignore, ignore_case, ignore_punctuation, ignore_numbers)\n        labels = preprocess_text(labels, regexes_to_ignore, ignore_case, ignore_punctuation, ignore_numbers)\n\n        score_list = np.array(predictions) == np.array(labels)\n        exact_match_score = np.mean(score_list)\n\n        return MetricsResult({\"exact_match\": exact_match_score})\n\n    def _calculate_sas(self):\n        return MetricsResult({\"exact_match\": None})\n\n\ndef eval(\n    runnable: Union[Pipeline, Component], inputs: List[Dict[str, Any]], expected_outputs: List[Dict[str, Any]]\n) -> EvaluationResult:\n    \"\"\"\n    Evaluates the provided Pipeline or component based on the given inputs and expected outputs.\n\n    This function facilitates the evaluation of a given runnable (either a Pipeline or a component) using the provided\n    inputs and corresponding expected outputs.\n\n    :param runnable: The runnable (Pipeline or component) used for evaluation.\n    :param inputs: List of inputs used for evaluation.\n    :param expected_outputs: List of expected outputs used for evaluation.\n\n    :return: An instance of EvaluationResult containing information about the evaluation, including the runnable,\n    inputs, outputs, and expected outputs.\n    \"\"\"\n\n    outputs = []\n\n    # Check that expected outputs has the correct shape\n    if len(inputs) != len(expected_outputs):\n        raise ValueError(\n            f\"The number of inputs ({len(inputs)}) does not match the number of expected outputs \"\n            f\"({len(expected_outputs)}). Please ensure that each input has a corresponding expected output.\"\n        )\n\n    for input_ in inputs:\n        output = runnable.run(input_)\n        outputs.append(output)\n\n    return EvaluationResult(runnable, inputs, outputs, expected_outputs)\n", "path": "haystack/evaluation/eval.py"}]}