We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Ranker scores shouldn't be discarded
**Is your feature request related to a problem? Please describe.**
The reason for using a ranker is that the scores should be more accurate than the retriever scores. For this reason, in that use case you would also want to use the ranker scores later on instead of the retriever score (e.g. when defining a threshold). You currently cannot do that as the ranker scores aren't saved anywhere (see [here](https://github.com/deepset-ai/haystack/blob/b87c0c950b2243f47fb249aa3865d4c46edb16df/haystack/nodes/ranker/sentence_transformers.py)) as the reader only updates the order and not the score. Having the order of documents not dependent on the scores is also unintuitive.

**Describe the solution you'd like**
Replace retriever scores with ranker scores.

**Describe alternatives you've considered**
Having an additional field to save ranker scores.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `haystack/nodes/ranker/sentence_transformers.py`
Content:
```
1 from typing import List, Optional, Union, Tuple, Iterator
2 import logging
3 from pathlib import Path
4 
5 import torch
6 from torch.nn import DataParallel
7 from transformers import AutoModelForSequenceClassification, AutoTokenizer
8 
9 from haystack.errors import HaystackError
10 from haystack.schema import Document
11 from haystack.nodes.ranker.base import BaseRanker
12 from haystack.modeling.utils import initialize_device_settings
13 
14 logger = logging.getLogger(__name__)
15 
16 
17 class SentenceTransformersRanker(BaseRanker):
18     """
19     Sentence Transformer based pre-trained Cross-Encoder model for Document Re-ranking (https://huggingface.co/cross-encoder).
20     Re-Ranking can be used on top of a retriever to boost the performance for document search. This is particularly useful if the retriever has a high recall but is bad in sorting the documents by relevance.
21 
22     SentenceTransformerRanker handles Cross-Encoder models
23         - use a single logit as similarity score e.g.  cross-encoder/ms-marco-MiniLM-L-12-v2
24         - use two output logits (no_answer, has_answer) e.g. deepset/gbert-base-germandpr-reranking
25     https://www.sbert.net/docs/pretrained-models/ce-msmarco.html#usage-with-transformers
26 
27     |  With a SentenceTransformersRanker, you can:
28      - directly get predictions via predict()
29 
30     Usage example:
31     ...
32     retriever = BM25Retriever(document_store=document_store)
33     ranker = SentenceTransformersRanker(model_name_or_path="cross-encoder/ms-marco-MiniLM-L-12-v2")
34     p = Pipeline()
35     p.add_node(component=retriever, name="ESRetriever", inputs=["Query"])
36     p.add_node(component=ranker, name="Ranker", inputs=["ESRetriever"])
37     """
38 
39     def __init__(
40         self,
41         model_name_or_path: Union[str, Path],
42         model_version: Optional[str] = None,
43         top_k: int = 10,
44         use_gpu: bool = True,
45         devices: Optional[List[Union[str, torch.device]]] = None,
46         batch_size: Optional[int] = None,
47     ):
48         """
49         :param model_name_or_path: Directory of a saved model or the name of a public model e.g.
50         'cross-encoder/ms-marco-MiniLM-L-12-v2'.
51         See https://huggingface.co/cross-encoder for full list of available models
52         :param model_version: The version of model to use from the HuggingFace model hub. Can be tag name, branch name, or commit hash.
53         :param top_k: The maximum number of documents to return
54         :param use_gpu: Whether to use all available GPUs or the CPU. Falls back on CPU if no GPU is available.
55         :param devices: List of GPU (or CPU) devices, to limit inference to certain GPUs and not use all available ones
56                         The strings will be converted into pytorch devices, so use the string notation described here:
57                         https://pytorch.org/docs/stable/tensor_attributes.html?highlight=torch%20device#torch.torch.device
58                         (e.g. ["cuda:0"]).
59         :param batch_size: Number of documents to process at a time.
60         """
61         super().__init__()
62 
63         self.top_k = top_k
64 
65         if devices is not None:
66             self.devices = [torch.device(device) for device in devices]
67         else:
68             self.devices, _ = initialize_device_settings(use_cuda=use_gpu, multi_gpu=True)
69 
70         self.transformer_model = AutoModelForSequenceClassification.from_pretrained(
71             pretrained_model_name_or_path=model_name_or_path, revision=model_version
72         )
73         self.transformer_model.to(str(self.devices[0]))
74         self.transformer_tokenizer = AutoTokenizer.from_pretrained(
75             pretrained_model_name_or_path=model_name_or_path, revision=model_version
76         )
77         self.transformer_model.eval()
78 
79         if len(self.devices) > 1:
80             self.model = DataParallel(self.transformer_model, device_ids=self.devices)
81 
82         self.batch_size = batch_size
83 
84     def predict(self, query: str, documents: List[Document], top_k: Optional[int] = None) -> List[Document]:
85         """
86         Use loaded ranker model to re-rank the supplied list of Document.
87 
88         Returns list of Document sorted by (desc.) similarity with the query.
89 
90         :param query: Query string
91         :param documents: List of Document to be re-ranked
92         :param top_k: The maximum number of documents to return
93         :return: List of Document
94         """
95         if top_k is None:
96             top_k = self.top_k
97 
98         features = self.transformer_tokenizer(
99             [query for doc in documents],
100             [doc.content for doc in documents],
101             padding=True,
102             truncation=True,
103             return_tensors="pt",
104         ).to(self.devices[0])
105 
106         # SentenceTransformerRanker uses:
107         # 1. the logit as similarity score/answerable classification
108         # 2. the logits as answerable classification  (no_answer / has_answer)
109         # https://www.sbert.net/docs/pretrained-models/ce-msmarco.html#usage-with-transformers
110         with torch.no_grad():
111             similarity_scores = self.transformer_model(**features).logits
112 
113         logits_dim = similarity_scores.shape[1]  # [batch_size, logits_dim]
114         sorted_scores_and_documents = sorted(
115             zip(similarity_scores, documents),
116             key=lambda similarity_document_tuple:
117             # assume the last element in logits represents the `has_answer` label
118             similarity_document_tuple[0][-1] if logits_dim >= 2 else similarity_document_tuple[0],
119             reverse=True,
120         )
121 
122         # rank documents according to scores
123         sorted_documents = [doc for _, doc in sorted_scores_and_documents]
124         return sorted_documents[:top_k]
125 
126     def predict_batch(
127         self,
128         queries: List[str],
129         documents: Union[List[Document], List[List[Document]]],
130         top_k: Optional[int] = None,
131         batch_size: Optional[int] = None,
132     ) -> Union[List[Document], List[List[Document]]]:
133         """
134         Use loaded ranker model to re-rank the supplied lists of Documents.
135 
136         Returns lists of Documents sorted by (desc.) similarity with the corresponding queries.
137 
138 
139         - If you provide a list containing a single query...
140 
141             - ... and a single list of Documents, the single list of Documents will be re-ranked based on the
142               supplied query.
143             - ... and a list of lists of Documents, each list of Documents will be re-ranked individually based on the
144               supplied query.
145 
146 
147         - If you provide a list of multiple queries...
148 
149             - ... you need to provide a list of lists of Documents. Each list of Documents will be re-ranked based on
150               its corresponding query.
151 
152         :param queries: Single query string or list of queries
153         :param documents: Single list of Documents or list of lists of Documents to be reranked.
154         :param top_k: The maximum number of documents to return per Document list.
155         :param batch_size: Number of Documents to process at a time.
156         """
157         if top_k is None:
158             top_k = self.top_k
159 
160         if batch_size is None:
161             batch_size = self.batch_size
162 
163         number_of_docs, all_queries, all_docs, single_list_of_docs = self._preprocess_batch_queries_and_docs(
164             queries=queries, documents=documents
165         )
166 
167         batches = self._get_batches(all_queries=all_queries, all_docs=all_docs, batch_size=batch_size)
168         preds = []
169         for cur_queries, cur_docs in batches:
170             features = self.transformer_tokenizer(
171                 cur_queries, [doc.content for doc in cur_docs], padding=True, truncation=True, return_tensors="pt"
172             ).to(self.devices[0])
173 
174             with torch.no_grad():
175                 similarity_scores = self.transformer_model(**features).logits
176                 preds.extend(similarity_scores)
177 
178         logits_dim = similarity_scores.shape[1]  # [batch_size, logits_dim]
179         if single_list_of_docs:
180             sorted_scores_and_documents = sorted(
181                 zip(similarity_scores, documents),
182                 key=lambda similarity_document_tuple:
183                 # assume the last element in logits represents the `has_answer` label
184                 similarity_document_tuple[0][-1] if logits_dim >= 2 else similarity_document_tuple[0],
185                 reverse=True,
186             )
187 
188             # rank documents according to scores
189             sorted_documents = [doc for _, doc in sorted_scores_and_documents if isinstance(doc, Document)]
190             return sorted_documents[:top_k]
191         else:
192             # Group predictions together
193             grouped_predictions = []
194             left_idx = 0
195             right_idx = 0
196             for number in number_of_docs:
197                 right_idx = left_idx + number
198                 grouped_predictions.append(similarity_scores[left_idx:right_idx])
199                 left_idx = right_idx
200 
201             result = []
202             for pred_group, doc_group in zip(grouped_predictions, documents):
203                 sorted_scores_and_documents = sorted(
204                     zip(pred_group, doc_group),  # type: ignore
205                     key=lambda similarity_document_tuple:
206                     # assume the last element in logits represents the `has_answer` label
207                     similarity_document_tuple[0][-1] if logits_dim >= 2 else similarity_document_tuple[0],
208                     reverse=True,
209                 )
210 
211                 # rank documents according to scores
212                 sorted_documents = [doc for _, doc in sorted_scores_and_documents if isinstance(doc, Document)][:top_k]
213                 result.append(sorted_documents)
214 
215             return result
216 
217     def _preprocess_batch_queries_and_docs(
218         self, queries: List[str], documents: Union[List[Document], List[List[Document]]]
219     ) -> Tuple[List[int], List[str], List[Document], bool]:
220         number_of_docs = []
221         all_queries = []
222         all_docs: List[Document] = []
223         single_list_of_docs = False
224 
225         # Docs case 1: single list of Documents -> rerank single list of Documents based on single query
226         if len(documents) > 0 and isinstance(documents[0], Document):
227             if len(queries) != 1:
228                 raise HaystackError("Number of queries must be 1 if a single list of Documents is provided.")
229             query = queries[0]
230             number_of_docs = [len(documents)]
231             all_queries = [query] * len(documents)
232             all_docs = documents  # type: ignore
233             single_list_of_docs = True
234 
235         # Docs case 2: list of lists of Documents -> rerank each list of Documents based on corresponding query
236         # If queries contains a single query, apply it to each list of Documents
237         if len(documents) > 0 and isinstance(documents[0], list):
238             if len(queries) == 1:
239                 queries = queries * len(documents)
240             if len(queries) != len(documents):
241                 raise HaystackError("Number of queries must be equal to number of provided Document lists.")
242             for query, cur_docs in zip(queries, documents):
243                 if not isinstance(cur_docs, list):
244                     raise HaystackError(f"cur_docs was of type {type(cur_docs)}, but expected a list of Documents.")
245                 number_of_docs.append(len(cur_docs))
246                 all_queries.extend([query] * len(cur_docs))
247                 all_docs.extend(cur_docs)
248 
249         return number_of_docs, all_queries, all_docs, single_list_of_docs
250 
251     @staticmethod
252     def _get_batches(
253         all_queries: List[str], all_docs: List[Document], batch_size: Optional[int]
254     ) -> Iterator[Tuple[List[str], List[Document]]]:
255         if batch_size is None:
256             yield all_queries, all_docs
257             return
258         else:
259             for index in range(0, len(all_queries), batch_size):
260                 yield all_queries[index : index + batch_size], all_docs[index : index + batch_size]
261
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/haystack/nodes/ranker/sentence_transformers.py b/haystack/nodes/ranker/sentence_transformers.py
--- a/haystack/nodes/ranker/sentence_transformers.py
+++ b/haystack/nodes/ranker/sentence_transformers.py
@@ -1,4 +1,4 @@
-from typing import List, Optional, Union, Tuple, Iterator
+from typing import List, Optional, Union, Tuple, Iterator, Any
 import logging
 from pathlib import Path
 
@@ -44,6 +44,7 @@
         use_gpu: bool = True,
         devices: Optional[List[Union[str, torch.device]]] = None,
         batch_size: Optional[int] = None,
+        scale_score: bool = True,
     ):
         """
         :param model_name_or_path: Directory of a saved model or the name of a public model e.g.
@@ -57,6 +58,9 @@
                         https://pytorch.org/docs/stable/tensor_attributes.html?highlight=torch%20device#torch.torch.device
                         (e.g. ["cuda:0"]).
         :param batch_size: Number of documents to process at a time.
+        :param scale_score: The raw predictions will be transformed using a Sigmoid activation function in case the model
+                            only predicts a single label. For multi-label predictions, no scaling is applied. Set this
+                            to False if you do not want any scaling of the raw predictions.
         """
         super().__init__()
 
@@ -76,6 +80,15 @@
         )
         self.transformer_model.eval()
 
+        # we use sigmoid activation function to scale the score in case there is only a single label
+        # we do not apply any scaling when scale_score is set to False
+        num_labels = self.transformer_model.num_labels
+        self.activation_function: torch.nn.Module
+        if num_labels == 1 and scale_score:
+            self.activation_function = torch.nn.Sigmoid()
+        else:
+            self.activation_function = torch.nn.Identity()
+
         if len(self.devices) > 1:
             self.model = DataParallel(self.transformer_model, device_ids=self.devices)
 
@@ -119,9 +132,31 @@
             reverse=True,
         )
 
-        # rank documents according to scores
-        sorted_documents = [doc for _, doc in sorted_scores_and_documents]
-        return sorted_documents[:top_k]
+        # add normalized scores to documents
+        sorted_documents = self._add_scores_to_documents(sorted_scores_and_documents[:top_k], logits_dim)
+
+        return sorted_documents
+
+    def _add_scores_to_documents(
+        self, sorted_scores_and_documents: List[Tuple[Any, Document]], logits_dim: int
+    ) -> List[Document]:
+        """
+        Normalize and add scores to retrieved result documents.
+
+        :param sorted_scores_and_documents: List of score, Document Tuples.
+        :param logits_dim: Dimensionality of the returned scores.
+        """
+        sorted_documents = []
+        for raw_score, doc in sorted_scores_and_documents:
+            if logits_dim >= 2:
+                score = self.activation_function(raw_score)[-1]
+            else:
+                score = self.activation_function(raw_score)[0]
+
+            doc.score = score.detach().cpu().numpy().tolist()
+            sorted_documents.append(doc)
+
+        return sorted_documents
 
     def predict_batch(
         self,
@@ -185,9 +220,11 @@
                 reverse=True,
             )
 
-            # rank documents according to scores
-            sorted_documents = [doc for _, doc in sorted_scores_and_documents if isinstance(doc, Document)]
-            return sorted_documents[:top_k]
+            # is this step needed?
+            sorted_documents = [(score, doc) for score, doc in sorted_scores_and_documents if isinstance(doc, Document)]
+            sorted_documents_with_scores = self._add_scores_to_documents(sorted_documents[:top_k], logits_dim)
+
+            return sorted_documents_with_scores
         else:
             # Group predictions together
             grouped_predictions = []
@@ -209,8 +246,12 @@
                 )
 
                 # rank documents according to scores
-                sorted_documents = [doc for _, doc in sorted_scores_and_documents if isinstance(doc, Document)][:top_k]
-                result.append(sorted_documents)
+                sorted_documents = [
+                    (score, doc) for score, doc in sorted_scores_and_documents if isinstance(doc, Document)
+                ]
+                sorted_documents_with_scores = self._add_scores_to_documents(sorted_documents[:top_k], logits_dim)
+
+                result.append(sorted_documents_with_scores)
 
             return result

{"golden_diff": "diff --git a/haystack/nodes/ranker/sentence_transformers.py b/haystack/nodes/ranker/sentence_transformers.py\n--- a/haystack/nodes/ranker/sentence_transformers.py\n+++ b/haystack/nodes/ranker/sentence_transformers.py\n@@ -1,4 +1,4 @@\n-from typing import List, Optional, Union, Tuple, Iterator\n+from typing import List, Optional, Union, Tuple, Iterator, Any\n import logging\n from pathlib import Path\n \n@@ -44,6 +44,7 @@\n         use_gpu: bool = True,\n         devices: Optional[List[Union[str, torch.device]]] = None,\n         batch_size: Optional[int] = None,\n+        scale_score: bool = True,\n     ):\n         \"\"\"\n         :param model_name_or_path: Directory of a saved model or the name of a public model e.g.\n@@ -57,6 +58,9 @@\n                         https://pytorch.org/docs/stable/tensor_attributes.html?highlight=torch%20device#torch.torch.device\n                         (e.g. [\"cuda:0\"]).\n         :param batch_size: Number of documents to process at a time.\n+        :param scale_score: The raw predictions will be transformed using a Sigmoid activation function in case the model\n+                            only predicts a single label. For multi-label predictions, no scaling is applied. Set this\n+                            to False if you do not want any scaling of the raw predictions.\n         \"\"\"\n         super().__init__()\n \n@@ -76,6 +80,15 @@\n         )\n         self.transformer_model.eval()\n \n+        # we use sigmoid activation function to scale the score in case there is only a single label\n+        # we do not apply any scaling when scale_score is set to False\n+        num_labels = self.transformer_model.num_labels\n+        self.activation_function: torch.nn.Module\n+        if num_labels == 1 and scale_score:\n+            self.activation_function = torch.nn.Sigmoid()\n+        else:\n+            self.activation_function = torch.nn.Identity()\n+\n         if len(self.devices) > 1:\n             self.model = DataParallel(self.transformer_model, device_ids=self.devices)\n \n@@ -119,9 +132,31 @@\n             reverse=True,\n         )\n \n-        # rank documents according to scores\n-        sorted_documents = [doc for _, doc in sorted_scores_and_documents]\n-        return sorted_documents[:top_k]\n+        # add normalized scores to documents\n+        sorted_documents = self._add_scores_to_documents(sorted_scores_and_documents[:top_k], logits_dim)\n+\n+        return sorted_documents\n+\n+    def _add_scores_to_documents(\n+        self, sorted_scores_and_documents: List[Tuple[Any, Document]], logits_dim: int\n+    ) -> List[Document]:\n+        \"\"\"\n+        Normalize and add scores to retrieved result documents.\n+\n+        :param sorted_scores_and_documents: List of score, Document Tuples.\n+        :param logits_dim: Dimensionality of the returned scores.\n+        \"\"\"\n+        sorted_documents = []\n+        for raw_score, doc in sorted_scores_and_documents:\n+            if logits_dim >= 2:\n+                score = self.activation_function(raw_score)[-1]\n+            else:\n+                score = self.activation_function(raw_score)[0]\n+\n+            doc.score = score.detach().cpu().numpy().tolist()\n+            sorted_documents.append(doc)\n+\n+        return sorted_documents\n \n     def predict_batch(\n         self,\n@@ -185,9 +220,11 @@\n                 reverse=True,\n             )\n \n-            # rank documents according to scores\n-            sorted_documents = [doc for _, doc in sorted_scores_and_documents if isinstance(doc, Document)]\n-            return sorted_documents[:top_k]\n+            # is this step needed?\n+            sorted_documents = [(score, doc) for score, doc in sorted_scores_and_documents if isinstance(doc, Document)]\n+            sorted_documents_with_scores = self._add_scores_to_documents(sorted_documents[:top_k], logits_dim)\n+\n+            return sorted_documents_with_scores\n         else:\n             # Group predictions together\n             grouped_predictions = []\n@@ -209,8 +246,12 @@\n                 )\n \n                 # rank documents according to scores\n-                sorted_documents = [doc for _, doc in sorted_scores_and_documents if isinstance(doc, Document)][:top_k]\n-                result.append(sorted_documents)\n+                sorted_documents = [\n+                    (score, doc) for score, doc in sorted_scores_and_documents if isinstance(doc, Document)\n+                ]\n+                sorted_documents_with_scores = self._add_scores_to_documents(sorted_documents[:top_k], logits_dim)\n+\n+                result.append(sorted_documents_with_scores)\n \n             return result\n", "issue": "Ranker scores shouldn't be discarded\n**Is your feature request related to a problem? Please describe.**\r\nThe reason for using a ranker is that the scores should be more accurate than the retriever scores. For this reason, in that use case you would also want to use the ranker scores later on instead of the retriever score (e.g. when defining a threshold). You currently cannot do that as the ranker scores aren't saved anywhere (see [here](https://github.com/deepset-ai/haystack/blob/b87c0c950b2243f47fb249aa3865d4c46edb16df/haystack/nodes/ranker/sentence_transformers.py)) as the reader only updates the order and not the score. Having the order of documents not dependent on the scores is also unintuitive.\r\n\r\n**Describe the solution you'd like**\r\nReplace retriever scores with ranker scores.\r\n\r\n**Describe alternatives you've considered**\r\nHaving an additional field to save ranker scores.\n", "before_files": [{"content": "from typing import List, Optional, Union, Tuple, Iterator\nimport logging\nfrom pathlib import Path\n\nimport torch\nfrom torch.nn import DataParallel\nfrom transformers import AutoModelForSequenceClassification, AutoTokenizer\n\nfrom haystack.errors import HaystackError\nfrom haystack.schema import Document\nfrom haystack.nodes.ranker.base import BaseRanker\nfrom haystack.modeling.utils import initialize_device_settings\n\nlogger = logging.getLogger(__name__)\n\n\nclass SentenceTransformersRanker(BaseRanker):\n    \"\"\"\n    Sentence Transformer based pre-trained Cross-Encoder model for Document Re-ranking (https://huggingface.co/cross-encoder).\n    Re-Ranking can be used on top of a retriever to boost the performance for document search. This is particularly useful if the retriever has a high recall but is bad in sorting the documents by relevance.\n\n    SentenceTransformerRanker handles Cross-Encoder models\n        - use a single logit as similarity score e.g.  cross-encoder/ms-marco-MiniLM-L-12-v2\n        - use two output logits (no_answer, has_answer) e.g. deepset/gbert-base-germandpr-reranking\n    https://www.sbert.net/docs/pretrained-models/ce-msmarco.html#usage-with-transformers\n\n    |  With a SentenceTransformersRanker, you can:\n     - directly get predictions via predict()\n\n    Usage example:\n    ...\n    retriever = BM25Retriever(document_store=document_store)\n    ranker = SentenceTransformersRanker(model_name_or_path=\"cross-encoder/ms-marco-MiniLM-L-12-v2\")\n    p = Pipeline()\n    p.add_node(component=retriever, name=\"ESRetriever\", inputs=[\"Query\"])\n    p.add_node(component=ranker, name=\"Ranker\", inputs=[\"ESRetriever\"])\n    \"\"\"\n\n    def __init__(\n        self,\n        model_name_or_path: Union[str, Path],\n        model_version: Optional[str] = None,\n        top_k: int = 10,\n        use_gpu: bool = True,\n        devices: Optional[List[Union[str, torch.device]]] = None,\n        batch_size: Optional[int] = None,\n    ):\n        \"\"\"\n        :param model_name_or_path: Directory of a saved model or the name of a public model e.g.\n        'cross-encoder/ms-marco-MiniLM-L-12-v2'.\n        See https://huggingface.co/cross-encoder for full list of available models\n        :param model_version: The version of model to use from the HuggingFace model hub. Can be tag name, branch name, or commit hash.\n        :param top_k: The maximum number of documents to return\n        :param use_gpu: Whether to use all available GPUs or the CPU. Falls back on CPU if no GPU is available.\n        :param devices: List of GPU (or CPU) devices, to limit inference to certain GPUs and not use all available ones\n                        The strings will be converted into pytorch devices, so use the string notation described here:\n                        https://pytorch.org/docs/stable/tensor_attributes.html?highlight=torch%20device#torch.torch.device\n                        (e.g. [\"cuda:0\"]).\n        :param batch_size: Number of documents to process at a time.\n        \"\"\"\n        super().__init__()\n\n        self.top_k = top_k\n\n        if devices is not None:\n            self.devices = [torch.device(device) for device in devices]\n        else:\n            self.devices, _ = initialize_device_settings(use_cuda=use_gpu, multi_gpu=True)\n\n        self.transformer_model = AutoModelForSequenceClassification.from_pretrained(\n            pretrained_model_name_or_path=model_name_or_path, revision=model_version\n        )\n        self.transformer_model.to(str(self.devices[0]))\n        self.transformer_tokenizer = AutoTokenizer.from_pretrained(\n            pretrained_model_name_or_path=model_name_or_path, revision=model_version\n        )\n        self.transformer_model.eval()\n\n        if len(self.devices) > 1:\n            self.model = DataParallel(self.transformer_model, device_ids=self.devices)\n\n        self.batch_size = batch_size\n\n    def predict(self, query: str, documents: List[Document], top_k: Optional[int] = None) -> List[Document]:\n        \"\"\"\n        Use loaded ranker model to re-rank the supplied list of Document.\n\n        Returns list of Document sorted by (desc.) similarity with the query.\n\n        :param query: Query string\n        :param documents: List of Document to be re-ranked\n        :param top_k: The maximum number of documents to return\n        :return: List of Document\n        \"\"\"\n        if top_k is None:\n            top_k = self.top_k\n\n        features = self.transformer_tokenizer(\n            [query for doc in documents],\n            [doc.content for doc in documents],\n            padding=True,\n            truncation=True,\n            return_tensors=\"pt\",\n        ).to(self.devices[0])\n\n        # SentenceTransformerRanker uses:\n        # 1. the logit as similarity score/answerable classification\n        # 2. the logits as answerable classification  (no_answer / has_answer)\n        # https://www.sbert.net/docs/pretrained-models/ce-msmarco.html#usage-with-transformers\n        with torch.no_grad():\n            similarity_scores = self.transformer_model(**features).logits\n\n        logits_dim = similarity_scores.shape[1]  # [batch_size, logits_dim]\n        sorted_scores_and_documents = sorted(\n            zip(similarity_scores, documents),\n            key=lambda similarity_document_tuple:\n            # assume the last element in logits represents the `has_answer` label\n            similarity_document_tuple[0][-1] if logits_dim >= 2 else similarity_document_tuple[0],\n            reverse=True,\n        )\n\n        # rank documents according to scores\n        sorted_documents = [doc for _, doc in sorted_scores_and_documents]\n        return sorted_documents[:top_k]\n\n    def predict_batch(\n        self,\n        queries: List[str],\n        documents: Union[List[Document], List[List[Document]]],\n        top_k: Optional[int] = None,\n        batch_size: Optional[int] = None,\n    ) -> Union[List[Document], List[List[Document]]]:\n        \"\"\"\n        Use loaded ranker model to re-rank the supplied lists of Documents.\n\n        Returns lists of Documents sorted by (desc.) similarity with the corresponding queries.\n\n\n        - If you provide a list containing a single query...\n\n            - ... and a single list of Documents, the single list of Documents will be re-ranked based on the\n              supplied query.\n            - ... and a list of lists of Documents, each list of Documents will be re-ranked individually based on the\n              supplied query.\n\n\n        - If you provide a list of multiple queries...\n\n            - ... you need to provide a list of lists of Documents. Each list of Documents will be re-ranked based on\n              its corresponding query.\n\n        :param queries: Single query string or list of queries\n        :param documents: Single list of Documents or list of lists of Documents to be reranked.\n        :param top_k: The maximum number of documents to return per Document list.\n        :param batch_size: Number of Documents to process at a time.\n        \"\"\"\n        if top_k is None:\n            top_k = self.top_k\n\n        if batch_size is None:\n            batch_size = self.batch_size\n\n        number_of_docs, all_queries, all_docs, single_list_of_docs = self._preprocess_batch_queries_and_docs(\n            queries=queries, documents=documents\n        )\n\n        batches = self._get_batches(all_queries=all_queries, all_docs=all_docs, batch_size=batch_size)\n        preds = []\n        for cur_queries, cur_docs in batches:\n            features = self.transformer_tokenizer(\n                cur_queries, [doc.content for doc in cur_docs], padding=True, truncation=True, return_tensors=\"pt\"\n            ).to(self.devices[0])\n\n            with torch.no_grad():\n                similarity_scores = self.transformer_model(**features).logits\n                preds.extend(similarity_scores)\n\n        logits_dim = similarity_scores.shape[1]  # [batch_size, logits_dim]\n        if single_list_of_docs:\n            sorted_scores_and_documents = sorted(\n                zip(similarity_scores, documents),\n                key=lambda similarity_document_tuple:\n                # assume the last element in logits represents the `has_answer` label\n                similarity_document_tuple[0][-1] if logits_dim >= 2 else similarity_document_tuple[0],\n                reverse=True,\n            )\n\n            # rank documents according to scores\n            sorted_documents = [doc for _, doc in sorted_scores_and_documents if isinstance(doc, Document)]\n            return sorted_documents[:top_k]\n        else:\n            # Group predictions together\n            grouped_predictions = []\n            left_idx = 0\n            right_idx = 0\n            for number in number_of_docs:\n                right_idx = left_idx + number\n                grouped_predictions.append(similarity_scores[left_idx:right_idx])\n                left_idx = right_idx\n\n            result = []\n            for pred_group, doc_group in zip(grouped_predictions, documents):\n                sorted_scores_and_documents = sorted(\n                    zip(pred_group, doc_group),  # type: ignore\n                    key=lambda similarity_document_tuple:\n                    # assume the last element in logits represents the `has_answer` label\n                    similarity_document_tuple[0][-1] if logits_dim >= 2 else similarity_document_tuple[0],\n                    reverse=True,\n                )\n\n                # rank documents according to scores\n                sorted_documents = [doc for _, doc in sorted_scores_and_documents if isinstance(doc, Document)][:top_k]\n                result.append(sorted_documents)\n\n            return result\n\n    def _preprocess_batch_queries_and_docs(\n        self, queries: List[str], documents: Union[List[Document], List[List[Document]]]\n    ) -> Tuple[List[int], List[str], List[Document], bool]:\n        number_of_docs = []\n        all_queries = []\n        all_docs: List[Document] = []\n        single_list_of_docs = False\n\n        # Docs case 1: single list of Documents -> rerank single list of Documents based on single query\n        if len(documents) > 0 and isinstance(documents[0], Document):\n            if len(queries) != 1:\n                raise HaystackError(\"Number of queries must be 1 if a single list of Documents is provided.\")\n            query = queries[0]\n            number_of_docs = [len(documents)]\n            all_queries = [query] * len(documents)\n            all_docs = documents  # type: ignore\n            single_list_of_docs = True\n\n        # Docs case 2: list of lists of Documents -> rerank each list of Documents based on corresponding query\n        # If queries contains a single query, apply it to each list of Documents\n        if len(documents) > 0 and isinstance(documents[0], list):\n            if len(queries) == 1:\n                queries = queries * len(documents)\n            if len(queries) != len(documents):\n                raise HaystackError(\"Number of queries must be equal to number of provided Document lists.\")\n            for query, cur_docs in zip(queries, documents):\n                if not isinstance(cur_docs, list):\n                    raise HaystackError(f\"cur_docs was of type {type(cur_docs)}, but expected a list of Documents.\")\n                number_of_docs.append(len(cur_docs))\n                all_queries.extend([query] * len(cur_docs))\n                all_docs.extend(cur_docs)\n\n        return number_of_docs, all_queries, all_docs, single_list_of_docs\n\n    @staticmethod\n    def _get_batches(\n        all_queries: List[str], all_docs: List[Document], batch_size: Optional[int]\n    ) -> Iterator[Tuple[List[str], List[Document]]]:\n        if batch_size is None:\n            yield all_queries, all_docs\n            return\n        else:\n            for index in range(0, len(all_queries), batch_size):\n                yield all_queries[index : index + batch_size], all_docs[index : index + batch_size]\n", "path": "haystack/nodes/ranker/sentence_transformers.py"}], "after_files": [{"content": "from typing import List, Optional, Union, Tuple, Iterator, Any\nimport logging\nfrom pathlib import Path\n\nimport torch\nfrom torch.nn import DataParallel\nfrom transformers import AutoModelForSequenceClassification, AutoTokenizer\n\nfrom haystack.errors import HaystackError\nfrom haystack.schema import Document\nfrom haystack.nodes.ranker.base import BaseRanker\nfrom haystack.modeling.utils import initialize_device_settings\n\nlogger = logging.getLogger(__name__)\n\n\nclass SentenceTransformersRanker(BaseRanker):\n    \"\"\"\n    Sentence Transformer based pre-trained Cross-Encoder model for Document Re-ranking (https://huggingface.co/cross-encoder).\n    Re-Ranking can be used on top of a retriever to boost the performance for document search. This is particularly useful if the retriever has a high recall but is bad in sorting the documents by relevance.\n\n    SentenceTransformerRanker handles Cross-Encoder models\n        - use a single logit as similarity score e.g.  cross-encoder/ms-marco-MiniLM-L-12-v2\n        - use two output logits (no_answer, has_answer) e.g. deepset/gbert-base-germandpr-reranking\n    https://www.sbert.net/docs/pretrained-models/ce-msmarco.html#usage-with-transformers\n\n    |  With a SentenceTransformersRanker, you can:\n     - directly get predictions via predict()\n\n    Usage example:\n    ...\n    retriever = BM25Retriever(document_store=document_store)\n    ranker = SentenceTransformersRanker(model_name_or_path=\"cross-encoder/ms-marco-MiniLM-L-12-v2\")\n    p = Pipeline()\n    p.add_node(component=retriever, name=\"ESRetriever\", inputs=[\"Query\"])\n    p.add_node(component=ranker, name=\"Ranker\", inputs=[\"ESRetriever\"])\n    \"\"\"\n\n    def __init__(\n        self,\n        model_name_or_path: Union[str, Path],\n        model_version: Optional[str] = None,\n        top_k: int = 10,\n        use_gpu: bool = True,\n        devices: Optional[List[Union[str, torch.device]]] = None,\n        batch_size: Optional[int] = None,\n        scale_score: bool = True,\n    ):\n        \"\"\"\n        :param model_name_or_path: Directory of a saved model or the name of a public model e.g.\n        'cross-encoder/ms-marco-MiniLM-L-12-v2'.\n        See https://huggingface.co/cross-encoder for full list of available models\n        :param model_version: The version of model to use from the HuggingFace model hub. Can be tag name, branch name, or commit hash.\n        :param top_k: The maximum number of documents to return\n        :param use_gpu: Whether to use all available GPUs or the CPU. Falls back on CPU if no GPU is available.\n        :param devices: List of GPU (or CPU) devices, to limit inference to certain GPUs and not use all available ones\n                        The strings will be converted into pytorch devices, so use the string notation described here:\n                        https://pytorch.org/docs/stable/tensor_attributes.html?highlight=torch%20device#torch.torch.device\n                        (e.g. [\"cuda:0\"]).\n        :param batch_size: Number of documents to process at a time.\n        :param scale_score: The raw predictions will be transformed using a Sigmoid activation function in case the model\n                            only predicts a single label. For multi-label predictions, no scaling is applied. Set this\n                            to False if you do not want any scaling of the raw predictions.\n        \"\"\"\n        super().__init__()\n\n        self.top_k = top_k\n\n        if devices is not None:\n            self.devices = [torch.device(device) for device in devices]\n        else:\n            self.devices, _ = initialize_device_settings(use_cuda=use_gpu, multi_gpu=True)\n\n        self.transformer_model = AutoModelForSequenceClassification.from_pretrained(\n            pretrained_model_name_or_path=model_name_or_path, revision=model_version\n        )\n        self.transformer_model.to(str(self.devices[0]))\n        self.transformer_tokenizer = AutoTokenizer.from_pretrained(\n            pretrained_model_name_or_path=model_name_or_path, revision=model_version\n        )\n        self.transformer_model.eval()\n\n        # we use sigmoid activation function to scale the score in case there is only a single label\n        # we do not apply any scaling when scale_score is set to False\n        num_labels = self.transformer_model.num_labels\n        self.activation_function: torch.nn.Module\n        if num_labels == 1 and scale_score:\n            self.activation_function = torch.nn.Sigmoid()\n        else:\n            self.activation_function = torch.nn.Identity()\n\n        if len(self.devices) > 1:\n            self.model = DataParallel(self.transformer_model, device_ids=self.devices)\n\n        self.batch_size = batch_size\n\n    def predict(self, query: str, documents: List[Document], top_k: Optional[int] = None) -> List[Document]:\n        \"\"\"\n        Use loaded ranker model to re-rank the supplied list of Document.\n\n        Returns list of Document sorted by (desc.) similarity with the query.\n\n        :param query: Query string\n        :param documents: List of Document to be re-ranked\n        :param top_k: The maximum number of documents to return\n        :return: List of Document\n        \"\"\"\n        if top_k is None:\n            top_k = self.top_k\n\n        features = self.transformer_tokenizer(\n            [query for doc in documents],\n            [doc.content for doc in documents],\n            padding=True,\n            truncation=True,\n            return_tensors=\"pt\",\n        ).to(self.devices[0])\n\n        # SentenceTransformerRanker uses:\n        # 1. the logit as similarity score/answerable classification\n        # 2. the logits as answerable classification  (no_answer / has_answer)\n        # https://www.sbert.net/docs/pretrained-models/ce-msmarco.html#usage-with-transformers\n        with torch.no_grad():\n            similarity_scores = self.transformer_model(**features).logits\n\n        logits_dim = similarity_scores.shape[1]  # [batch_size, logits_dim]\n        sorted_scores_and_documents = sorted(\n            zip(similarity_scores, documents),\n            key=lambda similarity_document_tuple:\n            # assume the last element in logits represents the `has_answer` label\n            similarity_document_tuple[0][-1] if logits_dim >= 2 else similarity_document_tuple[0],\n            reverse=True,\n        )\n\n        # add normalized scores to documents\n        sorted_documents = self._add_scores_to_documents(sorted_scores_and_documents[:top_k], logits_dim)\n\n        return sorted_documents\n\n    def _add_scores_to_documents(\n        self, sorted_scores_and_documents: List[Tuple[Any, Document]], logits_dim: int\n    ) -> List[Document]:\n        \"\"\"\n        Normalize and add scores to retrieved result documents.\n\n        :param sorted_scores_and_documents: List of score, Document Tuples.\n        :param logits_dim: Dimensionality of the returned scores.\n        \"\"\"\n        sorted_documents = []\n        for raw_score, doc in sorted_scores_and_documents:\n            if logits_dim >= 2:\n                score = self.activation_function(raw_score)[-1]\n            else:\n                score = self.activation_function(raw_score)[0]\n\n            doc.score = score.detach().cpu().numpy().tolist()\n            sorted_documents.append(doc)\n\n        return sorted_documents\n\n    def predict_batch(\n        self,\n        queries: List[str],\n        documents: Union[List[Document], List[List[Document]]],\n        top_k: Optional[int] = None,\n        batch_size: Optional[int] = None,\n    ) -> Union[List[Document], List[List[Document]]]:\n        \"\"\"\n        Use loaded ranker model to re-rank the supplied lists of Documents.\n\n        Returns lists of Documents sorted by (desc.) similarity with the corresponding queries.\n\n\n        - If you provide a list containing a single query...\n\n            - ... and a single list of Documents, the single list of Documents will be re-ranked based on the\n              supplied query.\n            - ... and a list of lists of Documents, each list of Documents will be re-ranked individually based on the\n              supplied query.\n\n\n        - If you provide a list of multiple queries...\n\n            - ... you need to provide a list of lists of Documents. Each list of Documents will be re-ranked based on\n              its corresponding query.\n\n        :param queries: Single query string or list of queries\n        :param documents: Single list of Documents or list of lists of Documents to be reranked.\n        :param top_k: The maximum number of documents to return per Document list.\n        :param batch_size: Number of Documents to process at a time.\n        \"\"\"\n        if top_k is None:\n            top_k = self.top_k\n\n        if batch_size is None:\n            batch_size = self.batch_size\n\n        number_of_docs, all_queries, all_docs, single_list_of_docs = self._preprocess_batch_queries_and_docs(\n            queries=queries, documents=documents\n        )\n\n        batches = self._get_batches(all_queries=all_queries, all_docs=all_docs, batch_size=batch_size)\n        preds = []\n        for cur_queries, cur_docs in batches:\n            features = self.transformer_tokenizer(\n                cur_queries, [doc.content for doc in cur_docs], padding=True, truncation=True, return_tensors=\"pt\"\n            ).to(self.devices[0])\n\n            with torch.no_grad():\n                similarity_scores = self.transformer_model(**features).logits\n                preds.extend(similarity_scores)\n\n        logits_dim = similarity_scores.shape[1]  # [batch_size, logits_dim]\n        if single_list_of_docs:\n            sorted_scores_and_documents = sorted(\n                zip(similarity_scores, documents),\n                key=lambda similarity_document_tuple:\n                # assume the last element in logits represents the `has_answer` label\n                similarity_document_tuple[0][-1] if logits_dim >= 2 else similarity_document_tuple[0],\n                reverse=True,\n            )\n\n            # is this step needed?\n            sorted_documents = [(score, doc) for score, doc in sorted_scores_and_documents if isinstance(doc, Document)]\n            sorted_documents_with_scores = self._add_scores_to_documents(sorted_documents[:top_k], logits_dim)\n\n            return sorted_documents_with_scores\n        else:\n            # Group predictions together\n            grouped_predictions = []\n            left_idx = 0\n            right_idx = 0\n            for number in number_of_docs:\n                right_idx = left_idx + number\n                grouped_predictions.append(similarity_scores[left_idx:right_idx])\n                left_idx = right_idx\n\n            result = []\n            for pred_group, doc_group in zip(grouped_predictions, documents):\n                sorted_scores_and_documents = sorted(\n                    zip(pred_group, doc_group),  # type: ignore\n                    key=lambda similarity_document_tuple:\n                    # assume the last element in logits represents the `has_answer` label\n                    similarity_document_tuple[0][-1] if logits_dim >= 2 else similarity_document_tuple[0],\n                    reverse=True,\n                )\n\n                # rank documents according to scores\n                sorted_documents = [\n                    (score, doc) for score, doc in sorted_scores_and_documents if isinstance(doc, Document)\n                ]\n                sorted_documents_with_scores = self._add_scores_to_documents(sorted_documents[:top_k], logits_dim)\n\n                result.append(sorted_documents_with_scores)\n\n            return result\n\n    def _preprocess_batch_queries_and_docs(\n        self, queries: List[str], documents: Union[List[Document], List[List[Document]]]\n    ) -> Tuple[List[int], List[str], List[Document], bool]:\n        number_of_docs = []\n        all_queries = []\n        all_docs: List[Document] = []\n        single_list_of_docs = False\n\n        # Docs case 1: single list of Documents -> rerank single list of Documents based on single query\n        if len(documents) > 0 and isinstance(documents[0], Document):\n            if len(queries) != 1:\n                raise HaystackError(\"Number of queries must be 1 if a single list of Documents is provided.\")\n            query = queries[0]\n            number_of_docs = [len(documents)]\n            all_queries = [query] * len(documents)\n            all_docs = documents  # type: ignore\n            single_list_of_docs = True\n\n        # Docs case 2: list of lists of Documents -> rerank each list of Documents based on corresponding query\n        # If queries contains a single query, apply it to each list of Documents\n        if len(documents) > 0 and isinstance(documents[0], list):\n            if len(queries) == 1:\n                queries = queries * len(documents)\n            if len(queries) != len(documents):\n                raise HaystackError(\"Number of queries must be equal to number of provided Document lists.\")\n            for query, cur_docs in zip(queries, documents):\n                if not isinstance(cur_docs, list):\n                    raise HaystackError(f\"cur_docs was of type {type(cur_docs)}, but expected a list of Documents.\")\n                number_of_docs.append(len(cur_docs))\n                all_queries.extend([query] * len(cur_docs))\n                all_docs.extend(cur_docs)\n\n        return number_of_docs, all_queries, all_docs, single_list_of_docs\n\n    @staticmethod\n    def _get_batches(\n        all_queries: List[str], all_docs: List[Document], batch_size: Optional[int]\n    ) -> Iterator[Tuple[List[str], List[Document]]]:\n        if batch_size is None:\n            yield all_queries, all_docs\n            return\n        else:\n            for index in range(0, len(all_queries), batch_size):\n                yield all_queries[index : index + batch_size], all_docs[index : index + batch_size]\n", "path": "haystack/nodes/ranker/sentence_transformers.py"}]}