We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
BUG: Active Learning agent not working with Scorer.model_output being a generator
We allow the `Scorer.model_output` to be a generator, as this is need for the huge memory sizes of segmentation:
https://github.com/lightly-ai/lightly/blob/ca83601cd72fde73ade02a156c213f0b0c07d034/lightly/active_learning/scorers/semantic_segmentation.py#L95-L97
In the `ActiveLearningAgent` it is checked if the length of the model output is as expected:
https://github.com/lightly-ai/lightly/blob/ca83601cd72fde73ade02a156c213f0b0c07d034/lightly/active_learning/agents/agent.py#L194-L196

However, this fails for generators: `TypeError: object of type 'generator' has no len()`

## Tasks

- [x] reproduce the error by creating a corresponding unittest
- [x] fix the error, e.g. by making the check on the values of the scores_dict, not on the model_output



--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `lightly/active_learning/agents/agent.py`
Content:
```
1 from typing import *
2 import warnings
3 
4 from lightly.active_learning.config.sampler_config import SamplerConfig
5 from lightly.active_learning.scorers.scorer import Scorer
6 from lightly.api.api_workflow_client import ApiWorkflowClient
7 from lightly.api.bitmask import BitMask
8 from lightly.openapi_generated.swagger_client.models import TagData
9 
10 
11 class ActiveLearningAgent:
12     """Interface for active learning queries.
13 
14     Attributes:
15         api_workflow_client:
16             The client to connect to the api.
17         query_set:
18             Set of filenames corresponding to samples which can possibly be selected.
19             Set to all samples in the query tag or to the whole dataset by default.
20         labeled_set:
21             Set of filenames corresponding to samples in the labeled set.
22             Set to all samples in the preselected tag or to an empty list by default.
23         unlabeled_set:
24             Set of filenames corresponding to samples which are in the query set
25             but not in the labeled set.
26         added_set:
27             Set of filenames corresponding to samples which were added to the 
28             labeled set in the last query.
29 
30     Examples:
31         >>> # set the token and dataset id
32         >>> token = '123'
33         >>> dataset_id = 'XYZ'
34         >>>
35         >>> # create an active learning agent
36         >>> client = ApiWorkflowClient(token, dataset_id)
37         >>> agent = ActiveLearningAgent(client)
38         >>>
39         >>> # make an initial active learning query
40         >>> sampler_config = SamplerConfig(n_samples=100, name='initial-set')
41         >>> agent.query(sampler_config)
42         >>> initial_set = agent.labeled_set
43         >>>
44         >>> # train and evaluate a model on the initial set
45         >>> # make predictions on the query set:
46         >>> query_set = agent.query_set
47         >>> # important:
48         >>> # be sure to keep the order of the query set when you make predictions
49         >>>
50         >>> # create active learning scorer
51         >>> scorer = ScorerClassification(predictions)
52         >>>
53         >>> # make a second active learning query
54         >>> sampler_config = SamplerConfig(n_samples=200, name='second-set')
55         >>> agent.query(sampler_config, scorer)
56         >>> added_set = agent.added_set # access only the samples added by this query
57 
58     """
59 
60     def __init__(self,
61                  api_workflow_client: ApiWorkflowClient,
62                  query_tag_name: str = 'initial-tag',
63                  preselected_tag_name: str = None):
64 
65         self.api_workflow_client = api_workflow_client
66 
67         # set the query_tag_id and preselected_tag_id
68         self._query_tag_id = None
69         self._preselected_tag_id = None
70 
71         # build lookup table for tag_name to tag_id
72         tag_name_id_dict = {}
73         for tag in self.api_workflow_client._get_all_tags():
74             tag_name_id_dict[tag.name] = tag.id
75         # use lookup table to set ids
76         self._query_tag_id = tag_name_id_dict[query_tag_name]
77         if preselected_tag_name is not None:
78             self._preselected_tag_id = tag_name_id_dict[preselected_tag_name]
79 
80         # set the filename lists based on preselected and query tag
81         self._query_tag_bitmask = self._get_query_tag_bitmask()
82         self._preselected_tag_bitmask = self._get_preselected_tag_bitmask()
83         # keep track of the last preselected tag to compute added samples
84         self._old_preselected_tag_bitmask = None
85 
86 
87     def _get_query_tag_bitmask(self):
88         """Initializes the query tag bitmask.
89 
90         """
91         # get query tag from api and set bitmask accordingly
92         query_tag_data = self.api_workflow_client.tags_api.get_tag_by_tag_id(
93             self.api_workflow_client.dataset_id,
94             tag_id=self._query_tag_id
95         )
96         query_tag_bitmask = BitMask.from_hex(query_tag_data.bit_mask_data)
97 
98         return query_tag_bitmask
99 
100     def _get_preselected_tag_bitmask(self):
101         """Initializes the preselected tag bitmask.
102 
103         """
104         if self._preselected_tag_id is None:
105             # if not specified, no samples belong to the preselected tag
106             preselected_tag_bitmask = BitMask.from_hex('0x0')
107         else:
108             # get preselected tag from api and set bitmask accordingly
109             preselected_tag_data = self.api_workflow_client.tags_api.get_tag_by_tag_id(
110                 self.api_workflow_client.dataset_id,
111                 tag_id=self._preselected_tag_id
112             )
113             preselected_tag_bitmask = BitMask.from_hex(preselected_tag_data.bit_mask_data)
114 
115         return preselected_tag_bitmask
116 
117     @property
118     def query_set(self):
119         """List of filenames for which to calculate active learning scores.
120 
121         """
122         return self._query_tag_bitmask.masked_select_from_list(
123             self.api_workflow_client.filenames_on_server
124         )
125 
126     @property
127     def labeled_set(self):
128         """List of filenames indicating selected samples.
129 
130         """
131         return self._preselected_tag_bitmask.masked_select_from_list(
132             self.api_workflow_client.filenames_on_server
133         )
134 
135     @property
136     def unlabeled_set(self):
137         """List of filenames which belong to the query set but are not selected.
138 
139         """
140         # unlabeled set is the query set minus the preselected set
141         unlabeled_tag_bitmask = self._query_tag_bitmask - self._preselected_tag_bitmask
142         return unlabeled_tag_bitmask.masked_select_from_list(
143             self.api_workflow_client.filenames_on_server
144         )
145 
146     @property
147     def added_set(self):
148         """List of filenames of newly added samples (in the last query).
149 
150         Raises:
151             RuntimeError if executed before a query.
152 
153         """
154         # the added set only exists after a query
155         if self._old_preselected_tag_bitmask is None:
156             raise RuntimeError('Cannot compute \"added set\" before querying.')
157         # added set is new preselected set minus the old one
158         added_tag_bitmask = self._preselected_tag_bitmask - self._old_preselected_tag_bitmask
159         return added_tag_bitmask.masked_select_from_list(
160             self.api_workflow_client.filenames_on_server
161         )
162 
163 
164     def query(self,
165               sampler_config: SamplerConfig,
166               al_scorer: Scorer = None) -> Tuple[List[str], List[str]]:
167         """Performs an active learning query.
168 
169         After the query, the labeled set is updated to contain all selected samples,
170         the added set is recalculated as (new labeled set - old labeled set), and
171         the query set stays the same.
172 
173         Args:
174             sampler_config:
175                 The sampling configuration.
176             al_scorer:
177                 An instance of a class inheriting from Scorer, e.g. a ClassificationScorer.
178 
179         """
180 
181         # handle illogical stopping condition
182         if sampler_config.n_samples < len(self.labeled_set):
183             warnings.warn(
184                 f'ActiveLearningAgent.query: The number of samples ({sampler_config.n_samples}) is '
185                 f'smaller than the number of preselected samples ({len(self.labeled_set)}).'
186                 'Skipping the active learning query.'
187             )
188             return
189 
190         # calculate active learning scores
191         scores_dict = None
192         if al_scorer is not None:
193             no_query_samples = len(self.query_set)
194             no_query_samples_with_scores = len(al_scorer.model_output)
195             if no_query_samples != no_query_samples_with_scores:
196                 raise ValueError(
197                     f'Number of query samples ({no_query_samples}) must match '
198                     f'the number of predictions ({no_query_samples_with_scores})!'
199                 )
200             scores_dict = al_scorer.calculate_scores()
201 
202         # perform the sampling
203         new_tag_data = self.api_workflow_client.sampling(
204             sampler_config=sampler_config,
205             al_scores=scores_dict,
206             preselected_tag_id=self._preselected_tag_id,
207             query_tag_id=self._query_tag_id
208         )
209 
210         # update the old preselected_tag
211         self._old_preselected_tag_bitmask = self._preselected_tag_bitmask
212         # set the newly chosen tag as the new preselected_tag
213         self._preselected_tag_id = new_tag_data.id
214         self._preselected_tag_bitmask = self._get_preselected_tag_bitmask()
215
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/lightly/active_learning/agents/agent.py b/lightly/active_learning/agents/agent.py
--- a/lightly/active_learning/agents/agent.py
+++ b/lightly/active_learning/agents/agent.py
@@ -190,15 +190,19 @@
         # calculate active learning scores
         scores_dict = None
         if al_scorer is not None:
-            no_query_samples = len(self.query_set)
-            no_query_samples_with_scores = len(al_scorer.model_output)
-            if no_query_samples != no_query_samples_with_scores:
-                raise ValueError(
-                    f'Number of query samples ({no_query_samples}) must match '
-                    f'the number of predictions ({no_query_samples_with_scores})!'
-                )
             scores_dict = al_scorer.calculate_scores()
 
+            # Check if the length of the query_set and each of the scores are the same
+            no_query_samples = len(self.query_set)
+            for score in scores_dict.values():
+                no_query_samples_with_scores = len(score)
+                if no_query_samples != no_query_samples_with_scores:
+                    raise ValueError(
+                        f'Number of query samples ({no_query_samples}) must match '
+                        f'the number of predictions ({no_query_samples_with_scores})!'
+                    )
+
+
         # perform the sampling
         new_tag_data = self.api_workflow_client.sampling(
             sampler_config=sampler_config,

{"golden_diff": "diff --git a/lightly/active_learning/agents/agent.py b/lightly/active_learning/agents/agent.py\n--- a/lightly/active_learning/agents/agent.py\n+++ b/lightly/active_learning/agents/agent.py\n@@ -190,15 +190,19 @@\n         # calculate active learning scores\n         scores_dict = None\n         if al_scorer is not None:\n-            no_query_samples = len(self.query_set)\n-            no_query_samples_with_scores = len(al_scorer.model_output)\n-            if no_query_samples != no_query_samples_with_scores:\n-                raise ValueError(\n-                    f'Number of query samples ({no_query_samples}) must match '\n-                    f'the number of predictions ({no_query_samples_with_scores})!'\n-                )\n             scores_dict = al_scorer.calculate_scores()\n \n+            # Check if the length of the query_set and each of the scores are the same\n+            no_query_samples = len(self.query_set)\n+            for score in scores_dict.values():\n+                no_query_samples_with_scores = len(score)\n+                if no_query_samples != no_query_samples_with_scores:\n+                    raise ValueError(\n+                        f'Number of query samples ({no_query_samples}) must match '\n+                        f'the number of predictions ({no_query_samples_with_scores})!'\n+                    )\n+\n+\n         # perform the sampling\n         new_tag_data = self.api_workflow_client.sampling(\n             sampler_config=sampler_config,\n", "issue": "BUG: Active Learning agent not working with Scorer.model_output being a generator\nWe allow the `Scorer.model_output` to be a generator, as this is need for the huge memory sizes of segmentation:\r\nhttps://github.com/lightly-ai/lightly/blob/ca83601cd72fde73ade02a156c213f0b0c07d034/lightly/active_learning/scorers/semantic_segmentation.py#L95-L97\r\nIn the `ActiveLearningAgent` it is checked if the length of the model output is as expected:\r\nhttps://github.com/lightly-ai/lightly/blob/ca83601cd72fde73ade02a156c213f0b0c07d034/lightly/active_learning/agents/agent.py#L194-L196\r\n\r\nHowever, this fails for generators: `TypeError: object of type 'generator' has no len()`\r\n\r\n## Tasks\r\n\r\n- [x] reproduce the error by creating a corresponding unittest\r\n- [x] fix the error, e.g. by making the check on the values of the scores_dict, not on the model_output\r\n\r\n\n", "before_files": [{"content": "from typing import *\nimport warnings\n\nfrom lightly.active_learning.config.sampler_config import SamplerConfig\nfrom lightly.active_learning.scorers.scorer import Scorer\nfrom lightly.api.api_workflow_client import ApiWorkflowClient\nfrom lightly.api.bitmask import BitMask\nfrom lightly.openapi_generated.swagger_client.models import TagData\n\n\nclass ActiveLearningAgent:\n    \"\"\"Interface for active learning queries.\n\n    Attributes:\n        api_workflow_client:\n            The client to connect to the api.\n        query_set:\n            Set of filenames corresponding to samples which can possibly be selected.\n            Set to all samples in the query tag or to the whole dataset by default.\n        labeled_set:\n            Set of filenames corresponding to samples in the labeled set.\n            Set to all samples in the preselected tag or to an empty list by default.\n        unlabeled_set:\n            Set of filenames corresponding to samples which are in the query set\n            but not in the labeled set.\n        added_set:\n            Set of filenames corresponding to samples which were added to the \n            labeled set in the last query.\n\n    Examples:\n        >>> # set the token and dataset id\n        >>> token = '123'\n        >>> dataset_id = 'XYZ'\n        >>>\n        >>> # create an active learning agent\n        >>> client = ApiWorkflowClient(token, dataset_id)\n        >>> agent = ActiveLearningAgent(client)\n        >>>\n        >>> # make an initial active learning query\n        >>> sampler_config = SamplerConfig(n_samples=100, name='initial-set')\n        >>> agent.query(sampler_config)\n        >>> initial_set = agent.labeled_set\n        >>>\n        >>> # train and evaluate a model on the initial set\n        >>> # make predictions on the query set:\n        >>> query_set = agent.query_set\n        >>> # important:\n        >>> # be sure to keep the order of the query set when you make predictions\n        >>>\n        >>> # create active learning scorer\n        >>> scorer = ScorerClassification(predictions)\n        >>>\n        >>> # make a second active learning query\n        >>> sampler_config = SamplerConfig(n_samples=200, name='second-set')\n        >>> agent.query(sampler_config, scorer)\n        >>> added_set = agent.added_set # access only the samples added by this query\n\n    \"\"\"\n\n    def __init__(self,\n                 api_workflow_client: ApiWorkflowClient,\n                 query_tag_name: str = 'initial-tag',\n                 preselected_tag_name: str = None):\n\n        self.api_workflow_client = api_workflow_client\n\n        # set the query_tag_id and preselected_tag_id\n        self._query_tag_id = None\n        self._preselected_tag_id = None\n\n        # build lookup table for tag_name to tag_id\n        tag_name_id_dict = {}\n        for tag in self.api_workflow_client._get_all_tags():\n            tag_name_id_dict[tag.name] = tag.id\n        # use lookup table to set ids\n        self._query_tag_id = tag_name_id_dict[query_tag_name]\n        if preselected_tag_name is not None:\n            self._preselected_tag_id = tag_name_id_dict[preselected_tag_name]\n\n        # set the filename lists based on preselected and query tag\n        self._query_tag_bitmask = self._get_query_tag_bitmask()\n        self._preselected_tag_bitmask = self._get_preselected_tag_bitmask()\n        # keep track of the last preselected tag to compute added samples\n        self._old_preselected_tag_bitmask = None\n\n\n    def _get_query_tag_bitmask(self):\n        \"\"\"Initializes the query tag bitmask.\n\n        \"\"\"\n        # get query tag from api and set bitmask accordingly\n        query_tag_data = self.api_workflow_client.tags_api.get_tag_by_tag_id(\n            self.api_workflow_client.dataset_id,\n            tag_id=self._query_tag_id\n        )\n        query_tag_bitmask = BitMask.from_hex(query_tag_data.bit_mask_data)\n\n        return query_tag_bitmask\n\n    def _get_preselected_tag_bitmask(self):\n        \"\"\"Initializes the preselected tag bitmask.\n\n        \"\"\"\n        if self._preselected_tag_id is None:\n            # if not specified, no samples belong to the preselected tag\n            preselected_tag_bitmask = BitMask.from_hex('0x0')\n        else:\n            # get preselected tag from api and set bitmask accordingly\n            preselected_tag_data = self.api_workflow_client.tags_api.get_tag_by_tag_id(\n                self.api_workflow_client.dataset_id,\n                tag_id=self._preselected_tag_id\n            )\n            preselected_tag_bitmask = BitMask.from_hex(preselected_tag_data.bit_mask_data)\n\n        return preselected_tag_bitmask\n\n    @property\n    def query_set(self):\n        \"\"\"List of filenames for which to calculate active learning scores.\n\n        \"\"\"\n        return self._query_tag_bitmask.masked_select_from_list(\n            self.api_workflow_client.filenames_on_server\n        )\n\n    @property\n    def labeled_set(self):\n        \"\"\"List of filenames indicating selected samples.\n\n        \"\"\"\n        return self._preselected_tag_bitmask.masked_select_from_list(\n            self.api_workflow_client.filenames_on_server\n        )\n\n    @property\n    def unlabeled_set(self):\n        \"\"\"List of filenames which belong to the query set but are not selected.\n\n        \"\"\"\n        # unlabeled set is the query set minus the preselected set\n        unlabeled_tag_bitmask = self._query_tag_bitmask - self._preselected_tag_bitmask\n        return unlabeled_tag_bitmask.masked_select_from_list(\n            self.api_workflow_client.filenames_on_server\n        )\n\n    @property\n    def added_set(self):\n        \"\"\"List of filenames of newly added samples (in the last query).\n\n        Raises:\n            RuntimeError if executed before a query.\n\n        \"\"\"\n        # the added set only exists after a query\n        if self._old_preselected_tag_bitmask is None:\n            raise RuntimeError('Cannot compute \\\"added set\\\" before querying.')\n        # added set is new preselected set minus the old one\n        added_tag_bitmask = self._preselected_tag_bitmask - self._old_preselected_tag_bitmask\n        return added_tag_bitmask.masked_select_from_list(\n            self.api_workflow_client.filenames_on_server\n        )\n\n\n    def query(self,\n              sampler_config: SamplerConfig,\n              al_scorer: Scorer = None) -> Tuple[List[str], List[str]]:\n        \"\"\"Performs an active learning query.\n\n        After the query, the labeled set is updated to contain all selected samples,\n        the added set is recalculated as (new labeled set - old labeled set), and\n        the query set stays the same.\n\n        Args:\n            sampler_config:\n                The sampling configuration.\n            al_scorer:\n                An instance of a class inheriting from Scorer, e.g. a ClassificationScorer.\n\n        \"\"\"\n\n        # handle illogical stopping condition\n        if sampler_config.n_samples < len(self.labeled_set):\n            warnings.warn(\n                f'ActiveLearningAgent.query: The number of samples ({sampler_config.n_samples}) is '\n                f'smaller than the number of preselected samples ({len(self.labeled_set)}).'\n                'Skipping the active learning query.'\n            )\n            return\n\n        # calculate active learning scores\n        scores_dict = None\n        if al_scorer is not None:\n            no_query_samples = len(self.query_set)\n            no_query_samples_with_scores = len(al_scorer.model_output)\n            if no_query_samples != no_query_samples_with_scores:\n                raise ValueError(\n                    f'Number of query samples ({no_query_samples}) must match '\n                    f'the number of predictions ({no_query_samples_with_scores})!'\n                )\n            scores_dict = al_scorer.calculate_scores()\n\n        # perform the sampling\n        new_tag_data = self.api_workflow_client.sampling(\n            sampler_config=sampler_config,\n            al_scores=scores_dict,\n            preselected_tag_id=self._preselected_tag_id,\n            query_tag_id=self._query_tag_id\n        )\n\n        # update the old preselected_tag\n        self._old_preselected_tag_bitmask = self._preselected_tag_bitmask\n        # set the newly chosen tag as the new preselected_tag\n        self._preselected_tag_id = new_tag_data.id\n        self._preselected_tag_bitmask = self._get_preselected_tag_bitmask()\n", "path": "lightly/active_learning/agents/agent.py"}], "after_files": [{"content": "from typing import *\nimport warnings\n\nfrom lightly.active_learning.config.sampler_config import SamplerConfig\nfrom lightly.active_learning.scorers.scorer import Scorer\nfrom lightly.api.api_workflow_client import ApiWorkflowClient\nfrom lightly.api.bitmask import BitMask\nfrom lightly.openapi_generated.swagger_client.models import TagData\n\n\nclass ActiveLearningAgent:\n    \"\"\"Interface for active learning queries.\n\n    Attributes:\n        api_workflow_client:\n            The client to connect to the api.\n        query_set:\n            Set of filenames corresponding to samples which can possibly be selected.\n            Set to all samples in the query tag or to the whole dataset by default.\n        labeled_set:\n            Set of filenames corresponding to samples in the labeled set.\n            Set to all samples in the preselected tag or to an empty list by default.\n        unlabeled_set:\n            Set of filenames corresponding to samples which are in the query set\n            but not in the labeled set.\n        added_set:\n            Set of filenames corresponding to samples which were added to the \n            labeled set in the last query.\n\n    Examples:\n        >>> # set the token and dataset id\n        >>> token = '123'\n        >>> dataset_id = 'XYZ'\n        >>>\n        >>> # create an active learning agent\n        >>> client = ApiWorkflowClient(token, dataset_id)\n        >>> agent = ActiveLearningAgent(client)\n        >>>\n        >>> # make an initial active learning query\n        >>> sampler_config = SamplerConfig(n_samples=100, name='initial-set')\n        >>> agent.query(sampler_config)\n        >>> initial_set = agent.labeled_set\n        >>>\n        >>> # train and evaluate a model on the initial set\n        >>> # make predictions on the query set:\n        >>> query_set = agent.query_set\n        >>> # important:\n        >>> # be sure to keep the order of the query set when you make predictions\n        >>>\n        >>> # create active learning scorer\n        >>> scorer = ScorerClassification(predictions)\n        >>>\n        >>> # make a second active learning query\n        >>> sampler_config = SamplerConfig(n_samples=200, name='second-set')\n        >>> agent.query(sampler_config, scorer)\n        >>> added_set = agent.added_set # access only the samples added by this query\n\n    \"\"\"\n\n    def __init__(self,\n                 api_workflow_client: ApiWorkflowClient,\n                 query_tag_name: str = 'initial-tag',\n                 preselected_tag_name: str = None):\n\n        self.api_workflow_client = api_workflow_client\n\n        # set the query_tag_id and preselected_tag_id\n        self._query_tag_id = None\n        self._preselected_tag_id = None\n\n        # build lookup table for tag_name to tag_id\n        tag_name_id_dict = {}\n        for tag in self.api_workflow_client._get_all_tags():\n            tag_name_id_dict[tag.name] = tag.id\n        # use lookup table to set ids\n        self._query_tag_id = tag_name_id_dict[query_tag_name]\n        if preselected_tag_name is not None:\n            self._preselected_tag_id = tag_name_id_dict[preselected_tag_name]\n\n        # set the filename lists based on preselected and query tag\n        self._query_tag_bitmask = self._get_query_tag_bitmask()\n        self._preselected_tag_bitmask = self._get_preselected_tag_bitmask()\n        # keep track of the last preselected tag to compute added samples\n        self._old_preselected_tag_bitmask = None\n\n\n    def _get_query_tag_bitmask(self):\n        \"\"\"Initializes the query tag bitmask.\n\n        \"\"\"\n        # get query tag from api and set bitmask accordingly\n        query_tag_data = self.api_workflow_client.tags_api.get_tag_by_tag_id(\n            self.api_workflow_client.dataset_id,\n            tag_id=self._query_tag_id\n        )\n        query_tag_bitmask = BitMask.from_hex(query_tag_data.bit_mask_data)\n\n        return query_tag_bitmask\n\n    def _get_preselected_tag_bitmask(self):\n        \"\"\"Initializes the preselected tag bitmask.\n\n        \"\"\"\n        if self._preselected_tag_id is None:\n            # if not specified, no samples belong to the preselected tag\n            preselected_tag_bitmask = BitMask.from_hex('0x0')\n        else:\n            # get preselected tag from api and set bitmask accordingly\n            preselected_tag_data = self.api_workflow_client.tags_api.get_tag_by_tag_id(\n                self.api_workflow_client.dataset_id,\n                tag_id=self._preselected_tag_id\n            )\n            preselected_tag_bitmask = BitMask.from_hex(preselected_tag_data.bit_mask_data)\n\n        return preselected_tag_bitmask\n\n    @property\n    def query_set(self):\n        \"\"\"List of filenames for which to calculate active learning scores.\n\n        \"\"\"\n        return self._query_tag_bitmask.masked_select_from_list(\n            self.api_workflow_client.filenames_on_server\n        )\n\n    @property\n    def labeled_set(self):\n        \"\"\"List of filenames indicating selected samples.\n\n        \"\"\"\n        return self._preselected_tag_bitmask.masked_select_from_list(\n            self.api_workflow_client.filenames_on_server\n        )\n\n    @property\n    def unlabeled_set(self):\n        \"\"\"List of filenames which belong to the query set but are not selected.\n\n        \"\"\"\n        # unlabeled set is the query set minus the preselected set\n        unlabeled_tag_bitmask = self._query_tag_bitmask - self._preselected_tag_bitmask\n        return unlabeled_tag_bitmask.masked_select_from_list(\n            self.api_workflow_client.filenames_on_server\n        )\n\n    @property\n    def added_set(self):\n        \"\"\"List of filenames of newly added samples (in the last query).\n\n        Raises:\n            RuntimeError if executed before a query.\n\n        \"\"\"\n        # the added set only exists after a query\n        if self._old_preselected_tag_bitmask is None:\n            raise RuntimeError('Cannot compute \\\"added set\\\" before querying.')\n        # added set is new preselected set minus the old one\n        added_tag_bitmask = self._preselected_tag_bitmask - self._old_preselected_tag_bitmask\n        return added_tag_bitmask.masked_select_from_list(\n            self.api_workflow_client.filenames_on_server\n        )\n\n\n    def query(self,\n              sampler_config: SamplerConfig,\n              al_scorer: Scorer = None) -> Tuple[List[str], List[str]]:\n        \"\"\"Performs an active learning query.\n\n        After the query, the labeled set is updated to contain all selected samples,\n        the added set is recalculated as (new labeled set - old labeled set), and\n        the query set stays the same.\n\n        Args:\n            sampler_config:\n                The sampling configuration.\n            al_scorer:\n                An instance of a class inheriting from Scorer, e.g. a ClassificationScorer.\n\n        \"\"\"\n\n        # handle illogical stopping condition\n        if sampler_config.n_samples < len(self.labeled_set):\n            warnings.warn(\n                f'ActiveLearningAgent.query: The number of samples ({sampler_config.n_samples}) is '\n                f'smaller than the number of preselected samples ({len(self.labeled_set)}).'\n                'Skipping the active learning query.'\n            )\n            return\n\n        # calculate active learning scores\n        scores_dict = None\n        if al_scorer is not None:\n            scores_dict = al_scorer.calculate_scores()\n\n            # Check if the length of the query_set and each of the scores are the same\n            no_query_samples = len(self.query_set)\n            for score in scores_dict.values():\n                no_query_samples_with_scores = len(score)\n                if no_query_samples != no_query_samples_with_scores:\n                    raise ValueError(\n                        f'Number of query samples ({no_query_samples}) must match '\n                        f'the number of predictions ({no_query_samples_with_scores})!'\n                    )\n\n\n        # perform the sampling\n        new_tag_data = self.api_workflow_client.sampling(\n            sampler_config=sampler_config,\n            al_scores=scores_dict,\n            preselected_tag_id=self._preselected_tag_id,\n            query_tag_id=self._query_tag_id\n        )\n\n        # update the old preselected_tag\n        self._old_preselected_tag_bitmask = self._preselected_tag_bitmask\n        # set the newly chosen tag as the new preselected_tag\n        self._preselected_tag_id = new_tag_data.id\n        self._preselected_tag_bitmask = self._get_preselected_tag_bitmask()\n", "path": "lightly/active_learning/agents/agent.py"}]}