We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Add leaderboard history
Each time the ranking for the leaderboard is generated save the resulting ranking dictionary so that we know what the leaderboard looked like at a certain date/time. Would be helpful if someone says in their paper "We were first on the leaderboard (at the time of writing)" and we can then independently verify this. 

Maybe this would require moving out the leaderboard into its own app, deprecating the results list view for a leaderboard detail view.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `app/grandchallenge/evaluation/views.py`
Content:
```
1 from datetime import datetime, timedelta
2 from typing import Dict
3 
4 from django.contrib.messages.views import SuccessMessageMixin
5 from django.contrib.postgres.aggregates import ArrayAgg
6 from django.core.exceptions import ObjectDoesNotExist
7 from django.core.files import File
8 from django.db.models import Q
9 from django.utils import timezone
10 from django.utils.functional import cached_property
11 from django.views.generic import CreateView, DetailView, ListView, UpdateView
12 
13 from grandchallenge.core.permissions.mixins import (
14     UserIsChallengeAdminMixin,
15     UserIsChallengeParticipantOrAdminMixin,
16 )
17 from grandchallenge.core.views import Column, PaginatedTableListView
18 from grandchallenge.evaluation.forms import (
19     ConfigForm,
20     LegacySubmissionForm,
21     MethodForm,
22     SubmissionForm,
23 )
24 from grandchallenge.evaluation.models import (
25     Config,
26     Evaluation,
27     Method,
28     Submission,
29 )
30 from grandchallenge.jqfileupload.widgets.uploader import StagedAjaxFile
31 from grandchallenge.subdomains.utils import reverse
32 from grandchallenge.teams.models import Team
33 
34 
35 class ConfigUpdate(UserIsChallengeAdminMixin, SuccessMessageMixin, UpdateView):
36     form_class = ConfigForm
37     success_message = "Configuration successfully updated"
38 
39     def get_object(self, queryset=None):
40         challenge = self.request.challenge
41         return challenge.evaluation_config
42 
43 
44 class MethodCreate(UserIsChallengeAdminMixin, CreateView):
45     model = Method
46     form_class = MethodForm
47 
48     def get_form_kwargs(self):
49         kwargs = super().get_form_kwargs()
50         kwargs.update({"user": self.request.user})
51         return kwargs
52 
53     def form_valid(self, form):
54         form.instance.creator = self.request.user
55         form.instance.challenge = self.request.challenge
56 
57         uploaded_file: StagedAjaxFile = form.cleaned_data["chunked_upload"][0]
58         form.instance.staged_image_uuid = uploaded_file.uuid
59 
60         return super().form_valid(form)
61 
62 
63 class MethodList(UserIsChallengeAdminMixin, ListView):
64     model = Method
65 
66     def get_queryset(self):
67         queryset = super().get_queryset()
68         return queryset.filter(challenge=self.request.challenge)
69 
70 
71 class MethodDetail(UserIsChallengeAdminMixin, DetailView):
72     model = Method
73 
74 
75 class SubmissionCreateBase(SuccessMessageMixin, CreateView):
76     """
77     Base class for the submission creation forms.
78 
79     It has no permissions, do not use it directly! See the subclasses.
80     """
81 
82     model = Submission
83     success_message = (
84         "Your submission was successful. "
85         "Your result will appear on the leaderboard when it is ready."
86     )
87 
88     def get_form_kwargs(self):
89         kwargs = super().get_form_kwargs()
90 
91         config: Config = Config.objects.get(challenge=self.request.challenge)
92 
93         kwargs.update(
94             {
95                 "user": self.request.user,
96                 "display_comment_field": config.allow_submission_comments,
97                 "supplementary_file_choice": config.supplementary_file_choice,
98                 "supplementary_file_label": config.supplementary_file_label,
99                 "supplementary_file_help_text": config.supplementary_file_help_text,
100                 "publication_url_choice": config.publication_url_choice,
101                 "algorithm_submission": config.submission_kind
102                 == config.SubmissionKind.ALGORITHM,
103             }
104         )
105 
106         return kwargs
107 
108     def get_context_data(self, **kwargs):
109         context = super().get_context_data(**kwargs)
110 
111         config = Config.objects.get(challenge=self.request.challenge)
112 
113         context.update(
114             self.get_next_submission(max_subs=config.daily_submission_limit)
115         )
116 
117         pending_evaluations = Evaluation.objects.filter(
118             submission__challenge=self.request.challenge,
119             submission__creator=self.request.user,
120             status__in=(Evaluation.PENDING, Evaluation.STARTED),
121         ).count()
122 
123         context.update({"pending_evaluations": pending_evaluations})
124 
125         return context
126 
127     def get_next_submission(
128         self, *, max_subs: int, period: timedelta = None, now: datetime = None
129     ) -> Dict:
130         """
131         Determines the number of submissions left for the user in a given time
132         period, and when they can next submit.
133 
134         :return: A dictionary containing remaining_submissions (int) and
135         next_submission_at (datetime)
136         """
137         if now is None:
138             now = timezone.now()
139 
140         if period is None:
141             period = timedelta(days=1)
142 
143         subs = (
144             Submission.objects.filter(
145                 challenge=self.request.challenge,
146                 creator=self.request.user,
147                 created__gte=now - period,
148             )
149             .exclude(evaluation__status=Evaluation.FAILURE)
150             .order_by("-created")
151             .distinct()
152         )
153 
154         try:
155             next_sub_at = subs[max_subs - 1].created + period
156         except (IndexError, AssertionError):
157             next_sub_at = now
158 
159         return {
160             "remaining_submissions": max_subs - len(subs),
161             "next_submission_at": next_sub_at,
162         }
163 
164     def form_valid(self, form):
165         if form.instance.creator is None:
166             form.instance.creator = self.request.user
167 
168         form.instance.challenge = self.request.challenge
169 
170         if "algorithm" in form.cleaned_data:
171             # Algorithm submission
172             form.instance.algorithm_image = form.cleaned_data[
173                 "algorithm"
174             ].latest_ready_image
175         else:
176             # Predictions file submission
177             uploaded_file = form.cleaned_data["chunked_upload"][0]
178             with uploaded_file.open() as f:
179                 form.instance.predictions_file.save(
180                     uploaded_file.name, File(f)
181                 )
182 
183         return super().form_valid(form)
184 
185     def get_success_url(self):
186         return reverse(
187             "evaluation:list",
188             kwargs={"challenge_short_name": self.object.challenge.short_name},
189         )
190 
191 
192 class SubmissionCreate(
193     UserIsChallengeParticipantOrAdminMixin, SubmissionCreateBase
194 ):
195     form_class = SubmissionForm
196 
197 
198 class LegacySubmissionCreate(UserIsChallengeAdminMixin, SubmissionCreateBase):
199     form_class = LegacySubmissionForm
200 
201 
202 class SubmissionList(UserIsChallengeParticipantOrAdminMixin, ListView):
203     model = Submission
204 
205     def get_queryset(self):
206         """Admins see everything, participants just their submissions."""
207         queryset = super().get_queryset()
208         challenge = self.request.challenge
209         if challenge.is_admin(self.request.user):
210             return queryset.filter(challenge=self.request.challenge)
211 
212         else:
213             return queryset.filter(
214                 Q(challenge=self.request.challenge),
215                 Q(creator__pk=self.request.user.pk),
216             )
217 
218 
219 class SubmissionDetail(UserIsChallengeAdminMixin, DetailView):
220     # TODO - if participant: list only their submissions
221     model = Submission
222 
223 
224 class TeamContextMixin:
225     def get_context_data(self, *args, **kwargs):
226         context = super().get_context_data(*args, **kwargs)
227 
228         evaluation_config = self.request.challenge.evaluation_config
229 
230         if evaluation_config.use_teams:
231             user_teams = {
232                 teammember.user.username: (team.name, team.get_absolute_url())
233                 for team in Team.objects.filter(
234                     challenge=self.request.challenge
235                 )
236                 .select_related("challenge")
237                 .prefetch_related("teammember_set__user")
238                 for teammember in team.teammember_set.all()
239             }
240         else:
241             user_teams = {}
242 
243         context.update(
244             {"evaluation_config": evaluation_config, "user_teams": user_teams}
245         )
246 
247         return context
248 
249 
250 class EvaluationList(
251     UserIsChallengeParticipantOrAdminMixin, TeamContextMixin, ListView
252 ):
253     model = Evaluation
254 
255     def get_queryset(self):
256         """Admins see everything, participants just their evaluations."""
257         challenge = self.request.challenge
258 
259         queryset = super().get_queryset()
260         queryset = queryset.select_related(
261             "submission__creator__user_profile", "submission__challenge"
262         ).filter(submission__challenge=challenge)
263 
264         if challenge.is_admin(self.request.user):
265             return queryset
266         else:
267             return queryset.filter(
268                 Q(submission__creator__pk=self.request.user.pk)
269             )
270 
271 
272 class EvaluationDetail(DetailView):
273     # TODO - if participant: list only their evaluations
274     model = Evaluation
275 
276     def get_context_data(self, **kwargs):
277         context = super().get_context_data(**kwargs)
278 
279         try:
280             metrics = self.object.outputs.get(
281                 interface__slug="metrics-json-file"
282             ).value
283         except ObjectDoesNotExist:
284             metrics = None
285 
286         context.update({"metrics": metrics})
287 
288         return context
289 
290 
291 class LeaderboardDetail(TeamContextMixin, PaginatedTableListView):
292     model = Evaluation
293     template_name = "evaluation/leaderboard_detail.html"
294     row_template = "evaluation/leaderboard_row.html"
295     search_fields = ["pk", "submission__creator__username"]
296 
297     @property
298     def columns(self):
299         columns = [
300             Column(title="#", sort_field="rank"),
301             Column(
302                 title="User (Team)" if self.config.use_teams else "User",
303                 sort_field="submission__creator__username",
304             ),
305             Column(title="Created", sort_field="created"),
306         ]
307 
308         if self.config.scoring_method_choice == self.config.MEAN:
309             columns.append(Column(title="Mean Position", sort_field="rank"))
310         elif self.config.scoring_method_choice == self.config.MEDIAN:
311             columns.append(Column(title="Median Position", sort_field="rank"))
312 
313         if self.config.scoring_method_choice == self.config.ABSOLUTE:
314             columns.append(
315                 Column(title=self.config.score_title, sort_field="rank")
316             )
317         else:
318             columns.append(
319                 Column(
320                     title=f"{self.config.score_title} (Position)",
321                     sort_field="rank",
322                     toggleable=True,
323                 )
324             )
325 
326         for c in self.config.extra_results_columns:
327             columns.append(
328                 Column(
329                     title=c["title"]
330                     if self.config.scoring_method_choice
331                     == self.config.ABSOLUTE
332                     else f"{c['title']} (Position)",
333                     sort_field="rank",
334                     toggleable=True,
335                 )
336             )
337 
338         if self.config.display_submission_comments:
339             columns.append(
340                 Column(title="Comment", sort_field="submission__comment")
341             )
342 
343         if self.config.show_publication_url:
344             columns.append(
345                 Column(
346                     title="Publication",
347                     sort_field="submission__publication_url",
348                 )
349             )
350 
351         if self.config.show_supplementary_file_link:
352             columns.append(
353                 Column(
354                     title=self.config.supplementary_file_label,
355                     sort_field="submission__supplementary_file",
356                 )
357             )
358 
359         return columns
360 
361     @cached_property
362     def config(self):
363         return self.request.challenge.evaluation_config
364 
365     def get_row_context(self, job, *args, **kwargs):
366         return {"evaluation": job, "evaluation_config": self.config}
367 
368     def get_unfiltered_queryset(self):
369         queryset = super().get_queryset()
370         queryset = (
371             queryset.select_related(
372                 "submission__creator__user_profile", "submission__challenge"
373             )
374             .filter(
375                 submission__challenge=self.request.challenge,
376                 published=True,
377                 status=Evaluation.SUCCESS,
378                 rank__gt=0,
379             )
380             .annotate(
381                 metrics=ArrayAgg(
382                     "outputs__value",
383                     filter=Q(outputs__interface__slug="metrics-json-file"),
384                 )
385             )
386         )
387         return queryset
388 
389 
390 class EvaluationUpdate(
391     UserIsChallengeAdminMixin, SuccessMessageMixin, UpdateView
392 ):
393     model = Evaluation
394     fields = ("published",)
395     success_message = "Result successfully updated."
396
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/app/grandchallenge/evaluation/views.py b/app/grandchallenge/evaluation/views.py
--- a/app/grandchallenge/evaluation/views.py
+++ b/app/grandchallenge/evaluation/views.py
@@ -1,6 +1,7 @@
 from datetime import datetime, timedelta
 from typing import Dict
 
+from dateutil.relativedelta import relativedelta
 from django.contrib.messages.views import SuccessMessageMixin
 from django.contrib.postgres.aggregates import ArrayAgg
 from django.core.exceptions import ObjectDoesNotExist
@@ -367,6 +368,7 @@
 
     def get_unfiltered_queryset(self):
         queryset = super().get_queryset()
+        queryset = self.filter_by_date(queryset=queryset)
         queryset = (
             queryset.select_related(
                 "submission__creator__user_profile", "submission__challenge"
@@ -386,6 +388,16 @@
         )
         return queryset
 
+    def filter_by_date(self, queryset):
+        if "leaderboardDate" in self.request.GET:
+            year, month, day = self.request.GET["leaderboardDate"].split("-")
+            before = datetime(
+                year=int(year), month=int(month), day=int(day),
+            ) + relativedelta(days=1)
+            return queryset.filter(submission__created__lt=before)
+        else:
+            return queryset
+
 
 class EvaluationUpdate(
     UserIsChallengeAdminMixin, SuccessMessageMixin, UpdateView

{"golden_diff": "diff --git a/app/grandchallenge/evaluation/views.py b/app/grandchallenge/evaluation/views.py\n--- a/app/grandchallenge/evaluation/views.py\n+++ b/app/grandchallenge/evaluation/views.py\n@@ -1,6 +1,7 @@\n from datetime import datetime, timedelta\n from typing import Dict\n \n+from dateutil.relativedelta import relativedelta\n from django.contrib.messages.views import SuccessMessageMixin\n from django.contrib.postgres.aggregates import ArrayAgg\n from django.core.exceptions import ObjectDoesNotExist\n@@ -367,6 +368,7 @@\n \n     def get_unfiltered_queryset(self):\n         queryset = super().get_queryset()\n+        queryset = self.filter_by_date(queryset=queryset)\n         queryset = (\n             queryset.select_related(\n                 \"submission__creator__user_profile\", \"submission__challenge\"\n@@ -386,6 +388,16 @@\n         )\n         return queryset\n \n+    def filter_by_date(self, queryset):\n+        if \"leaderboardDate\" in self.request.GET:\n+            year, month, day = self.request.GET[\"leaderboardDate\"].split(\"-\")\n+            before = datetime(\n+                year=int(year), month=int(month), day=int(day),\n+            ) + relativedelta(days=1)\n+            return queryset.filter(submission__created__lt=before)\n+        else:\n+            return queryset\n+\n \n class EvaluationUpdate(\n     UserIsChallengeAdminMixin, SuccessMessageMixin, UpdateView\n", "issue": "Add leaderboard history\nEach time the ranking for the leaderboard is generated save the resulting ranking dictionary so that we know what the leaderboard looked like at a certain date/time. Would be helpful if someone says in their paper \"We were first on the leaderboard (at the time of writing)\" and we can then independently verify this. \r\n\r\nMaybe this would require moving out the leaderboard into its own app, deprecating the results list view for a leaderboard detail view.\n", "before_files": [{"content": "from datetime import datetime, timedelta\nfrom typing import Dict\n\nfrom django.contrib.messages.views import SuccessMessageMixin\nfrom django.contrib.postgres.aggregates import ArrayAgg\nfrom django.core.exceptions import ObjectDoesNotExist\nfrom django.core.files import File\nfrom django.db.models import Q\nfrom django.utils import timezone\nfrom django.utils.functional import cached_property\nfrom django.views.generic import CreateView, DetailView, ListView, UpdateView\n\nfrom grandchallenge.core.permissions.mixins import (\n    UserIsChallengeAdminMixin,\n    UserIsChallengeParticipantOrAdminMixin,\n)\nfrom grandchallenge.core.views import Column, PaginatedTableListView\nfrom grandchallenge.evaluation.forms import (\n    ConfigForm,\n    LegacySubmissionForm,\n    MethodForm,\n    SubmissionForm,\n)\nfrom grandchallenge.evaluation.models import (\n    Config,\n    Evaluation,\n    Method,\n    Submission,\n)\nfrom grandchallenge.jqfileupload.widgets.uploader import StagedAjaxFile\nfrom grandchallenge.subdomains.utils import reverse\nfrom grandchallenge.teams.models import Team\n\n\nclass ConfigUpdate(UserIsChallengeAdminMixin, SuccessMessageMixin, UpdateView):\n    form_class = ConfigForm\n    success_message = \"Configuration successfully updated\"\n\n    def get_object(self, queryset=None):\n        challenge = self.request.challenge\n        return challenge.evaluation_config\n\n\nclass MethodCreate(UserIsChallengeAdminMixin, CreateView):\n    model = Method\n    form_class = MethodForm\n\n    def get_form_kwargs(self):\n        kwargs = super().get_form_kwargs()\n        kwargs.update({\"user\": self.request.user})\n        return kwargs\n\n    def form_valid(self, form):\n        form.instance.creator = self.request.user\n        form.instance.challenge = self.request.challenge\n\n        uploaded_file: StagedAjaxFile = form.cleaned_data[\"chunked_upload\"][0]\n        form.instance.staged_image_uuid = uploaded_file.uuid\n\n        return super().form_valid(form)\n\n\nclass MethodList(UserIsChallengeAdminMixin, ListView):\n    model = Method\n\n    def get_queryset(self):\n        queryset = super().get_queryset()\n        return queryset.filter(challenge=self.request.challenge)\n\n\nclass MethodDetail(UserIsChallengeAdminMixin, DetailView):\n    model = Method\n\n\nclass SubmissionCreateBase(SuccessMessageMixin, CreateView):\n    \"\"\"\n    Base class for the submission creation forms.\n\n    It has no permissions, do not use it directly! See the subclasses.\n    \"\"\"\n\n    model = Submission\n    success_message = (\n        \"Your submission was successful. \"\n        \"Your result will appear on the leaderboard when it is ready.\"\n    )\n\n    def get_form_kwargs(self):\n        kwargs = super().get_form_kwargs()\n\n        config: Config = Config.objects.get(challenge=self.request.challenge)\n\n        kwargs.update(\n            {\n                \"user\": self.request.user,\n                \"display_comment_field\": config.allow_submission_comments,\n                \"supplementary_file_choice\": config.supplementary_file_choice,\n                \"supplementary_file_label\": config.supplementary_file_label,\n                \"supplementary_file_help_text\": config.supplementary_file_help_text,\n                \"publication_url_choice\": config.publication_url_choice,\n                \"algorithm_submission\": config.submission_kind\n                == config.SubmissionKind.ALGORITHM,\n            }\n        )\n\n        return kwargs\n\n    def get_context_data(self, **kwargs):\n        context = super().get_context_data(**kwargs)\n\n        config = Config.objects.get(challenge=self.request.challenge)\n\n        context.update(\n            self.get_next_submission(max_subs=config.daily_submission_limit)\n        )\n\n        pending_evaluations = Evaluation.objects.filter(\n            submission__challenge=self.request.challenge,\n            submission__creator=self.request.user,\n            status__in=(Evaluation.PENDING, Evaluation.STARTED),\n        ).count()\n\n        context.update({\"pending_evaluations\": pending_evaluations})\n\n        return context\n\n    def get_next_submission(\n        self, *, max_subs: int, period: timedelta = None, now: datetime = None\n    ) -> Dict:\n        \"\"\"\n        Determines the number of submissions left for the user in a given time\n        period, and when they can next submit.\n\n        :return: A dictionary containing remaining_submissions (int) and\n        next_submission_at (datetime)\n        \"\"\"\n        if now is None:\n            now = timezone.now()\n\n        if period is None:\n            period = timedelta(days=1)\n\n        subs = (\n            Submission.objects.filter(\n                challenge=self.request.challenge,\n                creator=self.request.user,\n                created__gte=now - period,\n            )\n            .exclude(evaluation__status=Evaluation.FAILURE)\n            .order_by(\"-created\")\n            .distinct()\n        )\n\n        try:\n            next_sub_at = subs[max_subs - 1].created + period\n        except (IndexError, AssertionError):\n            next_sub_at = now\n\n        return {\n            \"remaining_submissions\": max_subs - len(subs),\n            \"next_submission_at\": next_sub_at,\n        }\n\n    def form_valid(self, form):\n        if form.instance.creator is None:\n            form.instance.creator = self.request.user\n\n        form.instance.challenge = self.request.challenge\n\n        if \"algorithm\" in form.cleaned_data:\n            # Algorithm submission\n            form.instance.algorithm_image = form.cleaned_data[\n                \"algorithm\"\n            ].latest_ready_image\n        else:\n            # Predictions file submission\n            uploaded_file = form.cleaned_data[\"chunked_upload\"][0]\n            with uploaded_file.open() as f:\n                form.instance.predictions_file.save(\n                    uploaded_file.name, File(f)\n                )\n\n        return super().form_valid(form)\n\n    def get_success_url(self):\n        return reverse(\n            \"evaluation:list\",\n            kwargs={\"challenge_short_name\": self.object.challenge.short_name},\n        )\n\n\nclass SubmissionCreate(\n    UserIsChallengeParticipantOrAdminMixin, SubmissionCreateBase\n):\n    form_class = SubmissionForm\n\n\nclass LegacySubmissionCreate(UserIsChallengeAdminMixin, SubmissionCreateBase):\n    form_class = LegacySubmissionForm\n\n\nclass SubmissionList(UserIsChallengeParticipantOrAdminMixin, ListView):\n    model = Submission\n\n    def get_queryset(self):\n        \"\"\"Admins see everything, participants just their submissions.\"\"\"\n        queryset = super().get_queryset()\n        challenge = self.request.challenge\n        if challenge.is_admin(self.request.user):\n            return queryset.filter(challenge=self.request.challenge)\n\n        else:\n            return queryset.filter(\n                Q(challenge=self.request.challenge),\n                Q(creator__pk=self.request.user.pk),\n            )\n\n\nclass SubmissionDetail(UserIsChallengeAdminMixin, DetailView):\n    # TODO - if participant: list only their submissions\n    model = Submission\n\n\nclass TeamContextMixin:\n    def get_context_data(self, *args, **kwargs):\n        context = super().get_context_data(*args, **kwargs)\n\n        evaluation_config = self.request.challenge.evaluation_config\n\n        if evaluation_config.use_teams:\n            user_teams = {\n                teammember.user.username: (team.name, team.get_absolute_url())\n                for team in Team.objects.filter(\n                    challenge=self.request.challenge\n                )\n                .select_related(\"challenge\")\n                .prefetch_related(\"teammember_set__user\")\n                for teammember in team.teammember_set.all()\n            }\n        else:\n            user_teams = {}\n\n        context.update(\n            {\"evaluation_config\": evaluation_config, \"user_teams\": user_teams}\n        )\n\n        return context\n\n\nclass EvaluationList(\n    UserIsChallengeParticipantOrAdminMixin, TeamContextMixin, ListView\n):\n    model = Evaluation\n\n    def get_queryset(self):\n        \"\"\"Admins see everything, participants just their evaluations.\"\"\"\n        challenge = self.request.challenge\n\n        queryset = super().get_queryset()\n        queryset = queryset.select_related(\n            \"submission__creator__user_profile\", \"submission__challenge\"\n        ).filter(submission__challenge=challenge)\n\n        if challenge.is_admin(self.request.user):\n            return queryset\n        else:\n            return queryset.filter(\n                Q(submission__creator__pk=self.request.user.pk)\n            )\n\n\nclass EvaluationDetail(DetailView):\n    # TODO - if participant: list only their evaluations\n    model = Evaluation\n\n    def get_context_data(self, **kwargs):\n        context = super().get_context_data(**kwargs)\n\n        try:\n            metrics = self.object.outputs.get(\n                interface__slug=\"metrics-json-file\"\n            ).value\n        except ObjectDoesNotExist:\n            metrics = None\n\n        context.update({\"metrics\": metrics})\n\n        return context\n\n\nclass LeaderboardDetail(TeamContextMixin, PaginatedTableListView):\n    model = Evaluation\n    template_name = \"evaluation/leaderboard_detail.html\"\n    row_template = \"evaluation/leaderboard_row.html\"\n    search_fields = [\"pk\", \"submission__creator__username\"]\n\n    @property\n    def columns(self):\n        columns = [\n            Column(title=\"#\", sort_field=\"rank\"),\n            Column(\n                title=\"User (Team)\" if self.config.use_teams else \"User\",\n                sort_field=\"submission__creator__username\",\n            ),\n            Column(title=\"Created\", sort_field=\"created\"),\n        ]\n\n        if self.config.scoring_method_choice == self.config.MEAN:\n            columns.append(Column(title=\"Mean Position\", sort_field=\"rank\"))\n        elif self.config.scoring_method_choice == self.config.MEDIAN:\n            columns.append(Column(title=\"Median Position\", sort_field=\"rank\"))\n\n        if self.config.scoring_method_choice == self.config.ABSOLUTE:\n            columns.append(\n                Column(title=self.config.score_title, sort_field=\"rank\")\n            )\n        else:\n            columns.append(\n                Column(\n                    title=f\"{self.config.score_title} (Position)\",\n                    sort_field=\"rank\",\n                    toggleable=True,\n                )\n            )\n\n        for c in self.config.extra_results_columns:\n            columns.append(\n                Column(\n                    title=c[\"title\"]\n                    if self.config.scoring_method_choice\n                    == self.config.ABSOLUTE\n                    else f\"{c['title']} (Position)\",\n                    sort_field=\"rank\",\n                    toggleable=True,\n                )\n            )\n\n        if self.config.display_submission_comments:\n            columns.append(\n                Column(title=\"Comment\", sort_field=\"submission__comment\")\n            )\n\n        if self.config.show_publication_url:\n            columns.append(\n                Column(\n                    title=\"Publication\",\n                    sort_field=\"submission__publication_url\",\n                )\n            )\n\n        if self.config.show_supplementary_file_link:\n            columns.append(\n                Column(\n                    title=self.config.supplementary_file_label,\n                    sort_field=\"submission__supplementary_file\",\n                )\n            )\n\n        return columns\n\n    @cached_property\n    def config(self):\n        return self.request.challenge.evaluation_config\n\n    def get_row_context(self, job, *args, **kwargs):\n        return {\"evaluation\": job, \"evaluation_config\": self.config}\n\n    def get_unfiltered_queryset(self):\n        queryset = super().get_queryset()\n        queryset = (\n            queryset.select_related(\n                \"submission__creator__user_profile\", \"submission__challenge\"\n            )\n            .filter(\n                submission__challenge=self.request.challenge,\n                published=True,\n                status=Evaluation.SUCCESS,\n                rank__gt=0,\n            )\n            .annotate(\n                metrics=ArrayAgg(\n                    \"outputs__value\",\n                    filter=Q(outputs__interface__slug=\"metrics-json-file\"),\n                )\n            )\n        )\n        return queryset\n\n\nclass EvaluationUpdate(\n    UserIsChallengeAdminMixin, SuccessMessageMixin, UpdateView\n):\n    model = Evaluation\n    fields = (\"published\",)\n    success_message = \"Result successfully updated.\"\n", "path": "app/grandchallenge/evaluation/views.py"}], "after_files": [{"content": "from datetime import datetime, timedelta\nfrom typing import Dict\n\nfrom dateutil.relativedelta import relativedelta\nfrom django.contrib.messages.views import SuccessMessageMixin\nfrom django.contrib.postgres.aggregates import ArrayAgg\nfrom django.core.exceptions import ObjectDoesNotExist\nfrom django.core.files import File\nfrom django.db.models import Q\nfrom django.utils import timezone\nfrom django.utils.functional import cached_property\nfrom django.views.generic import CreateView, DetailView, ListView, UpdateView\n\nfrom grandchallenge.core.permissions.mixins import (\n    UserIsChallengeAdminMixin,\n    UserIsChallengeParticipantOrAdminMixin,\n)\nfrom grandchallenge.core.views import Column, PaginatedTableListView\nfrom grandchallenge.evaluation.forms import (\n    ConfigForm,\n    LegacySubmissionForm,\n    MethodForm,\n    SubmissionForm,\n)\nfrom grandchallenge.evaluation.models import (\n    Config,\n    Evaluation,\n    Method,\n    Submission,\n)\nfrom grandchallenge.jqfileupload.widgets.uploader import StagedAjaxFile\nfrom grandchallenge.subdomains.utils import reverse\nfrom grandchallenge.teams.models import Team\n\n\nclass ConfigUpdate(UserIsChallengeAdminMixin, SuccessMessageMixin, UpdateView):\n    form_class = ConfigForm\n    success_message = \"Configuration successfully updated\"\n\n    def get_object(self, queryset=None):\n        challenge = self.request.challenge\n        return challenge.evaluation_config\n\n\nclass MethodCreate(UserIsChallengeAdminMixin, CreateView):\n    model = Method\n    form_class = MethodForm\n\n    def get_form_kwargs(self):\n        kwargs = super().get_form_kwargs()\n        kwargs.update({\"user\": self.request.user})\n        return kwargs\n\n    def form_valid(self, form):\n        form.instance.creator = self.request.user\n        form.instance.challenge = self.request.challenge\n\n        uploaded_file: StagedAjaxFile = form.cleaned_data[\"chunked_upload\"][0]\n        form.instance.staged_image_uuid = uploaded_file.uuid\n\n        return super().form_valid(form)\n\n\nclass MethodList(UserIsChallengeAdminMixin, ListView):\n    model = Method\n\n    def get_queryset(self):\n        queryset = super().get_queryset()\n        return queryset.filter(challenge=self.request.challenge)\n\n\nclass MethodDetail(UserIsChallengeAdminMixin, DetailView):\n    model = Method\n\n\nclass SubmissionCreateBase(SuccessMessageMixin, CreateView):\n    \"\"\"\n    Base class for the submission creation forms.\n\n    It has no permissions, do not use it directly! See the subclasses.\n    \"\"\"\n\n    model = Submission\n    success_message = (\n        \"Your submission was successful. \"\n        \"Your result will appear on the leaderboard when it is ready.\"\n    )\n\n    def get_form_kwargs(self):\n        kwargs = super().get_form_kwargs()\n\n        config: Config = Config.objects.get(challenge=self.request.challenge)\n\n        kwargs.update(\n            {\n                \"user\": self.request.user,\n                \"display_comment_field\": config.allow_submission_comments,\n                \"supplementary_file_choice\": config.supplementary_file_choice,\n                \"supplementary_file_label\": config.supplementary_file_label,\n                \"supplementary_file_help_text\": config.supplementary_file_help_text,\n                \"publication_url_choice\": config.publication_url_choice,\n                \"algorithm_submission\": config.submission_kind\n                == config.SubmissionKind.ALGORITHM,\n            }\n        )\n\n        return kwargs\n\n    def get_context_data(self, **kwargs):\n        context = super().get_context_data(**kwargs)\n\n        config = Config.objects.get(challenge=self.request.challenge)\n\n        context.update(\n            self.get_next_submission(max_subs=config.daily_submission_limit)\n        )\n\n        pending_evaluations = Evaluation.objects.filter(\n            submission__challenge=self.request.challenge,\n            submission__creator=self.request.user,\n            status__in=(Evaluation.PENDING, Evaluation.STARTED),\n        ).count()\n\n        context.update({\"pending_evaluations\": pending_evaluations})\n\n        return context\n\n    def get_next_submission(\n        self, *, max_subs: int, period: timedelta = None, now: datetime = None\n    ) -> Dict:\n        \"\"\"\n        Determines the number of submissions left for the user in a given time\n        period, and when they can next submit.\n\n        :return: A dictionary containing remaining_submissions (int) and\n        next_submission_at (datetime)\n        \"\"\"\n        if now is None:\n            now = timezone.now()\n\n        if period is None:\n            period = timedelta(days=1)\n\n        subs = (\n            Submission.objects.filter(\n                challenge=self.request.challenge,\n                creator=self.request.user,\n                created__gte=now - period,\n            )\n            .exclude(evaluation__status=Evaluation.FAILURE)\n            .order_by(\"-created\")\n            .distinct()\n        )\n\n        try:\n            next_sub_at = subs[max_subs - 1].created + period\n        except (IndexError, AssertionError):\n            next_sub_at = now\n\n        return {\n            \"remaining_submissions\": max_subs - len(subs),\n            \"next_submission_at\": next_sub_at,\n        }\n\n    def form_valid(self, form):\n        if form.instance.creator is None:\n            form.instance.creator = self.request.user\n\n        form.instance.challenge = self.request.challenge\n\n        if \"algorithm\" in form.cleaned_data:\n            # Algorithm submission\n            form.instance.algorithm_image = form.cleaned_data[\n                \"algorithm\"\n            ].latest_ready_image\n        else:\n            # Predictions file submission\n            uploaded_file = form.cleaned_data[\"chunked_upload\"][0]\n            with uploaded_file.open() as f:\n                form.instance.predictions_file.save(\n                    uploaded_file.name, File(f)\n                )\n\n        return super().form_valid(form)\n\n    def get_success_url(self):\n        return reverse(\n            \"evaluation:list\",\n            kwargs={\"challenge_short_name\": self.object.challenge.short_name},\n        )\n\n\nclass SubmissionCreate(\n    UserIsChallengeParticipantOrAdminMixin, SubmissionCreateBase\n):\n    form_class = SubmissionForm\n\n\nclass LegacySubmissionCreate(UserIsChallengeAdminMixin, SubmissionCreateBase):\n    form_class = LegacySubmissionForm\n\n\nclass SubmissionList(UserIsChallengeParticipantOrAdminMixin, ListView):\n    model = Submission\n\n    def get_queryset(self):\n        \"\"\"Admins see everything, participants just their submissions.\"\"\"\n        queryset = super().get_queryset()\n        challenge = self.request.challenge\n        if challenge.is_admin(self.request.user):\n            return queryset.filter(challenge=self.request.challenge)\n\n        else:\n            return queryset.filter(\n                Q(challenge=self.request.challenge),\n                Q(creator__pk=self.request.user.pk),\n            )\n\n\nclass SubmissionDetail(UserIsChallengeAdminMixin, DetailView):\n    # TODO - if participant: list only their submissions\n    model = Submission\n\n\nclass TeamContextMixin:\n    def get_context_data(self, *args, **kwargs):\n        context = super().get_context_data(*args, **kwargs)\n\n        evaluation_config = self.request.challenge.evaluation_config\n\n        if evaluation_config.use_teams:\n            user_teams = {\n                teammember.user.username: (team.name, team.get_absolute_url())\n                for team in Team.objects.filter(\n                    challenge=self.request.challenge\n                )\n                .select_related(\"challenge\")\n                .prefetch_related(\"teammember_set__user\")\n                for teammember in team.teammember_set.all()\n            }\n        else:\n            user_teams = {}\n\n        context.update(\n            {\"evaluation_config\": evaluation_config, \"user_teams\": user_teams}\n        )\n\n        return context\n\n\nclass EvaluationList(\n    UserIsChallengeParticipantOrAdminMixin, TeamContextMixin, ListView\n):\n    model = Evaluation\n\n    def get_queryset(self):\n        \"\"\"Admins see everything, participants just their evaluations.\"\"\"\n        challenge = self.request.challenge\n\n        queryset = super().get_queryset()\n        queryset = queryset.select_related(\n            \"submission__creator__user_profile\", \"submission__challenge\"\n        ).filter(submission__challenge=challenge)\n\n        if challenge.is_admin(self.request.user):\n            return queryset\n        else:\n            return queryset.filter(\n                Q(submission__creator__pk=self.request.user.pk)\n            )\n\n\nclass EvaluationDetail(DetailView):\n    # TODO - if participant: list only their evaluations\n    model = Evaluation\n\n    def get_context_data(self, **kwargs):\n        context = super().get_context_data(**kwargs)\n\n        try:\n            metrics = self.object.outputs.get(\n                interface__slug=\"metrics-json-file\"\n            ).value\n        except ObjectDoesNotExist:\n            metrics = None\n\n        context.update({\"metrics\": metrics})\n\n        return context\n\n\nclass LeaderboardDetail(TeamContextMixin, PaginatedTableListView):\n    model = Evaluation\n    template_name = \"evaluation/leaderboard_detail.html\"\n    row_template = \"evaluation/leaderboard_row.html\"\n    search_fields = [\"pk\", \"submission__creator__username\"]\n\n    @property\n    def columns(self):\n        columns = [\n            Column(title=\"#\", sort_field=\"rank\"),\n            Column(\n                title=\"User (Team)\" if self.config.use_teams else \"User\",\n                sort_field=\"submission__creator__username\",\n            ),\n            Column(title=\"Created\", sort_field=\"created\"),\n        ]\n\n        if self.config.scoring_method_choice == self.config.MEAN:\n            columns.append(Column(title=\"Mean Position\", sort_field=\"rank\"))\n        elif self.config.scoring_method_choice == self.config.MEDIAN:\n            columns.append(Column(title=\"Median Position\", sort_field=\"rank\"))\n\n        if self.config.scoring_method_choice == self.config.ABSOLUTE:\n            columns.append(\n                Column(title=self.config.score_title, sort_field=\"rank\")\n            )\n        else:\n            columns.append(\n                Column(\n                    title=f\"{self.config.score_title} (Position)\",\n                    sort_field=\"rank\",\n                    toggleable=True,\n                )\n            )\n\n        for c in self.config.extra_results_columns:\n            columns.append(\n                Column(\n                    title=c[\"title\"]\n                    if self.config.scoring_method_choice\n                    == self.config.ABSOLUTE\n                    else f\"{c['title']} (Position)\",\n                    sort_field=\"rank\",\n                    toggleable=True,\n                )\n            )\n\n        if self.config.display_submission_comments:\n            columns.append(\n                Column(title=\"Comment\", sort_field=\"submission__comment\")\n            )\n\n        if self.config.show_publication_url:\n            columns.append(\n                Column(\n                    title=\"Publication\",\n                    sort_field=\"submission__publication_url\",\n                )\n            )\n\n        if self.config.show_supplementary_file_link:\n            columns.append(\n                Column(\n                    title=self.config.supplementary_file_label,\n                    sort_field=\"submission__supplementary_file\",\n                )\n            )\n\n        return columns\n\n    @cached_property\n    def config(self):\n        return self.request.challenge.evaluation_config\n\n    def get_row_context(self, job, *args, **kwargs):\n        return {\"evaluation\": job, \"evaluation_config\": self.config}\n\n    def get_unfiltered_queryset(self):\n        queryset = super().get_queryset()\n        queryset = self.filter_by_date(queryset=queryset)\n        queryset = (\n            queryset.select_related(\n                \"submission__creator__user_profile\", \"submission__challenge\"\n            )\n            .filter(\n                submission__challenge=self.request.challenge,\n                published=True,\n                status=Evaluation.SUCCESS,\n                rank__gt=0,\n            )\n            .annotate(\n                metrics=ArrayAgg(\n                    \"outputs__value\",\n                    filter=Q(outputs__interface__slug=\"metrics-json-file\"),\n                )\n            )\n        )\n        return queryset\n\n    def filter_by_date(self, queryset):\n        if \"leaderboardDate\" in self.request.GET:\n            year, month, day = self.request.GET[\"leaderboardDate\"].split(\"-\")\n            before = datetime(\n                year=int(year), month=int(month), day=int(day),\n            ) + relativedelta(days=1)\n            return queryset.filter(submission__created__lt=before)\n        else:\n            return queryset\n\n\nclass EvaluationUpdate(\n    UserIsChallengeAdminMixin, SuccessMessageMixin, UpdateView\n):\n    model = Evaluation\n    fields = (\"published\",)\n    success_message = \"Result successfully updated.\"\n", "path": "app/grandchallenge/evaluation/views.py"}]}