We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Move `wait_for_build_completion` to acks late micro short queue and use delays
`wait_for_build_completion` uses a countdown that causes the same issues described in https://github.com/comic/grand-challenge.org/blob/ce130d451034377b0f3dfd83590268e48a66d518/app/grandchallenge/components/tasks.py#L255-L271. 

This task is not resource-intensive so:
- If the task is not idempotent it should be made so
- It should be run on the acks-late-micro-short queue
- Retries should use the same strategy as https://github.com/comic/grand-challenge.org/blob/ce130d451034377b0f3dfd83590268e48a66d518/app/grandchallenge/components/tasks.py#L323-L327

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `app/grandchallenge/codebuild/models.py`
Content:
```
1 import gzip
2 from tempfile import NamedTemporaryFile
3 
4 import boto3
5 from django.conf import settings
6 from django.core import files
7 from django.db import models
8 from django.utils.translation import gettext_lazy as _
9 
10 from grandchallenge.algorithms.models import AlgorithmImage
11 from grandchallenge.core.models import UUIDModel
12 from grandchallenge.core.storage import private_s3_storage
13 from grandchallenge.github.models import GitHubWebhookMessage
14 
15 
16 class BuildStatusChoices(models.TextChoices):
17     """From https://docs.aws.amazon.com/codebuild/latest/APIReference/API_Build.html"""
18 
19     SUCCEEDED = "SUCCEEDED", _("Succeeded")
20     FAILED = "FAILED", _("Failed")
21     FAULT = "FAULT", _("Fault")
22     TIMED_OUT = "TIMED_OUT", _("Timed Out")
23     IN_PROGRESS = "IN_PROGRESS", _("In Progress")
24     STOPPED = "STOPPED", _("Stopped")
25 
26 
27 class Build(UUIDModel):
28     webhook_message = models.ForeignKey(
29         GitHubWebhookMessage, on_delete=models.SET_NULL, null=True
30     )
31     algorithm_image = models.OneToOneField(
32         AlgorithmImage, on_delete=models.SET_NULL, null=True
33     )
34     build_config = models.JSONField()
35     build_id = models.CharField(max_length=1024)
36     status = models.CharField(
37         choices=BuildStatusChoices.choices, max_length=11
38     )
39     build_log = models.TextField(blank=True)
40 
41     BuildStatusChoices = BuildStatusChoices
42     __client = None
43 
44     @property
45     def client(self):
46         if self.__client is None:
47             self.__client = boto3.client("codebuild")
48         return self.__client
49 
50     @property
51     def build_number(self):
52         return self.build_id.split(":")[-1]
53 
54     def refresh_status(self):
55         build_statuses = self.client.batch_get_builds(ids=[self.build_id])
56         self.status = build_statuses["builds"][0]["buildStatus"]
57 
58     def refresh_logs(self):
59         try:
60             with private_s3_storage.open(
61                 f"codebuild/logs/{self.build_number}.gz"
62             ) as file:
63                 self.build_log = gzip.open(file).read().decode("utf-8")
64         except FileNotFoundError:
65             self.build_log = "Log file not available."
66 
67     def add_image_to_algorithm(self):
68         with private_s3_storage.open(
69             f"codebuild/artifacts/{self.build_number}/{self.build_config['projectName']}/container-image.tar.gz"
70         ) as file:
71             with NamedTemporaryFile(delete=True) as tmp_file:
72                 with open(tmp_file.name, "wb") as fd:
73                     for chunk in file.chunks():
74                         fd.write(chunk)
75 
76                 tmp_file.flush()
77                 temp_file = files.File(
78                     tmp_file, name=f"{str(self.pk)}.tar.gz",
79                 )
80 
81                 self.algorithm_image.image = temp_file
82                 self.algorithm_image.save()
83 
84     def _create_build(self):
85         self.build_config = {
86             "projectName": settings.CODEBUILD_PROJECT_NAME,
87             "sourceLocationOverride": f"{settings.PRIVATE_S3_STORAGE_KWARGS['bucket_name']}/{self.webhook_message.zipfile.name}",
88             "sourceTypeOverride": "S3",
89             "environmentVariablesOverride": [
90                 {
91                     "name": "IMAGE_REPO_NAME",
92                     "value": f"{AlgorithmImage._meta.app_label}/{AlgorithmImage._meta.model_name}",
93                 },
94                 {"name": "IMAGE_TAG", "value": str(self.algorithm_image.pk)},
95             ],
96         }
97 
98         build_data = self.client.start_build(**self.build_config)
99 
100         self.build_id = build_data["build"]["id"]
101         self.status = build_data["build"]["buildStatus"]
102 
103     def save(self, *args, **kwargs):
104         if self._state.adding:
105             self._create_build()
106 
107         super().save(*args, **kwargs)
108 
109     @property
110     def animate(self):
111         return self.status == BuildStatusChoices.IN_PROGRESS
112 
113     @property
114     def status_context(self):
115         if self.status == BuildStatusChoices.SUCCEEDED:
116             return "success"
117         elif self.status in {BuildStatusChoices.STOPPED}:
118             return "warning"
119         elif self.status in {
120             BuildStatusChoices.FAILED,
121             BuildStatusChoices.FAULT,
122             BuildStatusChoices.TIMED_OUT,
123         }:
124             return "danger"
125         elif self.status in {BuildStatusChoices.IN_PROGRESS}:
126             return "info"
127         else:
128             return "secondary"
129 
130     class Meta:
131         indexes = [
132             models.Index(fields=["build_id"]),
133         ]
134
```

Path: `app/grandchallenge/codebuild/tasks.py`
Content:
```
1 from celery import shared_task
2 from django.apps import apps
3 from django.core.exceptions import ObjectDoesNotExist
4 from django.db.transaction import on_commit
5 
6 from grandchallenge.algorithms.models import Algorithm, AlgorithmImage
7 
8 
9 @shared_task()
10 def create_codebuild_build(*, pk):
11     GitHubWebhookMessage = apps.get_model(  # noqa: N806
12         app_label="github", model_name="GitHubWebhookMessage"
13     )
14     ghwm = GitHubWebhookMessage.objects.get(pk=pk)
15 
16     Build = apps.get_model(  # noqa: N806
17         app_label="codebuild", model_name="Build"
18     )
19 
20     if Build.objects.filter(webhook_message=ghwm).exists():
21         # Build already exists
22         return
23 
24     try:
25         algorithm = Algorithm.objects.get(
26             repo_name=ghwm.payload["repository"]["full_name"]
27         )
28     except ObjectDoesNotExist:
29         # Repository is not linked to algorithm
30         return
31 
32     algorithm_image = AlgorithmImage.objects.create(
33         algorithm=algorithm,
34         requires_gpu=algorithm.image_requires_gpu,
35         requires_memory_gb=algorithm.image_requires_memory_gb,
36     )
37 
38     build = Build.objects.create(
39         webhook_message=ghwm, algorithm_image=algorithm_image,
40     )
41 
42     on_commit(
43         lambda: wait_for_build_completion.apply_async(
44             kwargs={"build_pk": str(build.pk)}
45         )
46     )
47 
48 
49 @shared_task(bind=True, max_retries=120)
50 def wait_for_build_completion(self, *, build_pk):
51     Build = apps.get_model(  # noqa: N806
52         app_label="codebuild", model_name="Build"
53     )
54 
55     build = Build.objects.get(pk=build_pk)
56     build.refresh_status()
57 
58     if build.status == build.BuildStatusChoices.IN_PROGRESS:
59         self.retry(countdown=30)
60     else:
61         build.refresh_logs()
62         build.save()
63         if build.status == build.BuildStatusChoices.SUCCEEDED:
64             on_commit(
65                 lambda: add_image_to_algorithm.apply_async(
66                     kwargs={"build_pk": str(build_pk)}
67                 )
68             )
69 
70 
71 @shared_task
72 def add_image_to_algorithm(*, build_pk):
73     Build = apps.get_model(  # noqa: N806
74         app_label="codebuild", model_name="Build"
75     )
76 
77     build = Build.objects.get(pk=build_pk)
78     build.add_image_to_algorithm()
79
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/app/grandchallenge/codebuild/models.py b/app/grandchallenge/codebuild/models.py
--- a/app/grandchallenge/codebuild/models.py
+++ b/app/grandchallenge/codebuild/models.py
@@ -65,6 +65,7 @@
             self.build_log = "Log file not available."
 
     def add_image_to_algorithm(self):
+        # TODO, this would be much faster using S3 copy, can then run on a smaller queue
         with private_s3_storage.open(
             f"codebuild/artifacts/{self.build_number}/{self.build_config['projectName']}/container-image.tar.gz"
         ) as file:
diff --git a/app/grandchallenge/codebuild/tasks.py b/app/grandchallenge/codebuild/tasks.py
--- a/app/grandchallenge/codebuild/tasks.py
+++ b/app/grandchallenge/codebuild/tasks.py
@@ -1,12 +1,15 @@
 from celery import shared_task
 from django.apps import apps
+from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
+from django.db import transaction
 from django.db.transaction import on_commit
 
 from grandchallenge.algorithms.models import Algorithm, AlgorithmImage
+from grandchallenge.components.tasks import _retry
 
 
-@shared_task()
+@shared_task(**settings.CELERY_TASK_DECORATOR_KWARGS["acks-late-micro-short"])
 def create_codebuild_build(*, pk):
     GitHubWebhookMessage = apps.get_model(  # noqa: N806
         app_label="github", model_name="GitHubWebhookMessage"
@@ -29,50 +32,59 @@
         # Repository is not linked to algorithm
         return
 
-    algorithm_image = AlgorithmImage.objects.create(
-        algorithm=algorithm,
-        requires_gpu=algorithm.image_requires_gpu,
-        requires_memory_gb=algorithm.image_requires_memory_gb,
-    )
-
-    build = Build.objects.create(
-        webhook_message=ghwm, algorithm_image=algorithm_image,
-    )
+    with transaction.atomic():
+        algorithm_image = AlgorithmImage.objects.create(
+            algorithm=algorithm,
+            requires_gpu=algorithm.image_requires_gpu,
+            requires_memory_gb=algorithm.image_requires_memory_gb,
+        )
+        build = Build.objects.create(
+            webhook_message=ghwm, algorithm_image=algorithm_image,
+        )
 
-    on_commit(
-        lambda: wait_for_build_completion.apply_async(
-            kwargs={"build_pk": str(build.pk)}
+        on_commit(
+            lambda: wait_for_build_completion.apply_async(
+                kwargs={"build_pk": str(build.pk)}
+            )
         )
-    )
 
 
-@shared_task(bind=True, max_retries=120)
-def wait_for_build_completion(self, *, build_pk):
+@shared_task(**settings.CELERY_TASK_DECORATOR_KWARGS["acks-late-micro-short"])
+def wait_for_build_completion(*, build_pk, retries=0):
     Build = apps.get_model(  # noqa: N806
         app_label="codebuild", model_name="Build"
     )
 
     build = Build.objects.get(pk=build_pk)
-    build.refresh_status()
-
-    if build.status == build.BuildStatusChoices.IN_PROGRESS:
-        self.retry(countdown=30)
-    else:
-        build.refresh_logs()
-        build.save()
-        if build.status == build.BuildStatusChoices.SUCCEEDED:
-            on_commit(
-                lambda: add_image_to_algorithm.apply_async(
-                    kwargs={"build_pk": str(build_pk)}
-                )
+
+    with transaction.atomic():
+        build.refresh_status()
+
+        if build.status == build.BuildStatusChoices.IN_PROGRESS:
+            _retry(
+                task=wait_for_build_completion,
+                signature_kwargs={"kwargs": {"build_pk": build_pk}},
+                retries=retries,
             )
+            return
+        else:
+            build.refresh_logs()
+            build.save()
+            if build.status == build.BuildStatusChoices.SUCCEEDED:
+                on_commit(
+                    lambda: add_image_to_algorithm.apply_async(
+                        kwargs={"build_pk": str(build_pk)}
+                    )
+                )
 
 
-@shared_task
+@shared_task(**settings.CELERY_TASK_DECORATOR_KWARGS["acks-late-2xlarge"])
 def add_image_to_algorithm(*, build_pk):
     Build = apps.get_model(  # noqa: N806
         app_label="codebuild", model_name="Build"
     )
 
     build = Build.objects.get(pk=build_pk)
-    build.add_image_to_algorithm()
+
+    if not build.algorithm_image.image:
+        build.add_image_to_algorithm()

{"golden_diff": "diff --git a/app/grandchallenge/codebuild/models.py b/app/grandchallenge/codebuild/models.py\n--- a/app/grandchallenge/codebuild/models.py\n+++ b/app/grandchallenge/codebuild/models.py\n@@ -65,6 +65,7 @@\n             self.build_log = \"Log file not available.\"\n \n     def add_image_to_algorithm(self):\n+        # TODO, this would be much faster using S3 copy, can then run on a smaller queue\n         with private_s3_storage.open(\n             f\"codebuild/artifacts/{self.build_number}/{self.build_config['projectName']}/container-image.tar.gz\"\n         ) as file:\ndiff --git a/app/grandchallenge/codebuild/tasks.py b/app/grandchallenge/codebuild/tasks.py\n--- a/app/grandchallenge/codebuild/tasks.py\n+++ b/app/grandchallenge/codebuild/tasks.py\n@@ -1,12 +1,15 @@\n from celery import shared_task\n from django.apps import apps\n+from django.conf import settings\n from django.core.exceptions import ObjectDoesNotExist\n+from django.db import transaction\n from django.db.transaction import on_commit\n \n from grandchallenge.algorithms.models import Algorithm, AlgorithmImage\n+from grandchallenge.components.tasks import _retry\n \n \n-@shared_task()\n+@shared_task(**settings.CELERY_TASK_DECORATOR_KWARGS[\"acks-late-micro-short\"])\n def create_codebuild_build(*, pk):\n     GitHubWebhookMessage = apps.get_model(  # noqa: N806\n         app_label=\"github\", model_name=\"GitHubWebhookMessage\"\n@@ -29,50 +32,59 @@\n         # Repository is not linked to algorithm\n         return\n \n-    algorithm_image = AlgorithmImage.objects.create(\n-        algorithm=algorithm,\n-        requires_gpu=algorithm.image_requires_gpu,\n-        requires_memory_gb=algorithm.image_requires_memory_gb,\n-    )\n-\n-    build = Build.objects.create(\n-        webhook_message=ghwm, algorithm_image=algorithm_image,\n-    )\n+    with transaction.atomic():\n+        algorithm_image = AlgorithmImage.objects.create(\n+            algorithm=algorithm,\n+            requires_gpu=algorithm.image_requires_gpu,\n+            requires_memory_gb=algorithm.image_requires_memory_gb,\n+        )\n+        build = Build.objects.create(\n+            webhook_message=ghwm, algorithm_image=algorithm_image,\n+        )\n \n-    on_commit(\n-        lambda: wait_for_build_completion.apply_async(\n-            kwargs={\"build_pk\": str(build.pk)}\n+        on_commit(\n+            lambda: wait_for_build_completion.apply_async(\n+                kwargs={\"build_pk\": str(build.pk)}\n+            )\n         )\n-    )\n \n \n-@shared_task(bind=True, max_retries=120)\n-def wait_for_build_completion(self, *, build_pk):\n+@shared_task(**settings.CELERY_TASK_DECORATOR_KWARGS[\"acks-late-micro-short\"])\n+def wait_for_build_completion(*, build_pk, retries=0):\n     Build = apps.get_model(  # noqa: N806\n         app_label=\"codebuild\", model_name=\"Build\"\n     )\n \n     build = Build.objects.get(pk=build_pk)\n-    build.refresh_status()\n-\n-    if build.status == build.BuildStatusChoices.IN_PROGRESS:\n-        self.retry(countdown=30)\n-    else:\n-        build.refresh_logs()\n-        build.save()\n-        if build.status == build.BuildStatusChoices.SUCCEEDED:\n-            on_commit(\n-                lambda: add_image_to_algorithm.apply_async(\n-                    kwargs={\"build_pk\": str(build_pk)}\n-                )\n+\n+    with transaction.atomic():\n+        build.refresh_status()\n+\n+        if build.status == build.BuildStatusChoices.IN_PROGRESS:\n+            _retry(\n+                task=wait_for_build_completion,\n+                signature_kwargs={\"kwargs\": {\"build_pk\": build_pk}},\n+                retries=retries,\n             )\n+            return\n+        else:\n+            build.refresh_logs()\n+            build.save()\n+            if build.status == build.BuildStatusChoices.SUCCEEDED:\n+                on_commit(\n+                    lambda: add_image_to_algorithm.apply_async(\n+                        kwargs={\"build_pk\": str(build_pk)}\n+                    )\n+                )\n \n \n-@shared_task\n+@shared_task(**settings.CELERY_TASK_DECORATOR_KWARGS[\"acks-late-2xlarge\"])\n def add_image_to_algorithm(*, build_pk):\n     Build = apps.get_model(  # noqa: N806\n         app_label=\"codebuild\", model_name=\"Build\"\n     )\n \n     build = Build.objects.get(pk=build_pk)\n-    build.add_image_to_algorithm()\n+\n+    if not build.algorithm_image.image:\n+        build.add_image_to_algorithm()\n", "issue": "Move `wait_for_build_completion` to acks late micro short queue and use delays\n`wait_for_build_completion` uses a countdown that causes the same issues described in https://github.com/comic/grand-challenge.org/blob/ce130d451034377b0f3dfd83590268e48a66d518/app/grandchallenge/components/tasks.py#L255-L271. \r\n\r\nThis task is not resource-intensive so:\r\n- If the task is not idempotent it should be made so\r\n- It should be run on the acks-late-micro-short queue\r\n- Retries should use the same strategy as https://github.com/comic/grand-challenge.org/blob/ce130d451034377b0f3dfd83590268e48a66d518/app/grandchallenge/components/tasks.py#L323-L327\n", "before_files": [{"content": "import gzip\nfrom tempfile import NamedTemporaryFile\n\nimport boto3\nfrom django.conf import settings\nfrom django.core import files\nfrom django.db import models\nfrom django.utils.translation import gettext_lazy as _\n\nfrom grandchallenge.algorithms.models import AlgorithmImage\nfrom grandchallenge.core.models import UUIDModel\nfrom grandchallenge.core.storage import private_s3_storage\nfrom grandchallenge.github.models import GitHubWebhookMessage\n\n\nclass BuildStatusChoices(models.TextChoices):\n    \"\"\"From https://docs.aws.amazon.com/codebuild/latest/APIReference/API_Build.html\"\"\"\n\n    SUCCEEDED = \"SUCCEEDED\", _(\"Succeeded\")\n    FAILED = \"FAILED\", _(\"Failed\")\n    FAULT = \"FAULT\", _(\"Fault\")\n    TIMED_OUT = \"TIMED_OUT\", _(\"Timed Out\")\n    IN_PROGRESS = \"IN_PROGRESS\", _(\"In Progress\")\n    STOPPED = \"STOPPED\", _(\"Stopped\")\n\n\nclass Build(UUIDModel):\n    webhook_message = models.ForeignKey(\n        GitHubWebhookMessage, on_delete=models.SET_NULL, null=True\n    )\n    algorithm_image = models.OneToOneField(\n        AlgorithmImage, on_delete=models.SET_NULL, null=True\n    )\n    build_config = models.JSONField()\n    build_id = models.CharField(max_length=1024)\n    status = models.CharField(\n        choices=BuildStatusChoices.choices, max_length=11\n    )\n    build_log = models.TextField(blank=True)\n\n    BuildStatusChoices = BuildStatusChoices\n    __client = None\n\n    @property\n    def client(self):\n        if self.__client is None:\n            self.__client = boto3.client(\"codebuild\")\n        return self.__client\n\n    @property\n    def build_number(self):\n        return self.build_id.split(\":\")[-1]\n\n    def refresh_status(self):\n        build_statuses = self.client.batch_get_builds(ids=[self.build_id])\n        self.status = build_statuses[\"builds\"][0][\"buildStatus\"]\n\n    def refresh_logs(self):\n        try:\n            with private_s3_storage.open(\n                f\"codebuild/logs/{self.build_number}.gz\"\n            ) as file:\n                self.build_log = gzip.open(file).read().decode(\"utf-8\")\n        except FileNotFoundError:\n            self.build_log = \"Log file not available.\"\n\n    def add_image_to_algorithm(self):\n        with private_s3_storage.open(\n            f\"codebuild/artifacts/{self.build_number}/{self.build_config['projectName']}/container-image.tar.gz\"\n        ) as file:\n            with NamedTemporaryFile(delete=True) as tmp_file:\n                with open(tmp_file.name, \"wb\") as fd:\n                    for chunk in file.chunks():\n                        fd.write(chunk)\n\n                tmp_file.flush()\n                temp_file = files.File(\n                    tmp_file, name=f\"{str(self.pk)}.tar.gz\",\n                )\n\n                self.algorithm_image.image = temp_file\n                self.algorithm_image.save()\n\n    def _create_build(self):\n        self.build_config = {\n            \"projectName\": settings.CODEBUILD_PROJECT_NAME,\n            \"sourceLocationOverride\": f\"{settings.PRIVATE_S3_STORAGE_KWARGS['bucket_name']}/{self.webhook_message.zipfile.name}\",\n            \"sourceTypeOverride\": \"S3\",\n            \"environmentVariablesOverride\": [\n                {\n                    \"name\": \"IMAGE_REPO_NAME\",\n                    \"value\": f\"{AlgorithmImage._meta.app_label}/{AlgorithmImage._meta.model_name}\",\n                },\n                {\"name\": \"IMAGE_TAG\", \"value\": str(self.algorithm_image.pk)},\n            ],\n        }\n\n        build_data = self.client.start_build(**self.build_config)\n\n        self.build_id = build_data[\"build\"][\"id\"]\n        self.status = build_data[\"build\"][\"buildStatus\"]\n\n    def save(self, *args, **kwargs):\n        if self._state.adding:\n            self._create_build()\n\n        super().save(*args, **kwargs)\n\n    @property\n    def animate(self):\n        return self.status == BuildStatusChoices.IN_PROGRESS\n\n    @property\n    def status_context(self):\n        if self.status == BuildStatusChoices.SUCCEEDED:\n            return \"success\"\n        elif self.status in {BuildStatusChoices.STOPPED}:\n            return \"warning\"\n        elif self.status in {\n            BuildStatusChoices.FAILED,\n            BuildStatusChoices.FAULT,\n            BuildStatusChoices.TIMED_OUT,\n        }:\n            return \"danger\"\n        elif self.status in {BuildStatusChoices.IN_PROGRESS}:\n            return \"info\"\n        else:\n            return \"secondary\"\n\n    class Meta:\n        indexes = [\n            models.Index(fields=[\"build_id\"]),\n        ]\n", "path": "app/grandchallenge/codebuild/models.py"}, {"content": "from celery import shared_task\nfrom django.apps import apps\nfrom django.core.exceptions import ObjectDoesNotExist\nfrom django.db.transaction import on_commit\n\nfrom grandchallenge.algorithms.models import Algorithm, AlgorithmImage\n\n\n@shared_task()\ndef create_codebuild_build(*, pk):\n    GitHubWebhookMessage = apps.get_model(  # noqa: N806\n        app_label=\"github\", model_name=\"GitHubWebhookMessage\"\n    )\n    ghwm = GitHubWebhookMessage.objects.get(pk=pk)\n\n    Build = apps.get_model(  # noqa: N806\n        app_label=\"codebuild\", model_name=\"Build\"\n    )\n\n    if Build.objects.filter(webhook_message=ghwm).exists():\n        # Build already exists\n        return\n\n    try:\n        algorithm = Algorithm.objects.get(\n            repo_name=ghwm.payload[\"repository\"][\"full_name\"]\n        )\n    except ObjectDoesNotExist:\n        # Repository is not linked to algorithm\n        return\n\n    algorithm_image = AlgorithmImage.objects.create(\n        algorithm=algorithm,\n        requires_gpu=algorithm.image_requires_gpu,\n        requires_memory_gb=algorithm.image_requires_memory_gb,\n    )\n\n    build = Build.objects.create(\n        webhook_message=ghwm, algorithm_image=algorithm_image,\n    )\n\n    on_commit(\n        lambda: wait_for_build_completion.apply_async(\n            kwargs={\"build_pk\": str(build.pk)}\n        )\n    )\n\n\n@shared_task(bind=True, max_retries=120)\ndef wait_for_build_completion(self, *, build_pk):\n    Build = apps.get_model(  # noqa: N806\n        app_label=\"codebuild\", model_name=\"Build\"\n    )\n\n    build = Build.objects.get(pk=build_pk)\n    build.refresh_status()\n\n    if build.status == build.BuildStatusChoices.IN_PROGRESS:\n        self.retry(countdown=30)\n    else:\n        build.refresh_logs()\n        build.save()\n        if build.status == build.BuildStatusChoices.SUCCEEDED:\n            on_commit(\n                lambda: add_image_to_algorithm.apply_async(\n                    kwargs={\"build_pk\": str(build_pk)}\n                )\n            )\n\n\n@shared_task\ndef add_image_to_algorithm(*, build_pk):\n    Build = apps.get_model(  # noqa: N806\n        app_label=\"codebuild\", model_name=\"Build\"\n    )\n\n    build = Build.objects.get(pk=build_pk)\n    build.add_image_to_algorithm()\n", "path": "app/grandchallenge/codebuild/tasks.py"}], "after_files": [{"content": "import gzip\nfrom tempfile import NamedTemporaryFile\n\nimport boto3\nfrom django.conf import settings\nfrom django.core import files\nfrom django.db import models\nfrom django.utils.translation import gettext_lazy as _\n\nfrom grandchallenge.algorithms.models import AlgorithmImage\nfrom grandchallenge.core.models import UUIDModel\nfrom grandchallenge.core.storage import private_s3_storage\nfrom grandchallenge.github.models import GitHubWebhookMessage\n\n\nclass BuildStatusChoices(models.TextChoices):\n    \"\"\"From https://docs.aws.amazon.com/codebuild/latest/APIReference/API_Build.html\"\"\"\n\n    SUCCEEDED = \"SUCCEEDED\", _(\"Succeeded\")\n    FAILED = \"FAILED\", _(\"Failed\")\n    FAULT = \"FAULT\", _(\"Fault\")\n    TIMED_OUT = \"TIMED_OUT\", _(\"Timed Out\")\n    IN_PROGRESS = \"IN_PROGRESS\", _(\"In Progress\")\n    STOPPED = \"STOPPED\", _(\"Stopped\")\n\n\nclass Build(UUIDModel):\n    webhook_message = models.ForeignKey(\n        GitHubWebhookMessage, on_delete=models.SET_NULL, null=True\n    )\n    algorithm_image = models.OneToOneField(\n        AlgorithmImage, on_delete=models.SET_NULL, null=True\n    )\n    build_config = models.JSONField()\n    build_id = models.CharField(max_length=1024)\n    status = models.CharField(\n        choices=BuildStatusChoices.choices, max_length=11\n    )\n    build_log = models.TextField(blank=True)\n\n    BuildStatusChoices = BuildStatusChoices\n    __client = None\n\n    @property\n    def client(self):\n        if self.__client is None:\n            self.__client = boto3.client(\"codebuild\")\n        return self.__client\n\n    @property\n    def build_number(self):\n        return self.build_id.split(\":\")[-1]\n\n    def refresh_status(self):\n        build_statuses = self.client.batch_get_builds(ids=[self.build_id])\n        self.status = build_statuses[\"builds\"][0][\"buildStatus\"]\n\n    def refresh_logs(self):\n        try:\n            with private_s3_storage.open(\n                f\"codebuild/logs/{self.build_number}.gz\"\n            ) as file:\n                self.build_log = gzip.open(file).read().decode(\"utf-8\")\n        except FileNotFoundError:\n            self.build_log = \"Log file not available.\"\n\n    def add_image_to_algorithm(self):\n        # TODO, this would be much faster using S3 copy, can then run on a smaller queue\n        with private_s3_storage.open(\n            f\"codebuild/artifacts/{self.build_number}/{self.build_config['projectName']}/container-image.tar.gz\"\n        ) as file:\n            with NamedTemporaryFile(delete=True) as tmp_file:\n                with open(tmp_file.name, \"wb\") as fd:\n                    for chunk in file.chunks():\n                        fd.write(chunk)\n\n                tmp_file.flush()\n                temp_file = files.File(\n                    tmp_file, name=f\"{str(self.pk)}.tar.gz\",\n                )\n\n                self.algorithm_image.image = temp_file\n                self.algorithm_image.save()\n\n    def _create_build(self):\n        self.build_config = {\n            \"projectName\": settings.CODEBUILD_PROJECT_NAME,\n            \"sourceLocationOverride\": f\"{settings.PRIVATE_S3_STORAGE_KWARGS['bucket_name']}/{self.webhook_message.zipfile.name}\",\n            \"sourceTypeOverride\": \"S3\",\n            \"environmentVariablesOverride\": [\n                {\n                    \"name\": \"IMAGE_REPO_NAME\",\n                    \"value\": f\"{AlgorithmImage._meta.app_label}/{AlgorithmImage._meta.model_name}\",\n                },\n                {\"name\": \"IMAGE_TAG\", \"value\": str(self.algorithm_image.pk)},\n            ],\n        }\n\n        build_data = self.client.start_build(**self.build_config)\n\n        self.build_id = build_data[\"build\"][\"id\"]\n        self.status = build_data[\"build\"][\"buildStatus\"]\n\n    def save(self, *args, **kwargs):\n        if self._state.adding:\n            self._create_build()\n\n        super().save(*args, **kwargs)\n\n    @property\n    def animate(self):\n        return self.status == BuildStatusChoices.IN_PROGRESS\n\n    @property\n    def status_context(self):\n        if self.status == BuildStatusChoices.SUCCEEDED:\n            return \"success\"\n        elif self.status in {BuildStatusChoices.STOPPED}:\n            return \"warning\"\n        elif self.status in {\n            BuildStatusChoices.FAILED,\n            BuildStatusChoices.FAULT,\n            BuildStatusChoices.TIMED_OUT,\n        }:\n            return \"danger\"\n        elif self.status in {BuildStatusChoices.IN_PROGRESS}:\n            return \"info\"\n        else:\n            return \"secondary\"\n\n    class Meta:\n        indexes = [\n            models.Index(fields=[\"build_id\"]),\n        ]\n", "path": "app/grandchallenge/codebuild/models.py"}, {"content": "from celery import shared_task\nfrom django.apps import apps\nfrom django.conf import settings\nfrom django.core.exceptions import ObjectDoesNotExist\nfrom django.db import transaction\nfrom django.db.transaction import on_commit\n\nfrom grandchallenge.algorithms.models import Algorithm, AlgorithmImage\nfrom grandchallenge.components.tasks import _retry\n\n\n@shared_task(**settings.CELERY_TASK_DECORATOR_KWARGS[\"acks-late-micro-short\"])\ndef create_codebuild_build(*, pk):\n    GitHubWebhookMessage = apps.get_model(  # noqa: N806\n        app_label=\"github\", model_name=\"GitHubWebhookMessage\"\n    )\n    ghwm = GitHubWebhookMessage.objects.get(pk=pk)\n\n    Build = apps.get_model(  # noqa: N806\n        app_label=\"codebuild\", model_name=\"Build\"\n    )\n\n    if Build.objects.filter(webhook_message=ghwm).exists():\n        # Build already exists\n        return\n\n    try:\n        algorithm = Algorithm.objects.get(\n            repo_name=ghwm.payload[\"repository\"][\"full_name\"]\n        )\n    except ObjectDoesNotExist:\n        # Repository is not linked to algorithm\n        return\n\n    with transaction.atomic():\n        algorithm_image = AlgorithmImage.objects.create(\n            algorithm=algorithm,\n            requires_gpu=algorithm.image_requires_gpu,\n            requires_memory_gb=algorithm.image_requires_memory_gb,\n        )\n        build = Build.objects.create(\n            webhook_message=ghwm, algorithm_image=algorithm_image,\n        )\n\n        on_commit(\n            lambda: wait_for_build_completion.apply_async(\n                kwargs={\"build_pk\": str(build.pk)}\n            )\n        )\n\n\n@shared_task(**settings.CELERY_TASK_DECORATOR_KWARGS[\"acks-late-micro-short\"])\ndef wait_for_build_completion(*, build_pk, retries=0):\n    Build = apps.get_model(  # noqa: N806\n        app_label=\"codebuild\", model_name=\"Build\"\n    )\n\n    build = Build.objects.get(pk=build_pk)\n\n    with transaction.atomic():\n        build.refresh_status()\n\n        if build.status == build.BuildStatusChoices.IN_PROGRESS:\n            _retry(\n                task=wait_for_build_completion,\n                signature_kwargs={\"kwargs\": {\"build_pk\": build_pk}},\n                retries=retries,\n            )\n            return\n        else:\n            build.refresh_logs()\n            build.save()\n            if build.status == build.BuildStatusChoices.SUCCEEDED:\n                on_commit(\n                    lambda: add_image_to_algorithm.apply_async(\n                        kwargs={\"build_pk\": str(build_pk)}\n                    )\n                )\n\n\n@shared_task(**settings.CELERY_TASK_DECORATOR_KWARGS[\"acks-late-2xlarge\"])\ndef add_image_to_algorithm(*, build_pk):\n    Build = apps.get_model(  # noqa: N806\n        app_label=\"codebuild\", model_name=\"Build\"\n    )\n\n    build = Build.objects.get(pk=build_pk)\n\n    if not build.algorithm_image.image:\n        build.add_image_to_algorithm()\n", "path": "app/grandchallenge/codebuild/tasks.py"}]}