We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Catch ConnectionError for container cleanup
A connection error can sometimes occur on this line:

https://github.com/comic/grand-challenge.org/blob/9b7230e63040ad08654cb6280594f19eeb033372/app/grandchallenge/container_exec/backends/docker.py#L85

Catch it and return a sensible error message, or schedule the task to be retried later.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `app/grandchallenge/container_exec/backends/docker.py`
Content:
```
1 import io
2 import json
3 import os
4 import tarfile
5 import uuid
6 from contextlib import contextmanager
7 from json import JSONDecodeError
8 from pathlib import Path
9 from random import randint
10 from time import sleep
11 from typing import Tuple
12 
13 import docker
14 from django.conf import settings
15 from django.core.files import File
16 from docker.api.container import ContainerApiMixin
17 from docker.errors import ContainerError, APIError
18 from docker.tls import TLSConfig
19 from docker.types import LogConfig
20 from requests import HTTPError
21 
22 
23 class Executor(object):
24     def __init__(
25         self,
26         *,
27         job_id: uuid.UUID,
28         input_files: Tuple[File, ...],
29         exec_image: File,
30         exec_image_sha256: str,
31         results_file: Path,
32     ):
33         super().__init__()
34         self._job_id = str(job_id)
35         self._input_files = input_files
36         self._exec_image = exec_image
37         self._exec_image_sha256 = exec_image_sha256
38         self._io_image = settings.CONTAINER_EXEC_IO_IMAGE
39         self._results_file = results_file
40 
41         client_kwargs = {"base_url": settings.CONTAINER_EXEC_DOCKER_BASE_URL}
42 
43         if settings.CONTAINER_EXEC_DOCKER_TLSVERIFY:
44             tlsconfig = TLSConfig(
45                 verify=True,
46                 client_cert=(
47                     settings.CONTAINER_EXEC_DOCKER_TLSCERT,
48                     settings.CONTAINER_EXEC_DOCKER_TLSKEY,
49                 ),
50                 ca_cert=settings.CONTAINER_EXEC_DOCKER_TLSCACERT,
51             )
52             client_kwargs.update({"tls": tlsconfig})
53 
54         self._client = docker.DockerClient(**client_kwargs)
55 
56         self._input_volume = f"{self._job_id}-input"
57         self._output_volume = f"{self._job_id}-output"
58 
59         self._run_kwargs = {
60             "labels": {"job_id": self._job_id},
61             "init": True,
62             "network_disabled": True,
63             "mem_limit": settings.CONTAINER_EXEC_MEMORY_LIMIT,
64             # Set to the same as mem_limit to avoid using swap
65             "memswap_limit": settings.CONTAINER_EXEC_MEMORY_LIMIT,
66             "cpu_period": settings.CONTAINER_EXEC_CPU_PERIOD,
67             "cpu_quota": settings.CONTAINER_EXEC_CPU_QUOTA,
68             # Use the default weight
69             "cpu_shares": 1024,
70             "runtime": settings.CONTAINER_EXEC_DOCKER_RUNTIME,
71             "cap_drop": ["all"],
72             "security_opt": ["no-new-privileges"],
73             "pids_limit": 64,
74             "log_config": LogConfig(
75                 type=LogConfig.types.JSON, config={"max-size": "1g"}
76             ),
77         }
78 
79     def __enter__(self):
80         return self
81 
82     def __exit__(self, exc_type, exc_val, exc_tb):
83         flt = {"label": f"job_id={self._job_id}"}
84 
85         for container in self._client.containers.list(filters=flt):
86             container.stop()
87 
88         self.__retry_docker_obj_prune(obj=self._client.containers, filters=flt)
89         self.__retry_docker_obj_prune(obj=self._client.volumes, filters=flt)
90 
91     @staticmethod
92     def __retry_docker_obj_prune(*, obj, filters: dict):
93         # Retry and exponential backoff of the prune command as only 1 prune
94         # operation can occur at a time on a docker host
95         num_retries = 0
96         e = APIError
97         while num_retries < 3:
98             try:
99                 obj.prune(filters=filters)
100                 break
101             except (APIError, HTTPError) as e:
102                 num_retries += 1
103                 sleep((2 ** num_retries) + (randint(0, 1000) / 1000))
104         else:
105             raise e
106 
107     def execute(self) -> dict:
108         self._pull_images()
109         self._create_io_volumes()
110         self._provision_input_volume()
111         self._chmod_volumes()
112         self._execute_container()
113         return self._get_result()
114 
115     def _pull_images(self):
116         self._client.images.pull(repository=self._io_image)
117 
118         if self._exec_image_sha256 not in [
119             img.id for img in self._client.images.list()
120         ]:
121             with self._exec_image.open("rb") as f:
122                 self._client.images.load(f)
123 
124     def _create_io_volumes(self):
125         for volume in [self._input_volume, self._output_volume]:
126             self._client.volumes.create(
127                 name=volume, labels=self._run_kwargs["labels"]
128             )
129 
130     def _provision_input_volume(self):
131         try:
132             with cleanup(
133                 self._client.containers.run(
134                     image=self._io_image,
135                     volumes={
136                         self._input_volume: {"bind": "/input/", "mode": "rw"}
137                     },
138                     name=f"{self._job_id}-writer",
139                     detach=True,
140                     tty=True,
141                     **self._run_kwargs,
142                 )
143             ) as writer:
144                 self._copy_input_files(writer=writer)
145         except Exception as exc:
146             raise RuntimeError(str(exc))
147 
148     def _copy_input_files(self, writer):
149         for file in self._input_files:
150             put_file(
151                 container=writer,
152                 src=file,
153                 dest=f"/input/{Path(file.name).name}",
154             )
155 
156     def _chmod_volumes(self):
157         """ Ensure that the i/o directories are writable """
158         try:
159             self._client.containers.run(
160                 image=self._io_image,
161                 volumes={
162                     self._input_volume: {"bind": "/input/", "mode": "rw"},
163                     self._output_volume: {"bind": "/output/", "mode": "rw"},
164                 },
165                 name=f"{self._job_id}-chmod-volumes",
166                 command=f"chmod -R 0777 /input/ /output/",
167                 remove=True,
168                 **self._run_kwargs,
169             )
170         except Exception as exc:
171             raise RuntimeError(str(exc))
172 
173     def _execute_container(self):
174         try:
175             self._client.containers.run(
176                 image=self._exec_image_sha256,
177                 volumes={
178                     self._input_volume: {"bind": "/input/", "mode": "rw"},
179                     self._output_volume: {"bind": "/output/", "mode": "rw"},
180                 },
181                 name=f"{self._job_id}-executor",
182                 remove=True,
183                 **self._run_kwargs,
184             )
185         except ContainerError as exc:
186             raise RuntimeError(exc.stderr.decode())
187 
188     def _get_result(self) -> dict:
189         """
190         Read and parse the created results file. Due to a bug in the docker
191         client, copy the file to memory first rather than cat and read
192         stdout.
193         """
194         try:
195             with cleanup(
196                 self._client.containers.run(
197                     image=self._io_image,
198                     volumes={
199                         self._output_volume: {"bind": "/output/", "mode": "ro"}
200                     },
201                     name=f"{self._job_id}-reader",
202                     detach=True,
203                     tty=True,
204                     **self._run_kwargs,
205                 )
206             ) as reader:
207                 result = get_file(container=reader, src=self._results_file)
208         except Exception as e:
209             raise RuntimeError(str(e))
210 
211         try:
212             result = json.loads(
213                 result.read().decode(),
214                 parse_constant=lambda x: None,  # Removes -inf, inf and NaN
215             )
216         except JSONDecodeError as exc:
217             raise RuntimeError(exc.msg)
218 
219         return result
220 
221 
222 @contextmanager
223 def cleanup(container: ContainerApiMixin):
224     """
225     Cleans up a docker container which is running in detached mode
226 
227     :param container: An instance of a container
228     :return:
229     """
230     try:
231         yield container
232 
233     finally:
234         container.stop()
235         container.remove(force=True)
236 
237 
238 def put_file(*, container: ContainerApiMixin, src: File, dest: str) -> ():
239     """
240     Puts a file on the host into a container.
241     This method will create an in memory tar archive, add the src file to this
242     and upload it to the docker container where it will be unarchived at dest.
243 
244     :param container: The container to write to
245     :param src: The path to the source file on the host
246     :param dest: The path to the target file in the container
247     :return:
248     """
249     tar_b = io.BytesIO()
250 
251     tarinfo = tarfile.TarInfo(name=os.path.basename(dest))
252     tarinfo.size = src.size
253 
254     with tarfile.open(fileobj=tar_b, mode="w") as tar, src.open("rb") as f:
255         tar.addfile(tarinfo, fileobj=f)
256 
257     tar_b.seek(0)
258     container.put_archive(os.path.dirname(dest), tar_b)
259 
260 
261 def get_file(*, container: ContainerApiMixin, src: Path):
262     tarstrm, info = container.get_archive(src)
263 
264     if info["size"] > 2E9:
265         raise ValueError(f"File {src} is too big to be decompressed.")
266 
267     file_obj = io.BytesIO()
268     for ts in tarstrm:
269         file_obj.write(ts)
270 
271     file_obj.seek(0)
272     tar = tarfile.open(mode="r", fileobj=file_obj)
273     content = tar.extractfile(src.name)
274 
275     return content
276
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/app/grandchallenge/container_exec/backends/docker.py b/app/grandchallenge/container_exec/backends/docker.py
--- a/app/grandchallenge/container_exec/backends/docker.py
+++ b/app/grandchallenge/container_exec/backends/docker.py
@@ -14,7 +14,7 @@
 from django.conf import settings
 from django.core.files import File
 from docker.api.container import ContainerApiMixin
-from docker.errors import ContainerError, APIError
+from docker.errors import ContainerError, APIError, NotFound
 from docker.tls import TLSConfig
 from docker.types import LogConfig
 from requests import HTTPError
@@ -82,11 +82,18 @@
     def __exit__(self, exc_type, exc_val, exc_tb):
         flt = {"label": f"job_id={self._job_id}"}
 
-        for container in self._client.containers.list(filters=flt):
-            container.stop()
+        try:
+            for c in self._client.containers.list(filters=flt):
+                c.stop()
 
-        self.__retry_docker_obj_prune(obj=self._client.containers, filters=flt)
-        self.__retry_docker_obj_prune(obj=self._client.volumes, filters=flt)
+            self.__retry_docker_obj_prune(
+                obj=self._client.containers, filters=flt
+            )
+            self.__retry_docker_obj_prune(
+                obj=self._client.volumes, filters=flt
+            )
+        except ConnectionError:
+            raise RuntimeError("Could not connect to worker.")
 
     @staticmethod
     def __retry_docker_obj_prune(*, obj, filters: dict):
@@ -205,6 +212,15 @@
                 )
             ) as reader:
                 result = get_file(container=reader, src=self._results_file)
+        except NotFound:
+            # The container exited without error, but no results file was
+            # produced. This shouldn't happen, but does with poorly programmed
+            # evaluation containers.
+            raise RuntimeError(
+                "The evaluation failed for an unknown reason as no results "
+                "file was produced. Please contact the organisers for "
+                "assistance."
+            )
         except Exception as e:
             raise RuntimeError(str(e))

{"golden_diff": "diff --git a/app/grandchallenge/container_exec/backends/docker.py b/app/grandchallenge/container_exec/backends/docker.py\n--- a/app/grandchallenge/container_exec/backends/docker.py\n+++ b/app/grandchallenge/container_exec/backends/docker.py\n@@ -14,7 +14,7 @@\n from django.conf import settings\n from django.core.files import File\n from docker.api.container import ContainerApiMixin\n-from docker.errors import ContainerError, APIError\n+from docker.errors import ContainerError, APIError, NotFound\n from docker.tls import TLSConfig\n from docker.types import LogConfig\n from requests import HTTPError\n@@ -82,11 +82,18 @@\n     def __exit__(self, exc_type, exc_val, exc_tb):\n         flt = {\"label\": f\"job_id={self._job_id}\"}\n \n-        for container in self._client.containers.list(filters=flt):\n-            container.stop()\n+        try:\n+            for c in self._client.containers.list(filters=flt):\n+                c.stop()\n \n-        self.__retry_docker_obj_prune(obj=self._client.containers, filters=flt)\n-        self.__retry_docker_obj_prune(obj=self._client.volumes, filters=flt)\n+            self.__retry_docker_obj_prune(\n+                obj=self._client.containers, filters=flt\n+            )\n+            self.__retry_docker_obj_prune(\n+                obj=self._client.volumes, filters=flt\n+            )\n+        except ConnectionError:\n+            raise RuntimeError(\"Could not connect to worker.\")\n \n     @staticmethod\n     def __retry_docker_obj_prune(*, obj, filters: dict):\n@@ -205,6 +212,15 @@\n                 )\n             ) as reader:\n                 result = get_file(container=reader, src=self._results_file)\n+        except NotFound:\n+            # The container exited without error, but no results file was\n+            # produced. This shouldn't happen, but does with poorly programmed\n+            # evaluation containers.\n+            raise RuntimeError(\n+                \"The evaluation failed for an unknown reason as no results \"\n+                \"file was produced. Please contact the organisers for \"\n+                \"assistance.\"\n+            )\n         except Exception as e:\n             raise RuntimeError(str(e))\n", "issue": "Catch ConnectionError for container cleanup\nA connection error can sometimes occur on this line:\r\n\r\nhttps://github.com/comic/grand-challenge.org/blob/9b7230e63040ad08654cb6280594f19eeb033372/app/grandchallenge/container_exec/backends/docker.py#L85\r\n\r\nCatch it and return a sensible error message, or schedule the task to be retried later.\n", "before_files": [{"content": "import io\nimport json\nimport os\nimport tarfile\nimport uuid\nfrom contextlib import contextmanager\nfrom json import JSONDecodeError\nfrom pathlib import Path\nfrom random import randint\nfrom time import sleep\nfrom typing import Tuple\n\nimport docker\nfrom django.conf import settings\nfrom django.core.files import File\nfrom docker.api.container import ContainerApiMixin\nfrom docker.errors import ContainerError, APIError\nfrom docker.tls import TLSConfig\nfrom docker.types import LogConfig\nfrom requests import HTTPError\n\n\nclass Executor(object):\n    def __init__(\n        self,\n        *,\n        job_id: uuid.UUID,\n        input_files: Tuple[File, ...],\n        exec_image: File,\n        exec_image_sha256: str,\n        results_file: Path,\n    ):\n        super().__init__()\n        self._job_id = str(job_id)\n        self._input_files = input_files\n        self._exec_image = exec_image\n        self._exec_image_sha256 = exec_image_sha256\n        self._io_image = settings.CONTAINER_EXEC_IO_IMAGE\n        self._results_file = results_file\n\n        client_kwargs = {\"base_url\": settings.CONTAINER_EXEC_DOCKER_BASE_URL}\n\n        if settings.CONTAINER_EXEC_DOCKER_TLSVERIFY:\n            tlsconfig = TLSConfig(\n                verify=True,\n                client_cert=(\n                    settings.CONTAINER_EXEC_DOCKER_TLSCERT,\n                    settings.CONTAINER_EXEC_DOCKER_TLSKEY,\n                ),\n                ca_cert=settings.CONTAINER_EXEC_DOCKER_TLSCACERT,\n            )\n            client_kwargs.update({\"tls\": tlsconfig})\n\n        self._client = docker.DockerClient(**client_kwargs)\n\n        self._input_volume = f\"{self._job_id}-input\"\n        self._output_volume = f\"{self._job_id}-output\"\n\n        self._run_kwargs = {\n            \"labels\": {\"job_id\": self._job_id},\n            \"init\": True,\n            \"network_disabled\": True,\n            \"mem_limit\": settings.CONTAINER_EXEC_MEMORY_LIMIT,\n            # Set to the same as mem_limit to avoid using swap\n            \"memswap_limit\": settings.CONTAINER_EXEC_MEMORY_LIMIT,\n            \"cpu_period\": settings.CONTAINER_EXEC_CPU_PERIOD,\n            \"cpu_quota\": settings.CONTAINER_EXEC_CPU_QUOTA,\n            # Use the default weight\n            \"cpu_shares\": 1024,\n            \"runtime\": settings.CONTAINER_EXEC_DOCKER_RUNTIME,\n            \"cap_drop\": [\"all\"],\n            \"security_opt\": [\"no-new-privileges\"],\n            \"pids_limit\": 64,\n            \"log_config\": LogConfig(\n                type=LogConfig.types.JSON, config={\"max-size\": \"1g\"}\n            ),\n        }\n\n    def __enter__(self):\n        return self\n\n    def __exit__(self, exc_type, exc_val, exc_tb):\n        flt = {\"label\": f\"job_id={self._job_id}\"}\n\n        for container in self._client.containers.list(filters=flt):\n            container.stop()\n\n        self.__retry_docker_obj_prune(obj=self._client.containers, filters=flt)\n        self.__retry_docker_obj_prune(obj=self._client.volumes, filters=flt)\n\n    @staticmethod\n    def __retry_docker_obj_prune(*, obj, filters: dict):\n        # Retry and exponential backoff of the prune command as only 1 prune\n        # operation can occur at a time on a docker host\n        num_retries = 0\n        e = APIError\n        while num_retries < 3:\n            try:\n                obj.prune(filters=filters)\n                break\n            except (APIError, HTTPError) as e:\n                num_retries += 1\n                sleep((2 ** num_retries) + (randint(0, 1000) / 1000))\n        else:\n            raise e\n\n    def execute(self) -> dict:\n        self._pull_images()\n        self._create_io_volumes()\n        self._provision_input_volume()\n        self._chmod_volumes()\n        self._execute_container()\n        return self._get_result()\n\n    def _pull_images(self):\n        self._client.images.pull(repository=self._io_image)\n\n        if self._exec_image_sha256 not in [\n            img.id for img in self._client.images.list()\n        ]:\n            with self._exec_image.open(\"rb\") as f:\n                self._client.images.load(f)\n\n    def _create_io_volumes(self):\n        for volume in [self._input_volume, self._output_volume]:\n            self._client.volumes.create(\n                name=volume, labels=self._run_kwargs[\"labels\"]\n            )\n\n    def _provision_input_volume(self):\n        try:\n            with cleanup(\n                self._client.containers.run(\n                    image=self._io_image,\n                    volumes={\n                        self._input_volume: {\"bind\": \"/input/\", \"mode\": \"rw\"}\n                    },\n                    name=f\"{self._job_id}-writer\",\n                    detach=True,\n                    tty=True,\n                    **self._run_kwargs,\n                )\n            ) as writer:\n                self._copy_input_files(writer=writer)\n        except Exception as exc:\n            raise RuntimeError(str(exc))\n\n    def _copy_input_files(self, writer):\n        for file in self._input_files:\n            put_file(\n                container=writer,\n                src=file,\n                dest=f\"/input/{Path(file.name).name}\",\n            )\n\n    def _chmod_volumes(self):\n        \"\"\" Ensure that the i/o directories are writable \"\"\"\n        try:\n            self._client.containers.run(\n                image=self._io_image,\n                volumes={\n                    self._input_volume: {\"bind\": \"/input/\", \"mode\": \"rw\"},\n                    self._output_volume: {\"bind\": \"/output/\", \"mode\": \"rw\"},\n                },\n                name=f\"{self._job_id}-chmod-volumes\",\n                command=f\"chmod -R 0777 /input/ /output/\",\n                remove=True,\n                **self._run_kwargs,\n            )\n        except Exception as exc:\n            raise RuntimeError(str(exc))\n\n    def _execute_container(self):\n        try:\n            self._client.containers.run(\n                image=self._exec_image_sha256,\n                volumes={\n                    self._input_volume: {\"bind\": \"/input/\", \"mode\": \"rw\"},\n                    self._output_volume: {\"bind\": \"/output/\", \"mode\": \"rw\"},\n                },\n                name=f\"{self._job_id}-executor\",\n                remove=True,\n                **self._run_kwargs,\n            )\n        except ContainerError as exc:\n            raise RuntimeError(exc.stderr.decode())\n\n    def _get_result(self) -> dict:\n        \"\"\"\n        Read and parse the created results file. Due to a bug in the docker\n        client, copy the file to memory first rather than cat and read\n        stdout.\n        \"\"\"\n        try:\n            with cleanup(\n                self._client.containers.run(\n                    image=self._io_image,\n                    volumes={\n                        self._output_volume: {\"bind\": \"/output/\", \"mode\": \"ro\"}\n                    },\n                    name=f\"{self._job_id}-reader\",\n                    detach=True,\n                    tty=True,\n                    **self._run_kwargs,\n                )\n            ) as reader:\n                result = get_file(container=reader, src=self._results_file)\n        except Exception as e:\n            raise RuntimeError(str(e))\n\n        try:\n            result = json.loads(\n                result.read().decode(),\n                parse_constant=lambda x: None,  # Removes -inf, inf and NaN\n            )\n        except JSONDecodeError as exc:\n            raise RuntimeError(exc.msg)\n\n        return result\n\n\n@contextmanager\ndef cleanup(container: ContainerApiMixin):\n    \"\"\"\n    Cleans up a docker container which is running in detached mode\n\n    :param container: An instance of a container\n    :return:\n    \"\"\"\n    try:\n        yield container\n\n    finally:\n        container.stop()\n        container.remove(force=True)\n\n\ndef put_file(*, container: ContainerApiMixin, src: File, dest: str) -> ():\n    \"\"\"\n    Puts a file on the host into a container.\n    This method will create an in memory tar archive, add the src file to this\n    and upload it to the docker container where it will be unarchived at dest.\n\n    :param container: The container to write to\n    :param src: The path to the source file on the host\n    :param dest: The path to the target file in the container\n    :return:\n    \"\"\"\n    tar_b = io.BytesIO()\n\n    tarinfo = tarfile.TarInfo(name=os.path.basename(dest))\n    tarinfo.size = src.size\n\n    with tarfile.open(fileobj=tar_b, mode=\"w\") as tar, src.open(\"rb\") as f:\n        tar.addfile(tarinfo, fileobj=f)\n\n    tar_b.seek(0)\n    container.put_archive(os.path.dirname(dest), tar_b)\n\n\ndef get_file(*, container: ContainerApiMixin, src: Path):\n    tarstrm, info = container.get_archive(src)\n\n    if info[\"size\"] > 2E9:\n        raise ValueError(f\"File {src} is too big to be decompressed.\")\n\n    file_obj = io.BytesIO()\n    for ts in tarstrm:\n        file_obj.write(ts)\n\n    file_obj.seek(0)\n    tar = tarfile.open(mode=\"r\", fileobj=file_obj)\n    content = tar.extractfile(src.name)\n\n    return content\n", "path": "app/grandchallenge/container_exec/backends/docker.py"}], "after_files": [{"content": "import io\nimport json\nimport os\nimport tarfile\nimport uuid\nfrom contextlib import contextmanager\nfrom json import JSONDecodeError\nfrom pathlib import Path\nfrom random import randint\nfrom time import sleep\nfrom typing import Tuple\n\nimport docker\nfrom django.conf import settings\nfrom django.core.files import File\nfrom docker.api.container import ContainerApiMixin\nfrom docker.errors import ContainerError, APIError, NotFound\nfrom docker.tls import TLSConfig\nfrom docker.types import LogConfig\nfrom requests import HTTPError\n\n\nclass Executor(object):\n    def __init__(\n        self,\n        *,\n        job_id: uuid.UUID,\n        input_files: Tuple[File, ...],\n        exec_image: File,\n        exec_image_sha256: str,\n        results_file: Path,\n    ):\n        super().__init__()\n        self._job_id = str(job_id)\n        self._input_files = input_files\n        self._exec_image = exec_image\n        self._exec_image_sha256 = exec_image_sha256\n        self._io_image = settings.CONTAINER_EXEC_IO_IMAGE\n        self._results_file = results_file\n\n        client_kwargs = {\"base_url\": settings.CONTAINER_EXEC_DOCKER_BASE_URL}\n\n        if settings.CONTAINER_EXEC_DOCKER_TLSVERIFY:\n            tlsconfig = TLSConfig(\n                verify=True,\n                client_cert=(\n                    settings.CONTAINER_EXEC_DOCKER_TLSCERT,\n                    settings.CONTAINER_EXEC_DOCKER_TLSKEY,\n                ),\n                ca_cert=settings.CONTAINER_EXEC_DOCKER_TLSCACERT,\n            )\n            client_kwargs.update({\"tls\": tlsconfig})\n\n        self._client = docker.DockerClient(**client_kwargs)\n\n        self._input_volume = f\"{self._job_id}-input\"\n        self._output_volume = f\"{self._job_id}-output\"\n\n        self._run_kwargs = {\n            \"labels\": {\"job_id\": self._job_id},\n            \"init\": True,\n            \"network_disabled\": True,\n            \"mem_limit\": settings.CONTAINER_EXEC_MEMORY_LIMIT,\n            # Set to the same as mem_limit to avoid using swap\n            \"memswap_limit\": settings.CONTAINER_EXEC_MEMORY_LIMIT,\n            \"cpu_period\": settings.CONTAINER_EXEC_CPU_PERIOD,\n            \"cpu_quota\": settings.CONTAINER_EXEC_CPU_QUOTA,\n            # Use the default weight\n            \"cpu_shares\": 1024,\n            \"runtime\": settings.CONTAINER_EXEC_DOCKER_RUNTIME,\n            \"cap_drop\": [\"all\"],\n            \"security_opt\": [\"no-new-privileges\"],\n            \"pids_limit\": 64,\n            \"log_config\": LogConfig(\n                type=LogConfig.types.JSON, config={\"max-size\": \"1g\"}\n            ),\n        }\n\n    def __enter__(self):\n        return self\n\n    def __exit__(self, exc_type, exc_val, exc_tb):\n        flt = {\"label\": f\"job_id={self._job_id}\"}\n\n        try:\n            for c in self._client.containers.list(filters=flt):\n                c.stop()\n\n            self.__retry_docker_obj_prune(\n                obj=self._client.containers, filters=flt\n            )\n            self.__retry_docker_obj_prune(\n                obj=self._client.volumes, filters=flt\n            )\n        except ConnectionError:\n            raise RuntimeError(\"Could not connect to worker.\")\n\n    @staticmethod\n    def __retry_docker_obj_prune(*, obj, filters: dict):\n        # Retry and exponential backoff of the prune command as only 1 prune\n        # operation can occur at a time on a docker host\n        num_retries = 0\n        e = APIError\n        while num_retries < 3:\n            try:\n                obj.prune(filters=filters)\n                break\n            except (APIError, HTTPError) as e:\n                num_retries += 1\n                sleep((2 ** num_retries) + (randint(0, 1000) / 1000))\n        else:\n            raise e\n\n    def execute(self) -> dict:\n        self._pull_images()\n        self._create_io_volumes()\n        self._provision_input_volume()\n        self._chmod_volumes()\n        self._execute_container()\n        return self._get_result()\n\n    def _pull_images(self):\n        self._client.images.pull(repository=self._io_image)\n\n        if self._exec_image_sha256 not in [\n            img.id for img in self._client.images.list()\n        ]:\n            with self._exec_image.open(\"rb\") as f:\n                self._client.images.load(f)\n\n    def _create_io_volumes(self):\n        for volume in [self._input_volume, self._output_volume]:\n            self._client.volumes.create(\n                name=volume, labels=self._run_kwargs[\"labels\"]\n            )\n\n    def _provision_input_volume(self):\n        try:\n            with cleanup(\n                self._client.containers.run(\n                    image=self._io_image,\n                    volumes={\n                        self._input_volume: {\"bind\": \"/input/\", \"mode\": \"rw\"}\n                    },\n                    name=f\"{self._job_id}-writer\",\n                    detach=True,\n                    tty=True,\n                    **self._run_kwargs,\n                )\n            ) as writer:\n                self._copy_input_files(writer=writer)\n        except Exception as exc:\n            raise RuntimeError(str(exc))\n\n    def _copy_input_files(self, writer):\n        for file in self._input_files:\n            put_file(\n                container=writer,\n                src=file,\n                dest=f\"/input/{Path(file.name).name}\",\n            )\n\n    def _chmod_volumes(self):\n        \"\"\" Ensure that the i/o directories are writable \"\"\"\n        try:\n            self._client.containers.run(\n                image=self._io_image,\n                volumes={\n                    self._input_volume: {\"bind\": \"/input/\", \"mode\": \"rw\"},\n                    self._output_volume: {\"bind\": \"/output/\", \"mode\": \"rw\"},\n                },\n                name=f\"{self._job_id}-chmod-volumes\",\n                command=f\"chmod -R 0777 /input/ /output/\",\n                remove=True,\n                **self._run_kwargs,\n            )\n        except Exception as exc:\n            raise RuntimeError(str(exc))\n\n    def _execute_container(self):\n        try:\n            self._client.containers.run(\n                image=self._exec_image_sha256,\n                volumes={\n                    self._input_volume: {\"bind\": \"/input/\", \"mode\": \"rw\"},\n                    self._output_volume: {\"bind\": \"/output/\", \"mode\": \"rw\"},\n                },\n                name=f\"{self._job_id}-executor\",\n                remove=True,\n                **self._run_kwargs,\n            )\n        except ContainerError as exc:\n            raise RuntimeError(exc.stderr.decode())\n\n    def _get_result(self) -> dict:\n        \"\"\"\n        Read and parse the created results file. Due to a bug in the docker\n        client, copy the file to memory first rather than cat and read\n        stdout.\n        \"\"\"\n        try:\n            with cleanup(\n                self._client.containers.run(\n                    image=self._io_image,\n                    volumes={\n                        self._output_volume: {\"bind\": \"/output/\", \"mode\": \"ro\"}\n                    },\n                    name=f\"{self._job_id}-reader\",\n                    detach=True,\n                    tty=True,\n                    **self._run_kwargs,\n                )\n            ) as reader:\n                result = get_file(container=reader, src=self._results_file)\n        except NotFound:\n            # The container exited without error, but no results file was\n            # produced. This shouldn't happen, but does with poorly programmed\n            # evaluation containers.\n            raise RuntimeError(\n                \"The evaluation failed for an unknown reason as no results \"\n                \"file was produced. Please contact the organisers for \"\n                \"assistance.\"\n            )\n        except Exception as e:\n            raise RuntimeError(str(e))\n\n        try:\n            result = json.loads(\n                result.read().decode(),\n                parse_constant=lambda x: None,  # Removes -inf, inf and NaN\n            )\n        except JSONDecodeError as exc:\n            raise RuntimeError(exc.msg)\n\n        return result\n\n\n@contextmanager\ndef cleanup(container: ContainerApiMixin):\n    \"\"\"\n    Cleans up a docker container which is running in detached mode\n\n    :param container: An instance of a container\n    :return:\n    \"\"\"\n    try:\n        yield container\n\n    finally:\n        container.stop()\n        container.remove(force=True)\n\n\ndef put_file(*, container: ContainerApiMixin, src: File, dest: str) -> ():\n    \"\"\"\n    Puts a file on the host into a container.\n    This method will create an in memory tar archive, add the src file to this\n    and upload it to the docker container where it will be unarchived at dest.\n\n    :param container: The container to write to\n    :param src: The path to the source file on the host\n    :param dest: The path to the target file in the container\n    :return:\n    \"\"\"\n    tar_b = io.BytesIO()\n\n    tarinfo = tarfile.TarInfo(name=os.path.basename(dest))\n    tarinfo.size = src.size\n\n    with tarfile.open(fileobj=tar_b, mode=\"w\") as tar, src.open(\"rb\") as f:\n        tar.addfile(tarinfo, fileobj=f)\n\n    tar_b.seek(0)\n    container.put_archive(os.path.dirname(dest), tar_b)\n\n\ndef get_file(*, container: ContainerApiMixin, src: Path):\n    tarstrm, info = container.get_archive(src)\n\n    if info[\"size\"] > 2E9:\n        raise ValueError(f\"File {src} is too big to be decompressed.\")\n\n    file_obj = io.BytesIO()\n    for ts in tarstrm:\n        file_obj.write(ts)\n\n    file_obj.seek(0)\n    tar = tarfile.open(mode=\"r\", fileobj=file_obj)\n    content = tar.extractfile(src.name)\n\n    return content\n", "path": "app/grandchallenge/container_exec/backends/docker.py"}]}