We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
singularity image pull triggered for every step when using beegfs


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `cwltool/singularity.py`
Content:
```
1 """Support for executing Docker containers using Singularity."""
2 from __future__ import absolute_import
3 import logging
4 import os
5 import re
6 import shutil
7 import sys
8 from io import open  # pylint: disable=redefined-builtin
9 from typing import (Dict, List, Text, Optional, MutableMapping)
10 from .errors import WorkflowException
11 from .job import ContainerCommandLineJob
12 from .pathmapper import PathMapper, ensure_writable
13 from .process import (UnsupportedRequirement)
14 from .utils import docker_windows_path_adjust
15 from schema_salad.sourceline import SourceLine
16 if os.name == 'posix' and sys.version_info[0] < 3:
17     from subprocess32 import (check_call, check_output,  # pylint: disable=import-error
18                               CalledProcessError, DEVNULL, PIPE, Popen,
19                               TimeoutExpired)
20 elif os.name == 'posix':
21     from subprocess import (check_call, check_output,  # type: ignore
22                             CalledProcessError, DEVNULL, PIPE, Popen,
23                             TimeoutExpired)
24 else:  # we're not on Unix, so none of this matters
25     pass
26 
27 _logger = logging.getLogger("cwltool")
28 _USERNS = None
29 
30 def _singularity_supports_userns():  # type: ()->bool
31     global _USERNS  # pylint: disable=global-statement
32     if _USERNS is None:
33         try:
34             result = Popen(
35                 [u"singularity", u"exec", u"--userns", u"/etc", u"true"],
36                 stderr=PIPE, stdout=DEVNULL,
37                 universal_newlines=True).communicate(timeout=60)[1]
38             _USERNS = "No valid /bin/sh" in result
39         except TimeoutExpired:
40             _USERNS = False
41     return _USERNS
42 
43 def _normalizeImageId(string):  # type: (Text)->Text
44     candidate = re.sub(pattern=r'([a-z]*://)', repl=r'', string=string)
45     return re.sub(pattern=r'[:/]', repl=r'-', string=candidate) + ".img"
46 
47 
48 class SingularityCommandLineJob(ContainerCommandLineJob):
49 
50     @staticmethod
51     def get_image(dockerRequirement,  # type: Dict[Text, Text]
52                   pull_image,         # type: bool
53                   force_pull=False    # type: bool
54                  ):
55         # type: (...) -> bool
56         """
57         Acquire the software container image in the specified dockerRequirement
58         using Singularity and returns the success as a bool. Updates the
59         provided dockerRequirement with the specific dockerImageId to the full
60         path of the local image, if found. Likewise the
61         dockerRequirement['dockerPull'] is updated to a docker:// URI if needed.
62         """
63         found = False
64 
65         candidates = []
66 
67         if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
68             match = re.search(pattern=r'([a-z]*://)', string=dockerRequirement["dockerPull"])
69             candidate = _normalizeImageId(dockerRequirement['dockerPull'])
70             candidates.append(candidate)
71             dockerRequirement['dockerImageId'] = candidate
72             if not match:
73                 dockerRequirement["dockerPull"] = "docker://" + dockerRequirement["dockerPull"]
74         elif "dockerImageId" in dockerRequirement:
75             candidates.append(dockerRequirement['dockerImageId'])
76             candidates.append(_normalizeImageId(dockerRequirement['dockerImageId']))
77 
78         # check if Singularity image is available in $SINGULARITY_CACHEDIR
79         for target in ("SINGULARITY_CACHEDIR", "SINGULARITY_PULLFOLDER",
80                 os.getcwd()):
81             if target in os.environ:
82                 for candidate in candidates:
83                     path = os.path.join(os.environ[target], candidate)
84                     if os.path.isfile(path):
85                         _logger.info("Using local copy of Singularity image "
86                                      "found in {}".format(target))
87                         dockerRequirement["dockerImageId"] = path
88                         found = True
89 
90         if (force_pull or not found) and pull_image:
91             cmd = []  # type: List[Text]
92             if "dockerPull" in dockerRequirement:
93                 cmd = ["singularity", "pull", "--force", "--name",
94                        str(dockerRequirement["dockerImageId"]),
95                        str(dockerRequirement["dockerPull"])]
96                 _logger.info(Text(cmd))
97                 check_call(cmd, stdout=sys.stderr)
98                 found = True
99             elif "dockerFile" in dockerRequirement:
100                 raise WorkflowException(SourceLine(
101                     dockerRequirement, 'dockerFile').makeError(
102                     "dockerFile is not currently supported when using the "
103                     "Singularity runtime for Docker containers."))
104             elif "dockerLoad" in dockerRequirement:
105                 raise WorkflowException(SourceLine(
106                     dockerRequirement, 'dockerLoad').makeError(
107                     "dockerLoad is not currently supported when using the "
108                     "Singularity runtime for Docker containers."))
109             elif "dockerImport" in dockerRequirement:
110                 raise WorkflowException(SourceLine(
111                     dockerRequirement, 'dockerImport').makeError(
112                     "dockerImport is not currently supported when using the "
113                     "Singularity runtime for Docker containers."))
114 
115         return found
116 
117     def get_from_requirements(self,
118                               r,                      # type: Optional[Dict[Text, Text]]
119                               req,                    # type: bool
120                               pull_image,             # type: bool
121                               force_pull=False,       # type: bool
122                               tmp_outdir_prefix=None  # type: Text
123                              ):  # type: (...) -> Text
124         """
125         Returns the filename of the Singularity image (e.g.
126         hello-world-latest.img).
127         """
128 
129         if r:
130             errmsg = None
131             try:
132                 check_output(["singularity", "--version"])
133             except CalledProcessError as err:
134                 errmsg = "Cannot execute 'singularity --version' {}".format(err)
135             except OSError as err:
136                 errmsg = "'singularity' executable not found: {}".format(err)
137 
138             if errmsg:
139                 if req:
140                     raise WorkflowException(errmsg)
141                 else:
142                     return None
143 
144             if self.get_image(r, pull_image, force_pull):
145                 return os.path.abspath(r["dockerImageId"])
146             else:
147                 if req:
148                     raise WorkflowException(u"Container image {} not "
149                                             "found".format(r["dockerImageId"]))
150 
151         return None
152 
153     def add_volumes(self, pathmapper, runtime, stage_output):
154         # type: (PathMapper, List[Text], bool) -> None
155 
156         host_outdir = self.outdir
157         container_outdir = self.builder.outdir
158         for _, vol in pathmapper.items():
159             if not vol.staged:
160                 continue
161             if stage_output and not vol.target.startswith(container_outdir):
162                 containertgt = container_outdir + vol.target[len(host_outdir):]
163             else:
164                 containertgt = vol.target
165             if vol.target.startswith(container_outdir + "/"):
166                 host_outdir_tgt = os.path.join(
167                     host_outdir, vol.target[len(container_outdir) + 1:])
168             else:
169                 host_outdir_tgt = None
170             if vol.type in ("File", "Directory"):
171                 if not vol.resolved.startswith("_:"):
172                     runtime.append(u"--bind")
173                     runtime.append("{}:{}:ro".format(
174                         docker_windows_path_adjust(vol.resolved),
175                         docker_windows_path_adjust(containertgt)))
176             elif vol.type == "WritableFile":
177                 if self.inplace_update:
178                     runtime.append(u"--bind")
179                     runtime.append(u"{}:{}:rw".format(
180                         docker_windows_path_adjust(vol.resolved),
181                         docker_windows_path_adjust(containertgt)))
182                 else:
183                     shutil.copy(vol.resolved, host_outdir_tgt)
184                     ensure_writable(host_outdir_tgt)
185             elif vol.type == "WritableDirectory":
186                 if vol.resolved.startswith("_:"):
187                     os.makedirs(host_outdir_tgt, 0o0755)
188                 else:
189                     if self.inplace_update:
190                         runtime.append(u"--bind")
191                         runtime.append(u"{}:{}:rw".format(
192                             docker_windows_path_adjust(vol.resolved),
193                             docker_windows_path_adjust(containertgt)))
194                     else:
195                         shutil.copytree(vol.resolved, vol.target)
196             elif vol.type == "CreateFile":
197                 createtmp = os.path.join(host_outdir, os.path.basename(vol.target))
198                 with open(createtmp, "wb") as tmp:
199                     tmp.write(vol.resolved.encode("utf-8"))
200                 runtime.append(u"--bind")
201                 runtime.append(u"{}:{}:ro".format(
202                     docker_windows_path_adjust(createtmp),
203                     docker_windows_path_adjust(vol.target)))
204 
205     def create_runtime(self,
206                        env,                        # type: MutableMapping[Text, Text]
207                        rm_container=True,          # type: bool
208                        record_container_id=False,  # type: bool
209                        cidfile_dir="",             # type: Text
210                        cidfile_prefix="",          # type: Text
211                        **kwargs
212                       ):
213         # type: (...) -> List
214         """ Returns the Singularity runtime list of commands and options."""
215 
216         runtime = [u"singularity", u"--quiet", u"exec", u"--contain", u"--pid",
217                    u"--ipc"]
218         if _singularity_supports_userns():
219             runtime.append(u"--userns")
220         runtime.append(u"--bind")
221         runtime.append(u"{}:{}:rw".format(
222             docker_windows_path_adjust(os.path.realpath(self.outdir)),
223             self.builder.outdir))
224         runtime.append(u"--bind")
225         runtime.append(u"{}:{}:rw".format(
226             docker_windows_path_adjust(os.path.realpath(self.tmpdir)), "/tmp"))
227 
228         self.add_volumes(self.pathmapper, runtime, stage_output=False)
229         if self.generatemapper:
230             self.add_volumes(self.generatemapper, runtime, stage_output=True)
231 
232         runtime.append(u"--pwd")
233         runtime.append("%s" % (docker_windows_path_adjust(self.builder.outdir)))
234 
235         if kwargs.get("custom_net", None) is not None:
236             raise UnsupportedRequirement(
237                 "Singularity implementation does not support custom networking")
238         elif kwargs.get("disable_net", None):
239             runtime.append(u"--net")
240 
241         env["SINGULARITYENV_TMPDIR"] = "/tmp"
242         env["SINGULARITYENV_HOME"] = self.builder.outdir
243 
244         for name, value in self.environment.items():
245             env["SINGULARITYENV_{}".format(name)] = value
246         return runtime
247
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/cwltool/singularity.py b/cwltool/singularity.py
--- a/cwltool/singularity.py
+++ b/cwltool/singularity.py
@@ -76,16 +76,18 @@
             candidates.append(_normalizeImageId(dockerRequirement['dockerImageId']))
 
         # check if Singularity image is available in $SINGULARITY_CACHEDIR
-        for target in ("SINGULARITY_CACHEDIR", "SINGULARITY_PULLFOLDER",
-                os.getcwd()):
-            if target in os.environ:
-                for candidate in candidates:
-                    path = os.path.join(os.environ[target], candidate)
-                    if os.path.isfile(path):
-                        _logger.info("Using local copy of Singularity image "
-                                     "found in {}".format(target))
-                        dockerRequirement["dockerImageId"] = path
-                        found = True
+        targets = [os.getcwd()]
+        for env in ("SINGULARITY_CACHEDIR", "SINGULARITY_PULLFOLDER"):
+            if env in os.environ:
+                targets.append(os.environ[env])
+        for target in targets:
+            for candidate in candidates:
+                path = os.path.join(target, candidate)
+                if os.path.isfile(path):
+                    _logger.info("Using local copy of Singularity image "
+                                 "found in {}".format(target))
+                    dockerRequirement["dockerImageId"] = path
+                    found = True
 
         if (force_pull or not found) and pull_image:
             cmd = []  # type: List[Text]

{"golden_diff": "diff --git a/cwltool/singularity.py b/cwltool/singularity.py\n--- a/cwltool/singularity.py\n+++ b/cwltool/singularity.py\n@@ -76,16 +76,18 @@\n             candidates.append(_normalizeImageId(dockerRequirement['dockerImageId']))\n \n         # check if Singularity image is available in $SINGULARITY_CACHEDIR\n-        for target in (\"SINGULARITY_CACHEDIR\", \"SINGULARITY_PULLFOLDER\",\n-                os.getcwd()):\n-            if target in os.environ:\n-                for candidate in candidates:\n-                    path = os.path.join(os.environ[target], candidate)\n-                    if os.path.isfile(path):\n-                        _logger.info(\"Using local copy of Singularity image \"\n-                                     \"found in {}\".format(target))\n-                        dockerRequirement[\"dockerImageId\"] = path\n-                        found = True\n+        targets = [os.getcwd()]\n+        for env in (\"SINGULARITY_CACHEDIR\", \"SINGULARITY_PULLFOLDER\"):\n+            if env in os.environ:\n+                targets.append(os.environ[env])\n+        for target in targets:\n+            for candidate in candidates:\n+                path = os.path.join(target, candidate)\n+                if os.path.isfile(path):\n+                    _logger.info(\"Using local copy of Singularity image \"\n+                                 \"found in {}\".format(target))\n+                    dockerRequirement[\"dockerImageId\"] = path\n+                    found = True\n \n         if (force_pull or not found) and pull_image:\n             cmd = []  # type: List[Text]\n", "issue": "singularity image pull triggered for every step when using beegfs\n\n", "before_files": [{"content": "\"\"\"Support for executing Docker containers using Singularity.\"\"\"\nfrom __future__ import absolute_import\nimport logging\nimport os\nimport re\nimport shutil\nimport sys\nfrom io import open  # pylint: disable=redefined-builtin\nfrom typing import (Dict, List, Text, Optional, MutableMapping)\nfrom .errors import WorkflowException\nfrom .job import ContainerCommandLineJob\nfrom .pathmapper import PathMapper, ensure_writable\nfrom .process import (UnsupportedRequirement)\nfrom .utils import docker_windows_path_adjust\nfrom schema_salad.sourceline import SourceLine\nif os.name == 'posix' and sys.version_info[0] < 3:\n    from subprocess32 import (check_call, check_output,  # pylint: disable=import-error\n                              CalledProcessError, DEVNULL, PIPE, Popen,\n                              TimeoutExpired)\nelif os.name == 'posix':\n    from subprocess import (check_call, check_output,  # type: ignore\n                            CalledProcessError, DEVNULL, PIPE, Popen,\n                            TimeoutExpired)\nelse:  # we're not on Unix, so none of this matters\n    pass\n\n_logger = logging.getLogger(\"cwltool\")\n_USERNS = None\n\ndef _singularity_supports_userns():  # type: ()->bool\n    global _USERNS  # pylint: disable=global-statement\n    if _USERNS is None:\n        try:\n            result = Popen(\n                [u\"singularity\", u\"exec\", u\"--userns\", u\"/etc\", u\"true\"],\n                stderr=PIPE, stdout=DEVNULL,\n                universal_newlines=True).communicate(timeout=60)[1]\n            _USERNS = \"No valid /bin/sh\" in result\n        except TimeoutExpired:\n            _USERNS = False\n    return _USERNS\n\ndef _normalizeImageId(string):  # type: (Text)->Text\n    candidate = re.sub(pattern=r'([a-z]*://)', repl=r'', string=string)\n    return re.sub(pattern=r'[:/]', repl=r'-', string=candidate) + \".img\"\n\n\nclass SingularityCommandLineJob(ContainerCommandLineJob):\n\n    @staticmethod\n    def get_image(dockerRequirement,  # type: Dict[Text, Text]\n                  pull_image,         # type: bool\n                  force_pull=False    # type: bool\n                 ):\n        # type: (...) -> bool\n        \"\"\"\n        Acquire the software container image in the specified dockerRequirement\n        using Singularity and returns the success as a bool. Updates the\n        provided dockerRequirement with the specific dockerImageId to the full\n        path of the local image, if found. Likewise the\n        dockerRequirement['dockerPull'] is updated to a docker:// URI if needed.\n        \"\"\"\n        found = False\n\n        candidates = []\n\n        if \"dockerImageId\" not in dockerRequirement and \"dockerPull\" in dockerRequirement:\n            match = re.search(pattern=r'([a-z]*://)', string=dockerRequirement[\"dockerPull\"])\n            candidate = _normalizeImageId(dockerRequirement['dockerPull'])\n            candidates.append(candidate)\n            dockerRequirement['dockerImageId'] = candidate\n            if not match:\n                dockerRequirement[\"dockerPull\"] = \"docker://\" + dockerRequirement[\"dockerPull\"]\n        elif \"dockerImageId\" in dockerRequirement:\n            candidates.append(dockerRequirement['dockerImageId'])\n            candidates.append(_normalizeImageId(dockerRequirement['dockerImageId']))\n\n        # check if Singularity image is available in $SINGULARITY_CACHEDIR\n        for target in (\"SINGULARITY_CACHEDIR\", \"SINGULARITY_PULLFOLDER\",\n                os.getcwd()):\n            if target in os.environ:\n                for candidate in candidates:\n                    path = os.path.join(os.environ[target], candidate)\n                    if os.path.isfile(path):\n                        _logger.info(\"Using local copy of Singularity image \"\n                                     \"found in {}\".format(target))\n                        dockerRequirement[\"dockerImageId\"] = path\n                        found = True\n\n        if (force_pull or not found) and pull_image:\n            cmd = []  # type: List[Text]\n            if \"dockerPull\" in dockerRequirement:\n                cmd = [\"singularity\", \"pull\", \"--force\", \"--name\",\n                       str(dockerRequirement[\"dockerImageId\"]),\n                       str(dockerRequirement[\"dockerPull\"])]\n                _logger.info(Text(cmd))\n                check_call(cmd, stdout=sys.stderr)\n                found = True\n            elif \"dockerFile\" in dockerRequirement:\n                raise WorkflowException(SourceLine(\n                    dockerRequirement, 'dockerFile').makeError(\n                    \"dockerFile is not currently supported when using the \"\n                    \"Singularity runtime for Docker containers.\"))\n            elif \"dockerLoad\" in dockerRequirement:\n                raise WorkflowException(SourceLine(\n                    dockerRequirement, 'dockerLoad').makeError(\n                    \"dockerLoad is not currently supported when using the \"\n                    \"Singularity runtime for Docker containers.\"))\n            elif \"dockerImport\" in dockerRequirement:\n                raise WorkflowException(SourceLine(\n                    dockerRequirement, 'dockerImport').makeError(\n                    \"dockerImport is not currently supported when using the \"\n                    \"Singularity runtime for Docker containers.\"))\n\n        return found\n\n    def get_from_requirements(self,\n                              r,                      # type: Optional[Dict[Text, Text]]\n                              req,                    # type: bool\n                              pull_image,             # type: bool\n                              force_pull=False,       # type: bool\n                              tmp_outdir_prefix=None  # type: Text\n                             ):  # type: (...) -> Text\n        \"\"\"\n        Returns the filename of the Singularity image (e.g.\n        hello-world-latest.img).\n        \"\"\"\n\n        if r:\n            errmsg = None\n            try:\n                check_output([\"singularity\", \"--version\"])\n            except CalledProcessError as err:\n                errmsg = \"Cannot execute 'singularity --version' {}\".format(err)\n            except OSError as err:\n                errmsg = \"'singularity' executable not found: {}\".format(err)\n\n            if errmsg:\n                if req:\n                    raise WorkflowException(errmsg)\n                else:\n                    return None\n\n            if self.get_image(r, pull_image, force_pull):\n                return os.path.abspath(r[\"dockerImageId\"])\n            else:\n                if req:\n                    raise WorkflowException(u\"Container image {} not \"\n                                            \"found\".format(r[\"dockerImageId\"]))\n\n        return None\n\n    def add_volumes(self, pathmapper, runtime, stage_output):\n        # type: (PathMapper, List[Text], bool) -> None\n\n        host_outdir = self.outdir\n        container_outdir = self.builder.outdir\n        for _, vol in pathmapper.items():\n            if not vol.staged:\n                continue\n            if stage_output and not vol.target.startswith(container_outdir):\n                containertgt = container_outdir + vol.target[len(host_outdir):]\n            else:\n                containertgt = vol.target\n            if vol.target.startswith(container_outdir + \"/\"):\n                host_outdir_tgt = os.path.join(\n                    host_outdir, vol.target[len(container_outdir) + 1:])\n            else:\n                host_outdir_tgt = None\n            if vol.type in (\"File\", \"Directory\"):\n                if not vol.resolved.startswith(\"_:\"):\n                    runtime.append(u\"--bind\")\n                    runtime.append(\"{}:{}:ro\".format(\n                        docker_windows_path_adjust(vol.resolved),\n                        docker_windows_path_adjust(containertgt)))\n            elif vol.type == \"WritableFile\":\n                if self.inplace_update:\n                    runtime.append(u\"--bind\")\n                    runtime.append(u\"{}:{}:rw\".format(\n                        docker_windows_path_adjust(vol.resolved),\n                        docker_windows_path_adjust(containertgt)))\n                else:\n                    shutil.copy(vol.resolved, host_outdir_tgt)\n                    ensure_writable(host_outdir_tgt)\n            elif vol.type == \"WritableDirectory\":\n                if vol.resolved.startswith(\"_:\"):\n                    os.makedirs(host_outdir_tgt, 0o0755)\n                else:\n                    if self.inplace_update:\n                        runtime.append(u\"--bind\")\n                        runtime.append(u\"{}:{}:rw\".format(\n                            docker_windows_path_adjust(vol.resolved),\n                            docker_windows_path_adjust(containertgt)))\n                    else:\n                        shutil.copytree(vol.resolved, vol.target)\n            elif vol.type == \"CreateFile\":\n                createtmp = os.path.join(host_outdir, os.path.basename(vol.target))\n                with open(createtmp, \"wb\") as tmp:\n                    tmp.write(vol.resolved.encode(\"utf-8\"))\n                runtime.append(u\"--bind\")\n                runtime.append(u\"{}:{}:ro\".format(\n                    docker_windows_path_adjust(createtmp),\n                    docker_windows_path_adjust(vol.target)))\n\n    def create_runtime(self,\n                       env,                        # type: MutableMapping[Text, Text]\n                       rm_container=True,          # type: bool\n                       record_container_id=False,  # type: bool\n                       cidfile_dir=\"\",             # type: Text\n                       cidfile_prefix=\"\",          # type: Text\n                       **kwargs\n                      ):\n        # type: (...) -> List\n        \"\"\" Returns the Singularity runtime list of commands and options.\"\"\"\n\n        runtime = [u\"singularity\", u\"--quiet\", u\"exec\", u\"--contain\", u\"--pid\",\n                   u\"--ipc\"]\n        if _singularity_supports_userns():\n            runtime.append(u\"--userns\")\n        runtime.append(u\"--bind\")\n        runtime.append(u\"{}:{}:rw\".format(\n            docker_windows_path_adjust(os.path.realpath(self.outdir)),\n            self.builder.outdir))\n        runtime.append(u\"--bind\")\n        runtime.append(u\"{}:{}:rw\".format(\n            docker_windows_path_adjust(os.path.realpath(self.tmpdir)), \"/tmp\"))\n\n        self.add_volumes(self.pathmapper, runtime, stage_output=False)\n        if self.generatemapper:\n            self.add_volumes(self.generatemapper, runtime, stage_output=True)\n\n        runtime.append(u\"--pwd\")\n        runtime.append(\"%s\" % (docker_windows_path_adjust(self.builder.outdir)))\n\n        if kwargs.get(\"custom_net\", None) is not None:\n            raise UnsupportedRequirement(\n                \"Singularity implementation does not support custom networking\")\n        elif kwargs.get(\"disable_net\", None):\n            runtime.append(u\"--net\")\n\n        env[\"SINGULARITYENV_TMPDIR\"] = \"/tmp\"\n        env[\"SINGULARITYENV_HOME\"] = self.builder.outdir\n\n        for name, value in self.environment.items():\n            env[\"SINGULARITYENV_{}\".format(name)] = value\n        return runtime\n", "path": "cwltool/singularity.py"}], "after_files": [{"content": "\"\"\"Support for executing Docker containers using Singularity.\"\"\"\nfrom __future__ import absolute_import\nimport logging\nimport os\nimport re\nimport shutil\nimport sys\nfrom io import open  # pylint: disable=redefined-builtin\nfrom typing import (Dict, List, Text, Optional, MutableMapping)\nfrom .errors import WorkflowException\nfrom .job import ContainerCommandLineJob\nfrom .pathmapper import PathMapper, ensure_writable\nfrom .process import (UnsupportedRequirement)\nfrom .utils import docker_windows_path_adjust\nfrom schema_salad.sourceline import SourceLine\nif os.name == 'posix' and sys.version_info[0] < 3:\n    from subprocess32 import (check_call, check_output,  # pylint: disable=import-error\n                              CalledProcessError, DEVNULL, PIPE, Popen,\n                              TimeoutExpired)\nelif os.name == 'posix':\n    from subprocess import (check_call, check_output,  # type: ignore\n                            CalledProcessError, DEVNULL, PIPE, Popen,\n                            TimeoutExpired)\nelse:  # we're not on Unix, so none of this matters\n    pass\n\n_logger = logging.getLogger(\"cwltool\")\n_USERNS = None\n\ndef _singularity_supports_userns():  # type: ()->bool\n    global _USERNS  # pylint: disable=global-statement\n    if _USERNS is None:\n        try:\n            result = Popen(\n                [u\"singularity\", u\"exec\", u\"--userns\", u\"/etc\", u\"true\"],\n                stderr=PIPE, stdout=DEVNULL,\n                universal_newlines=True).communicate(timeout=60)[1]\n            _USERNS = \"No valid /bin/sh\" in result\n        except TimeoutExpired:\n            _USERNS = False\n    return _USERNS\n\ndef _normalizeImageId(string):  # type: (Text)->Text\n    candidate = re.sub(pattern=r'([a-z]*://)', repl=r'', string=string)\n    return re.sub(pattern=r'[:/]', repl=r'-', string=candidate) + \".img\"\n\n\nclass SingularityCommandLineJob(ContainerCommandLineJob):\n\n    @staticmethod\n    def get_image(dockerRequirement,  # type: Dict[Text, Text]\n                  pull_image,         # type: bool\n                  force_pull=False    # type: bool\n                 ):\n        # type: (...) -> bool\n        \"\"\"\n        Acquire the software container image in the specified dockerRequirement\n        using Singularity and returns the success as a bool. Updates the\n        provided dockerRequirement with the specific dockerImageId to the full\n        path of the local image, if found. Likewise the\n        dockerRequirement['dockerPull'] is updated to a docker:// URI if needed.\n        \"\"\"\n        found = False\n\n        candidates = []\n\n        if \"dockerImageId\" not in dockerRequirement and \"dockerPull\" in dockerRequirement:\n            match = re.search(pattern=r'([a-z]*://)', string=dockerRequirement[\"dockerPull\"])\n            candidate = _normalizeImageId(dockerRequirement['dockerPull'])\n            candidates.append(candidate)\n            dockerRequirement['dockerImageId'] = candidate\n            if not match:\n                dockerRequirement[\"dockerPull\"] = \"docker://\" + dockerRequirement[\"dockerPull\"]\n        elif \"dockerImageId\" in dockerRequirement:\n            candidates.append(dockerRequirement['dockerImageId'])\n            candidates.append(_normalizeImageId(dockerRequirement['dockerImageId']))\n\n        # check if Singularity image is available in $SINGULARITY_CACHEDIR\n        targets = [os.getcwd()]\n        for env in (\"SINGULARITY_CACHEDIR\", \"SINGULARITY_PULLFOLDER\"):\n            if env in os.environ:\n                targets.append(os.environ[env])\n        for target in targets:\n            for candidate in candidates:\n                path = os.path.join(target, candidate)\n                if os.path.isfile(path):\n                    _logger.info(\"Using local copy of Singularity image \"\n                                 \"found in {}\".format(target))\n                    dockerRequirement[\"dockerImageId\"] = path\n                    found = True\n\n        if (force_pull or not found) and pull_image:\n            cmd = []  # type: List[Text]\n            if \"dockerPull\" in dockerRequirement:\n                cmd = [\"singularity\", \"pull\", \"--force\", \"--name\",\n                       str(dockerRequirement[\"dockerImageId\"]),\n                       str(dockerRequirement[\"dockerPull\"])]\n                _logger.info(Text(cmd))\n                check_call(cmd, stdout=sys.stderr)\n                found = True\n            elif \"dockerFile\" in dockerRequirement:\n                raise WorkflowException(SourceLine(\n                    dockerRequirement, 'dockerFile').makeError(\n                    \"dockerFile is not currently supported when using the \"\n                    \"Singularity runtime for Docker containers.\"))\n            elif \"dockerLoad\" in dockerRequirement:\n                raise WorkflowException(SourceLine(\n                    dockerRequirement, 'dockerLoad').makeError(\n                    \"dockerLoad is not currently supported when using the \"\n                    \"Singularity runtime for Docker containers.\"))\n            elif \"dockerImport\" in dockerRequirement:\n                raise WorkflowException(SourceLine(\n                    dockerRequirement, 'dockerImport').makeError(\n                    \"dockerImport is not currently supported when using the \"\n                    \"Singularity runtime for Docker containers.\"))\n\n        return found\n\n    def get_from_requirements(self,\n                              r,                      # type: Optional[Dict[Text, Text]]\n                              req,                    # type: bool\n                              pull_image,             # type: bool\n                              force_pull=False,       # type: bool\n                              tmp_outdir_prefix=None  # type: Text\n                             ):  # type: (...) -> Text\n        \"\"\"\n        Returns the filename of the Singularity image (e.g.\n        hello-world-latest.img).\n        \"\"\"\n\n        if r:\n            errmsg = None\n            try:\n                check_output([\"singularity\", \"--version\"])\n            except CalledProcessError as err:\n                errmsg = \"Cannot execute 'singularity --version' {}\".format(err)\n            except OSError as err:\n                errmsg = \"'singularity' executable not found: {}\".format(err)\n\n            if errmsg:\n                if req:\n                    raise WorkflowException(errmsg)\n                else:\n                    return None\n\n            if self.get_image(r, pull_image, force_pull):\n                return os.path.abspath(r[\"dockerImageId\"])\n            else:\n                if req:\n                    raise WorkflowException(u\"Container image {} not \"\n                                            \"found\".format(r[\"dockerImageId\"]))\n\n        return None\n\n    def add_volumes(self, pathmapper, runtime, stage_output):\n        # type: (PathMapper, List[Text], bool) -> None\n\n        host_outdir = self.outdir\n        container_outdir = self.builder.outdir\n        for _, vol in pathmapper.items():\n            if not vol.staged:\n                continue\n            if stage_output and not vol.target.startswith(container_outdir):\n                containertgt = container_outdir + vol.target[len(host_outdir):]\n            else:\n                containertgt = vol.target\n            if vol.target.startswith(container_outdir + \"/\"):\n                host_outdir_tgt = os.path.join(\n                    host_outdir, vol.target[len(container_outdir) + 1:])\n            else:\n                host_outdir_tgt = None\n            if vol.type in (\"File\", \"Directory\"):\n                if not vol.resolved.startswith(\"_:\"):\n                    runtime.append(u\"--bind\")\n                    runtime.append(\"{}:{}:ro\".format(\n                        docker_windows_path_adjust(vol.resolved),\n                        docker_windows_path_adjust(containertgt)))\n            elif vol.type == \"WritableFile\":\n                if self.inplace_update:\n                    runtime.append(u\"--bind\")\n                    runtime.append(u\"{}:{}:rw\".format(\n                        docker_windows_path_adjust(vol.resolved),\n                        docker_windows_path_adjust(containertgt)))\n                else:\n                    shutil.copy(vol.resolved, host_outdir_tgt)\n                    ensure_writable(host_outdir_tgt)\n            elif vol.type == \"WritableDirectory\":\n                if vol.resolved.startswith(\"_:\"):\n                    os.makedirs(host_outdir_tgt, 0o0755)\n                else:\n                    if self.inplace_update:\n                        runtime.append(u\"--bind\")\n                        runtime.append(u\"{}:{}:rw\".format(\n                            docker_windows_path_adjust(vol.resolved),\n                            docker_windows_path_adjust(containertgt)))\n                    else:\n                        shutil.copytree(vol.resolved, vol.target)\n            elif vol.type == \"CreateFile\":\n                createtmp = os.path.join(host_outdir, os.path.basename(vol.target))\n                with open(createtmp, \"wb\") as tmp:\n                    tmp.write(vol.resolved.encode(\"utf-8\"))\n                runtime.append(u\"--bind\")\n                runtime.append(u\"{}:{}:ro\".format(\n                    docker_windows_path_adjust(createtmp),\n                    docker_windows_path_adjust(vol.target)))\n\n    def create_runtime(self,\n                       env,                        # type: MutableMapping[Text, Text]\n                       rm_container=True,          # type: bool\n                       record_container_id=False,  # type: bool\n                       cidfile_dir=\"\",             # type: Text\n                       cidfile_prefix=\"\",          # type: Text\n                       **kwargs\n                      ):\n        # type: (...) -> List\n        \"\"\" Returns the Singularity runtime list of commands and options.\"\"\"\n\n        runtime = [u\"singularity\", u\"--quiet\", u\"exec\", u\"--contain\", u\"--pid\",\n                   u\"--ipc\"]\n        if _singularity_supports_userns():\n            runtime.append(u\"--userns\")\n        runtime.append(u\"--bind\")\n        runtime.append(u\"{}:{}:rw\".format(\n            docker_windows_path_adjust(os.path.realpath(self.outdir)),\n            self.builder.outdir))\n        runtime.append(u\"--bind\")\n        runtime.append(u\"{}:{}:rw\".format(\n            docker_windows_path_adjust(os.path.realpath(self.tmpdir)), \"/tmp\"))\n\n        self.add_volumes(self.pathmapper, runtime, stage_output=False)\n        if self.generatemapper:\n            self.add_volumes(self.generatemapper, runtime, stage_output=True)\n\n        runtime.append(u\"--pwd\")\n        runtime.append(\"%s\" % (docker_windows_path_adjust(self.builder.outdir)))\n\n        if kwargs.get(\"custom_net\", None) is not None:\n            raise UnsupportedRequirement(\n                \"Singularity implementation does not support custom networking\")\n        elif kwargs.get(\"disable_net\", None):\n            runtime.append(u\"--net\")\n\n        env[\"SINGULARITYENV_TMPDIR\"] = \"/tmp\"\n        env[\"SINGULARITYENV_HOME\"] = self.builder.outdir\n\n        for name, value in self.environment.items():\n            env[\"SINGULARITYENV_{}\".format(name)] = value\n        return runtime\n", "path": "cwltool/singularity.py"}]}