We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Download: Also fetch institutional configs
Now that the institute config files are held in their own repo (see #212), they will not be available for users running offline.

Should be an easy enough fix though - the `nf-core download` command needs to be extended to also download the configs repo and set the base path for the import correctly.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `nf_core/download.py`
Content:
```
1 #!/usr/bin/env python
2 """Downloads a nf-core pipeline to the local file system."""
3 
4 from __future__ import print_function
5 
6 import errno
7 from io import BytesIO
8 import logging
9 import hashlib
10 import os
11 import requests
12 import subprocess
13 import sys
14 from zipfile import ZipFile
15 
16 import nf_core.list
17 import nf_core.utils
18 
19 
20 class DownloadWorkflow(object):
21     """Downloads a nf-core workflow from Github to the local file system.
22 
23     Can also download its Singularity container image if required.
24 
25     Args:
26         pipeline (str): A nf-core pipeline name.
27         release (str): The workflow release version to download, like `1.0`. Defaults to None.
28         singularity (bool): Flag, if the Singularity container should be downloaded as well. Defaults to False.
29         outdir (str): Path to the local download directory. Defaults to None.
30     """
31     def __init__(self, pipeline, release=None, singularity=False, outdir=None):
32         self.pipeline = pipeline
33         self.release = release
34         self.singularity = singularity
35         self.outdir = outdir
36 
37         self.wf_name = None
38         self.wf_sha = None
39         self.wf_download_url = None
40         self.config = dict()
41         self.containers = list()
42 
43     def download_workflow(self):
44         """Starts a nf-core workflow download."""
45         # Get workflow details
46         try:
47             self.fetch_workflow_details(nf_core.list.Workflows())
48         except LookupError:
49             sys.exit(1)
50 
51         # Check that the outdir doesn't already exist
52         if os.path.exists(self.outdir):
53             logging.error("Output directory '{}' already exists".format(self.outdir))
54             sys.exit(1)
55 
56         logging.info(
57             "Saving {}".format(self.pipeline) +
58             "\n Pipeline release: {}".format(self.release) +
59             "\n Pull singularity containers: {}".format('Yes' if self.singularity else 'No') +
60             "\n Output directory: {}".format(self.outdir)
61         )
62 
63         # Download the pipeline files
64         logging.info("Downloading workflow files from GitHub")
65         self.download_wf_files()
66 
67         # Download the singularity images
68         if self.singularity:
69             logging.debug("Fetching container names for workflow")
70             self.find_container_images()
71             if len(self.containers) == 0:
72                 logging.info("No container names found in workflow")
73             else:
74                 os.mkdir(os.path.join(self.outdir, 'singularity-images'))
75                 logging.info("Downloading {} singularity container{}".format(len(self.containers), 's' if len(self.containers) > 1 else ''))
76                 for container in self.containers:
77                     try:
78                         # Download from Dockerhub in all cases
79                         self.pull_singularity_image(container)
80                     except RuntimeWarning as r:
81                         # Raise exception if this is not possible
82                         logging.error("Not able to pull image. Service might be down or internet connection is dead.")
83                         raise r
84 
85 
86 
87     def fetch_workflow_details(self, wfs):
88         """Fetches details of a nf-core workflow to download.
89 
90         Args:
91             wfs (nf_core.list.Workflows): A nf_core.list.Workflows object
92 
93         Raises:
94             LockupError, if the pipeline can not be found.
95         """
96         wfs.get_remote_workflows()
97 
98         # Get workflow download details
99         for wf in wfs.remote_workflows:
100             if wf.full_name == self.pipeline or wf.name == self.pipeline:
101 
102                 # Set pipeline name
103                 self.wf_name = wf.name
104 
105                 # Find latest release hash
106                 if self.release is None and len(wf.releases) > 0:
107                     # Sort list of releases so that most recent is first
108                     wf.releases = sorted(wf.releases, key=lambda k: k.get('published_at_timestamp', 0), reverse=True)
109                     self.release = wf.releases[0]['tag_name']
110                     self.wf_sha = wf.releases[0]['tag_sha']
111                     logging.debug("No release specified. Using latest release: {}".format(self.release))
112                 # Find specified release hash
113                 elif self.release is not None:
114                     for r in wf.releases:
115                         if r['tag_name'] == self.release.lstrip('v'):
116                             self.wf_sha = r['tag_sha']
117                             break
118                     else:
119                         logging.error("Not able to find release '{}' for {}".format(self.release, wf.full_name))
120                         logging.info("Available {} releases: {}".format(wf.full_name, ', '.join([r['tag_name'] for r in wf.releases])))
121                         raise LookupError("Not able to find release '{}' for {}".format(self.release, wf.full_name))
122 
123                 # Must be a dev-only pipeline
124                 elif not self.release:
125                     self.release = 'dev'
126                     self.wf_sha = 'master' # Cheating a little, but GitHub download link works
127                     logging.warn("Pipeline is in development - downloading current code on master branch.\n" +
128                         "This is likely to change soon should not be considered fully reproducible.")
129 
130                 # Set outdir name if not defined
131                 if not self.outdir:
132                     self.outdir = 'nf-core-{}'.format(wf.name)
133                     if self.release is not None:
134                         self.outdir += '-{}'.format(self.release)
135 
136                 # Set the download URL and return
137                 self.wf_download_url = 'https://github.com/{}/archive/{}.zip'.format(wf.full_name, self.wf_sha)
138                 return
139 
140         # If we got this far, must not be a nf-core pipeline
141         if self.pipeline.count('/') == 1:
142             # Looks like a GitHub address - try working with this repo
143             logging.warn("Pipeline name doesn't match any nf-core workflows")
144             logging.info("Pipeline name looks like a GitHub address - attempting to download anyway")
145             self.wf_name = self.pipeline
146             if not self.release:
147                 self.release = 'master'
148             self.wf_sha = self.release
149             if not self.outdir:
150                 self.outdir = self.pipeline.replace('/', '-').lower()
151                 if self.release is not None:
152                     self.outdir += '-{}'.format(self.release)
153             # Set the download URL and return
154             self.wf_download_url = 'https://github.com/{}/archive/{}.zip'.format(self.pipeline, self.release)
155         else:
156             logging.error("Not able to find pipeline '{}'".format(self.pipeline))
157             logging.info("Available pipelines: {}".format(', '.join([w.name for w in wfs.remote_workflows])))
158             raise LookupError("Not able to find pipeline '{}'".format(self.pipeline))
159 
160     def download_wf_files(self):
161         """Downloads workflow files from Github to the :attr:`self.outdir`.
162         """
163         logging.debug("Downloading {}".format(self.wf_download_url))
164 
165         # Download GitHub zip file into memory and extract
166         url = requests.get(self.wf_download_url)
167         zipfile = ZipFile(BytesIO(url.content))
168         zipfile.extractall(self.outdir)
169 
170         # Rename the internal directory name to be more friendly
171         gh_name = '{}-{}'.format(self.wf_name, self.wf_sha).split('/')[-1]
172         os.rename(os.path.join(self.outdir, gh_name), os.path.join(self.outdir, 'workflow'))
173 
174         # Make downloaded files executable
175         for dirpath, subdirs, filelist in os.walk(os.path.join(self.outdir, 'workflow')):
176             for fname in filelist:
177                 os.chmod(os.path.join(dirpath, fname), 0o775)
178 
179     def find_container_images(self):
180         """ Find container image names for workflow """
181 
182         # Use linting code to parse the pipeline nextflow config
183         self.config = nf_core.utils.fetch_wf_config(os.path.join(self.outdir, 'workflow'))
184 
185         # Find any config variables that look like a container
186         for k,v in self.config.items():
187             if k.startswith('process.') and k.endswith('.container'):
188                 self.containers.append(v.strip('"').strip("'"))
189 
190 
191     def pull_singularity_image(self, container):
192         """Uses a local installation of singularity to pull an image from Docker Hub.
193 
194         Args:
195             container (str): A pipeline's container name. Usually it is of similar format
196                 to `nfcore/name:dev`.
197 
198         Raises:
199             Various exceptions possible from `subprocess` execution of Singularity.
200         """
201         out_name = '{}.simg'.format(container.replace('nfcore', 'nf-core').replace('/','-').replace(':', '-'))
202         out_path = os.path.abspath(os.path.join(self.outdir, 'singularity-images', out_name))
203         address = 'docker://{}'.format(container.replace('docker://', ''))
204         singularity_command = ["singularity", "pull", "--name", out_path, address]
205         logging.info("Building singularity image from dockerhub: {}".format(address))
206         logging.debug("Singularity command: {}".format(' '.join(singularity_command)))
207 
208         # Try to use singularity to pull image
209         try:
210             subprocess.call(singularity_command)
211         except OSError as e:
212             if e.errno == errno.ENOENT:
213                 # Singularity is not installed
214                 logging.error('Singularity is not installed!')
215             else:
216                 # Something else went wrong with singularity command
217                 raise e
218 
219     def validate_md5(self, fname, expected):
220         """Calculates the md5sum for a file on the disk and validate with expected.
221 
222         Args:
223             fname (str): Path to a local file.
224             expected (str): The expected md5sum.
225 
226         Raises:
227             IOError, if the md5sum does not match the remote sum.
228         """
229         logging.debug("Validating image hash: {}".format(fname))
230 
231         # Calculate the md5 for the file on disk
232         hash_md5 = hashlib.md5()
233         with open(fname, "rb") as f:
234             for chunk in iter(lambda: f.read(4096), b""):
235                 hash_md5.update(chunk)
236         file_hash = hash_md5.hexdigest()
237 
238         if file_hash == expected:
239             logging.debug('md5 sum of image matches expected: {}'.format(expected))
240         else:
241             raise IOError ("{} md5 does not match remote: {} - {}".format(fname, expected, file_hash))
242
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/nf_core/download.py b/nf_core/download.py
--- a/nf_core/download.py
+++ b/nf_core/download.py
@@ -64,6 +64,11 @@
         logging.info("Downloading workflow files from GitHub")
         self.download_wf_files()
 
+        # Download the centralised configs
+        logging.info("Downloading centralised configs from GitHub")
+        self.download_configs()
+        self.wf_use_local_configs()
+
         # Download the singularity images
         if self.singularity:
             logging.debug("Fetching container names for workflow")
@@ -176,6 +181,46 @@
             for fname in filelist:
                 os.chmod(os.path.join(dirpath, fname), 0o775)
 
+    def download_configs(self):
+        """Downloads the centralised config profiles from nf-core/configs to :attr:`self.outdir`.
+        """
+        configs_zip_url = "https://github.com/nf-core/configs/archive/master.zip"
+        configs_local_dir = "configs-master"
+        logging.debug("Downloading {}".format(configs_zip_url))
+
+        # Download GitHub zip file into memory and extract
+        url = requests.get(configs_zip_url)
+        zipfile = ZipFile(BytesIO(url.content))
+        zipfile.extractall(self.outdir)
+
+        # Rename the internal directory name to be more friendly
+        os.rename(os.path.join(self.outdir, configs_local_dir), os.path.join(self.outdir, 'configs'))
+
+        # Make downloaded files executable
+        for dirpath, subdirs, filelist in os.walk(os.path.join(self.outdir, 'configs')):
+            for fname in filelist:
+                os.chmod(os.path.join(dirpath, fname), 0o775)
+
+    def wf_use_local_configs(self):
+        """Edit the downloaded nextflow.config file to use the local config files
+        """
+        nfconfig_fn = os.path.join(self.outdir, 'workflow', 'nextflow.config')
+        find_str = 'https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}'
+        repl_str = '../configs/'
+        logging.debug("Editing params.custom_config_base in {}".format(nfconfig_fn))
+
+        # Load the nextflow.config file into memory
+        with open(nfconfig_fn, 'r') as nfconfig_fh:
+          nfconfig = nfconfig_fh.read()
+
+        # Replace the target string
+        nfconfig = nfconfig.replace(find_str, repl_str)
+
+        # Write the file out again
+        with open(nfconfig_fn, 'w') as nfconfig_fh:
+          nfconfig_fh.write(nfconfig)
+
+
     def find_container_images(self):
         """ Find container image names for workflow """

{"golden_diff": "diff --git a/nf_core/download.py b/nf_core/download.py\n--- a/nf_core/download.py\n+++ b/nf_core/download.py\n@@ -64,6 +64,11 @@\n         logging.info(\"Downloading workflow files from GitHub\")\n         self.download_wf_files()\n \n+        # Download the centralised configs\n+        logging.info(\"Downloading centralised configs from GitHub\")\n+        self.download_configs()\n+        self.wf_use_local_configs()\n+\n         # Download the singularity images\n         if self.singularity:\n             logging.debug(\"Fetching container names for workflow\")\n@@ -176,6 +181,46 @@\n             for fname in filelist:\n                 os.chmod(os.path.join(dirpath, fname), 0o775)\n \n+    def download_configs(self):\n+        \"\"\"Downloads the centralised config profiles from nf-core/configs to :attr:`self.outdir`.\n+        \"\"\"\n+        configs_zip_url = \"https://github.com/nf-core/configs/archive/master.zip\"\n+        configs_local_dir = \"configs-master\"\n+        logging.debug(\"Downloading {}\".format(configs_zip_url))\n+\n+        # Download GitHub zip file into memory and extract\n+        url = requests.get(configs_zip_url)\n+        zipfile = ZipFile(BytesIO(url.content))\n+        zipfile.extractall(self.outdir)\n+\n+        # Rename the internal directory name to be more friendly\n+        os.rename(os.path.join(self.outdir, configs_local_dir), os.path.join(self.outdir, 'configs'))\n+\n+        # Make downloaded files executable\n+        for dirpath, subdirs, filelist in os.walk(os.path.join(self.outdir, 'configs')):\n+            for fname in filelist:\n+                os.chmod(os.path.join(dirpath, fname), 0o775)\n+\n+    def wf_use_local_configs(self):\n+        \"\"\"Edit the downloaded nextflow.config file to use the local config files\n+        \"\"\"\n+        nfconfig_fn = os.path.join(self.outdir, 'workflow', 'nextflow.config')\n+        find_str = 'https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}'\n+        repl_str = '../configs/'\n+        logging.debug(\"Editing params.custom_config_base in {}\".format(nfconfig_fn))\n+\n+        # Load the nextflow.config file into memory\n+        with open(nfconfig_fn, 'r') as nfconfig_fh:\n+          nfconfig = nfconfig_fh.read()\n+\n+        # Replace the target string\n+        nfconfig = nfconfig.replace(find_str, repl_str)\n+\n+        # Write the file out again\n+        with open(nfconfig_fn, 'w') as nfconfig_fh:\n+          nfconfig_fh.write(nfconfig)\n+\n+\n     def find_container_images(self):\n         \"\"\" Find container image names for workflow \"\"\"\n", "issue": "Download: Also fetch institutional configs\nNow that the institute config files are held in their own repo (see #212), they will not be available for users running offline.\r\n\r\nShould be an easy enough fix though - the `nf-core download` command needs to be extended to also download the configs repo and set the base path for the import correctly.\n", "before_files": [{"content": "#!/usr/bin/env python\n\"\"\"Downloads a nf-core pipeline to the local file system.\"\"\"\n\nfrom __future__ import print_function\n\nimport errno\nfrom io import BytesIO\nimport logging\nimport hashlib\nimport os\nimport requests\nimport subprocess\nimport sys\nfrom zipfile import ZipFile\n\nimport nf_core.list\nimport nf_core.utils\n\n\nclass DownloadWorkflow(object):\n    \"\"\"Downloads a nf-core workflow from Github to the local file system.\n\n    Can also download its Singularity container image if required.\n\n    Args:\n        pipeline (str): A nf-core pipeline name.\n        release (str): The workflow release version to download, like `1.0`. Defaults to None.\n        singularity (bool): Flag, if the Singularity container should be downloaded as well. Defaults to False.\n        outdir (str): Path to the local download directory. Defaults to None.\n    \"\"\"\n    def __init__(self, pipeline, release=None, singularity=False, outdir=None):\n        self.pipeline = pipeline\n        self.release = release\n        self.singularity = singularity\n        self.outdir = outdir\n\n        self.wf_name = None\n        self.wf_sha = None\n        self.wf_download_url = None\n        self.config = dict()\n        self.containers = list()\n\n    def download_workflow(self):\n        \"\"\"Starts a nf-core workflow download.\"\"\"\n        # Get workflow details\n        try:\n            self.fetch_workflow_details(nf_core.list.Workflows())\n        except LookupError:\n            sys.exit(1)\n\n        # Check that the outdir doesn't already exist\n        if os.path.exists(self.outdir):\n            logging.error(\"Output directory '{}' already exists\".format(self.outdir))\n            sys.exit(1)\n\n        logging.info(\n            \"Saving {}\".format(self.pipeline) +\n            \"\\n Pipeline release: {}\".format(self.release) +\n            \"\\n Pull singularity containers: {}\".format('Yes' if self.singularity else 'No') +\n            \"\\n Output directory: {}\".format(self.outdir)\n        )\n\n        # Download the pipeline files\n        logging.info(\"Downloading workflow files from GitHub\")\n        self.download_wf_files()\n\n        # Download the singularity images\n        if self.singularity:\n            logging.debug(\"Fetching container names for workflow\")\n            self.find_container_images()\n            if len(self.containers) == 0:\n                logging.info(\"No container names found in workflow\")\n            else:\n                os.mkdir(os.path.join(self.outdir, 'singularity-images'))\n                logging.info(\"Downloading {} singularity container{}\".format(len(self.containers), 's' if len(self.containers) > 1 else ''))\n                for container in self.containers:\n                    try:\n                        # Download from Dockerhub in all cases\n                        self.pull_singularity_image(container)\n                    except RuntimeWarning as r:\n                        # Raise exception if this is not possible\n                        logging.error(\"Not able to pull image. Service might be down or internet connection is dead.\")\n                        raise r\n\n\n\n    def fetch_workflow_details(self, wfs):\n        \"\"\"Fetches details of a nf-core workflow to download.\n\n        Args:\n            wfs (nf_core.list.Workflows): A nf_core.list.Workflows object\n\n        Raises:\n            LockupError, if the pipeline can not be found.\n        \"\"\"\n        wfs.get_remote_workflows()\n\n        # Get workflow download details\n        for wf in wfs.remote_workflows:\n            if wf.full_name == self.pipeline or wf.name == self.pipeline:\n\n                # Set pipeline name\n                self.wf_name = wf.name\n\n                # Find latest release hash\n                if self.release is None and len(wf.releases) > 0:\n                    # Sort list of releases so that most recent is first\n                    wf.releases = sorted(wf.releases, key=lambda k: k.get('published_at_timestamp', 0), reverse=True)\n                    self.release = wf.releases[0]['tag_name']\n                    self.wf_sha = wf.releases[0]['tag_sha']\n                    logging.debug(\"No release specified. Using latest release: {}\".format(self.release))\n                # Find specified release hash\n                elif self.release is not None:\n                    for r in wf.releases:\n                        if r['tag_name'] == self.release.lstrip('v'):\n                            self.wf_sha = r['tag_sha']\n                            break\n                    else:\n                        logging.error(\"Not able to find release '{}' for {}\".format(self.release, wf.full_name))\n                        logging.info(\"Available {} releases: {}\".format(wf.full_name, ', '.join([r['tag_name'] for r in wf.releases])))\n                        raise LookupError(\"Not able to find release '{}' for {}\".format(self.release, wf.full_name))\n\n                # Must be a dev-only pipeline\n                elif not self.release:\n                    self.release = 'dev'\n                    self.wf_sha = 'master' # Cheating a little, but GitHub download link works\n                    logging.warn(\"Pipeline is in development - downloading current code on master branch.\\n\" +\n                        \"This is likely to change soon should not be considered fully reproducible.\")\n\n                # Set outdir name if not defined\n                if not self.outdir:\n                    self.outdir = 'nf-core-{}'.format(wf.name)\n                    if self.release is not None:\n                        self.outdir += '-{}'.format(self.release)\n\n                # Set the download URL and return\n                self.wf_download_url = 'https://github.com/{}/archive/{}.zip'.format(wf.full_name, self.wf_sha)\n                return\n\n        # If we got this far, must not be a nf-core pipeline\n        if self.pipeline.count('/') == 1:\n            # Looks like a GitHub address - try working with this repo\n            logging.warn(\"Pipeline name doesn't match any nf-core workflows\")\n            logging.info(\"Pipeline name looks like a GitHub address - attempting to download anyway\")\n            self.wf_name = self.pipeline\n            if not self.release:\n                self.release = 'master'\n            self.wf_sha = self.release\n            if not self.outdir:\n                self.outdir = self.pipeline.replace('/', '-').lower()\n                if self.release is not None:\n                    self.outdir += '-{}'.format(self.release)\n            # Set the download URL and return\n            self.wf_download_url = 'https://github.com/{}/archive/{}.zip'.format(self.pipeline, self.release)\n        else:\n            logging.error(\"Not able to find pipeline '{}'\".format(self.pipeline))\n            logging.info(\"Available pipelines: {}\".format(', '.join([w.name for w in wfs.remote_workflows])))\n            raise LookupError(\"Not able to find pipeline '{}'\".format(self.pipeline))\n\n    def download_wf_files(self):\n        \"\"\"Downloads workflow files from Github to the :attr:`self.outdir`.\n        \"\"\"\n        logging.debug(\"Downloading {}\".format(self.wf_download_url))\n\n        # Download GitHub zip file into memory and extract\n        url = requests.get(self.wf_download_url)\n        zipfile = ZipFile(BytesIO(url.content))\n        zipfile.extractall(self.outdir)\n\n        # Rename the internal directory name to be more friendly\n        gh_name = '{}-{}'.format(self.wf_name, self.wf_sha).split('/')[-1]\n        os.rename(os.path.join(self.outdir, gh_name), os.path.join(self.outdir, 'workflow'))\n\n        # Make downloaded files executable\n        for dirpath, subdirs, filelist in os.walk(os.path.join(self.outdir, 'workflow')):\n            for fname in filelist:\n                os.chmod(os.path.join(dirpath, fname), 0o775)\n\n    def find_container_images(self):\n        \"\"\" Find container image names for workflow \"\"\"\n\n        # Use linting code to parse the pipeline nextflow config\n        self.config = nf_core.utils.fetch_wf_config(os.path.join(self.outdir, 'workflow'))\n\n        # Find any config variables that look like a container\n        for k,v in self.config.items():\n            if k.startswith('process.') and k.endswith('.container'):\n                self.containers.append(v.strip('\"').strip(\"'\"))\n\n\n    def pull_singularity_image(self, container):\n        \"\"\"Uses a local installation of singularity to pull an image from Docker Hub.\n\n        Args:\n            container (str): A pipeline's container name. Usually it is of similar format\n                to `nfcore/name:dev`.\n\n        Raises:\n            Various exceptions possible from `subprocess` execution of Singularity.\n        \"\"\"\n        out_name = '{}.simg'.format(container.replace('nfcore', 'nf-core').replace('/','-').replace(':', '-'))\n        out_path = os.path.abspath(os.path.join(self.outdir, 'singularity-images', out_name))\n        address = 'docker://{}'.format(container.replace('docker://', ''))\n        singularity_command = [\"singularity\", \"pull\", \"--name\", out_path, address]\n        logging.info(\"Building singularity image from dockerhub: {}\".format(address))\n        logging.debug(\"Singularity command: {}\".format(' '.join(singularity_command)))\n\n        # Try to use singularity to pull image\n        try:\n            subprocess.call(singularity_command)\n        except OSError as e:\n            if e.errno == errno.ENOENT:\n                # Singularity is not installed\n                logging.error('Singularity is not installed!')\n            else:\n                # Something else went wrong with singularity command\n                raise e\n\n    def validate_md5(self, fname, expected):\n        \"\"\"Calculates the md5sum for a file on the disk and validate with expected.\n\n        Args:\n            fname (str): Path to a local file.\n            expected (str): The expected md5sum.\n\n        Raises:\n            IOError, if the md5sum does not match the remote sum.\n        \"\"\"\n        logging.debug(\"Validating image hash: {}\".format(fname))\n\n        # Calculate the md5 for the file on disk\n        hash_md5 = hashlib.md5()\n        with open(fname, \"rb\") as f:\n            for chunk in iter(lambda: f.read(4096), b\"\"):\n                hash_md5.update(chunk)\n        file_hash = hash_md5.hexdigest()\n\n        if file_hash == expected:\n            logging.debug('md5 sum of image matches expected: {}'.format(expected))\n        else:\n            raise IOError (\"{} md5 does not match remote: {} - {}\".format(fname, expected, file_hash))\n", "path": "nf_core/download.py"}], "after_files": [{"content": "#!/usr/bin/env python\n\"\"\"Downloads a nf-core pipeline to the local file system.\"\"\"\n\nfrom __future__ import print_function\n\nimport errno\nfrom io import BytesIO\nimport logging\nimport hashlib\nimport os\nimport requests\nimport subprocess\nimport sys\nfrom zipfile import ZipFile\n\nimport nf_core.list\nimport nf_core.utils\n\n\nclass DownloadWorkflow(object):\n    \"\"\"Downloads a nf-core workflow from Github to the local file system.\n\n    Can also download its Singularity container image if required.\n\n    Args:\n        pipeline (str): A nf-core pipeline name.\n        release (str): The workflow release version to download, like `1.0`. Defaults to None.\n        singularity (bool): Flag, if the Singularity container should be downloaded as well. Defaults to False.\n        outdir (str): Path to the local download directory. Defaults to None.\n    \"\"\"\n    def __init__(self, pipeline, release=None, singularity=False, outdir=None):\n        self.pipeline = pipeline\n        self.release = release\n        self.singularity = singularity\n        self.outdir = outdir\n\n        self.wf_name = None\n        self.wf_sha = None\n        self.wf_download_url = None\n        self.config = dict()\n        self.containers = list()\n\n    def download_workflow(self):\n        \"\"\"Starts a nf-core workflow download.\"\"\"\n        # Get workflow details\n        try:\n            self.fetch_workflow_details(nf_core.list.Workflows())\n        except LookupError:\n            sys.exit(1)\n\n        # Check that the outdir doesn't already exist\n        if os.path.exists(self.outdir):\n            logging.error(\"Output directory '{}' already exists\".format(self.outdir))\n            sys.exit(1)\n\n        logging.info(\n            \"Saving {}\".format(self.pipeline) +\n            \"\\n Pipeline release: {}\".format(self.release) +\n            \"\\n Pull singularity containers: {}\".format('Yes' if self.singularity else 'No') +\n            \"\\n Output directory: {}\".format(self.outdir)\n        )\n\n        # Download the pipeline files\n        logging.info(\"Downloading workflow files from GitHub\")\n        self.download_wf_files()\n\n        # Download the centralised configs\n        logging.info(\"Downloading centralised configs from GitHub\")\n        self.download_configs()\n        self.wf_use_local_configs()\n\n        # Download the singularity images\n        if self.singularity:\n            logging.debug(\"Fetching container names for workflow\")\n            self.find_container_images()\n            if len(self.containers) == 0:\n                logging.info(\"No container names found in workflow\")\n            else:\n                os.mkdir(os.path.join(self.outdir, 'singularity-images'))\n                logging.info(\"Downloading {} singularity container{}\".format(len(self.containers), 's' if len(self.containers) > 1 else ''))\n                for container in self.containers:\n                    try:\n                        # Download from Dockerhub in all cases\n                        self.pull_singularity_image(container)\n                    except RuntimeWarning as r:\n                        # Raise exception if this is not possible\n                        logging.error(\"Not able to pull image. Service might be down or internet connection is dead.\")\n                        raise r\n\n\n\n    def fetch_workflow_details(self, wfs):\n        \"\"\"Fetches details of a nf-core workflow to download.\n\n        Args:\n            wfs (nf_core.list.Workflows): A nf_core.list.Workflows object\n\n        Raises:\n            LockupError, if the pipeline can not be found.\n        \"\"\"\n        wfs.get_remote_workflows()\n\n        # Get workflow download details\n        for wf in wfs.remote_workflows:\n            if wf.full_name == self.pipeline or wf.name == self.pipeline:\n\n                # Set pipeline name\n                self.wf_name = wf.name\n\n                # Find latest release hash\n                if self.release is None and len(wf.releases) > 0:\n                    # Sort list of releases so that most recent is first\n                    wf.releases = sorted(wf.releases, key=lambda k: k.get('published_at_timestamp', 0), reverse=True)\n                    self.release = wf.releases[0]['tag_name']\n                    self.wf_sha = wf.releases[0]['tag_sha']\n                    logging.debug(\"No release specified. Using latest release: {}\".format(self.release))\n                # Find specified release hash\n                elif self.release is not None:\n                    for r in wf.releases:\n                        if r['tag_name'] == self.release.lstrip('v'):\n                            self.wf_sha = r['tag_sha']\n                            break\n                    else:\n                        logging.error(\"Not able to find release '{}' for {}\".format(self.release, wf.full_name))\n                        logging.info(\"Available {} releases: {}\".format(wf.full_name, ', '.join([r['tag_name'] for r in wf.releases])))\n                        raise LookupError(\"Not able to find release '{}' for {}\".format(self.release, wf.full_name))\n\n                # Must be a dev-only pipeline\n                elif not self.release:\n                    self.release = 'dev'\n                    self.wf_sha = 'master' # Cheating a little, but GitHub download link works\n                    logging.warn(\"Pipeline is in development - downloading current code on master branch.\\n\" +\n                        \"This is likely to change soon should not be considered fully reproducible.\")\n\n                # Set outdir name if not defined\n                if not self.outdir:\n                    self.outdir = 'nf-core-{}'.format(wf.name)\n                    if self.release is not None:\n                        self.outdir += '-{}'.format(self.release)\n\n                # Set the download URL and return\n                self.wf_download_url = 'https://github.com/{}/archive/{}.zip'.format(wf.full_name, self.wf_sha)\n                return\n\n        # If we got this far, must not be a nf-core pipeline\n        if self.pipeline.count('/') == 1:\n            # Looks like a GitHub address - try working with this repo\n            logging.warn(\"Pipeline name doesn't match any nf-core workflows\")\n            logging.info(\"Pipeline name looks like a GitHub address - attempting to download anyway\")\n            self.wf_name = self.pipeline\n            if not self.release:\n                self.release = 'master'\n            self.wf_sha = self.release\n            if not self.outdir:\n                self.outdir = self.pipeline.replace('/', '-').lower()\n                if self.release is not None:\n                    self.outdir += '-{}'.format(self.release)\n            # Set the download URL and return\n            self.wf_download_url = 'https://github.com/{}/archive/{}.zip'.format(self.pipeline, self.release)\n        else:\n            logging.error(\"Not able to find pipeline '{}'\".format(self.pipeline))\n            logging.info(\"Available pipelines: {}\".format(', '.join([w.name for w in wfs.remote_workflows])))\n            raise LookupError(\"Not able to find pipeline '{}'\".format(self.pipeline))\n\n    def download_wf_files(self):\n        \"\"\"Downloads workflow files from Github to the :attr:`self.outdir`.\n        \"\"\"\n        logging.debug(\"Downloading {}\".format(self.wf_download_url))\n\n        # Download GitHub zip file into memory and extract\n        url = requests.get(self.wf_download_url)\n        zipfile = ZipFile(BytesIO(url.content))\n        zipfile.extractall(self.outdir)\n\n        # Rename the internal directory name to be more friendly\n        gh_name = '{}-{}'.format(self.wf_name, self.wf_sha).split('/')[-1]\n        os.rename(os.path.join(self.outdir, gh_name), os.path.join(self.outdir, 'workflow'))\n\n        # Make downloaded files executable\n        for dirpath, subdirs, filelist in os.walk(os.path.join(self.outdir, 'workflow')):\n            for fname in filelist:\n                os.chmod(os.path.join(dirpath, fname), 0o775)\n\n    def download_configs(self):\n        \"\"\"Downloads the centralised config profiles from nf-core/configs to :attr:`self.outdir`.\n        \"\"\"\n        configs_zip_url = \"https://github.com/nf-core/configs/archive/master.zip\"\n        configs_local_dir = \"configs-master\"\n        logging.debug(\"Downloading {}\".format(configs_zip_url))\n\n        # Download GitHub zip file into memory and extract\n        url = requests.get(configs_zip_url)\n        zipfile = ZipFile(BytesIO(url.content))\n        zipfile.extractall(self.outdir)\n\n        # Rename the internal directory name to be more friendly\n        os.rename(os.path.join(self.outdir, configs_local_dir), os.path.join(self.outdir, 'configs'))\n\n        # Make downloaded files executable\n        for dirpath, subdirs, filelist in os.walk(os.path.join(self.outdir, 'configs')):\n            for fname in filelist:\n                os.chmod(os.path.join(dirpath, fname), 0o775)\n\n    def wf_use_local_configs(self):\n        \"\"\"Edit the downloaded nextflow.config file to use the local config files\n        \"\"\"\n        nfconfig_fn = os.path.join(self.outdir, 'workflow', 'nextflow.config')\n        find_str = 'https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}'\n        repl_str = '../configs/'\n        logging.debug(\"Editing params.custom_config_base in {}\".format(nfconfig_fn))\n\n        # Load the nextflow.config file into memory\n        with open(nfconfig_fn, 'r') as nfconfig_fh:\n          nfconfig = nfconfig_fh.read()\n\n        # Replace the target string\n        nfconfig = nfconfig.replace(find_str, repl_str)\n\n        # Write the file out again\n        with open(nfconfig_fn, 'w') as nfconfig_fh:\n          nfconfig_fh.write(nfconfig)\n\n\n    def find_container_images(self):\n        \"\"\" Find container image names for workflow \"\"\"\n\n        # Use linting code to parse the pipeline nextflow config\n        self.config = nf_core.utils.fetch_wf_config(os.path.join(self.outdir, 'workflow'))\n\n        # Find any config variables that look like a container\n        for k,v in self.config.items():\n            if k.startswith('process.') and k.endswith('.container'):\n                self.containers.append(v.strip('\"').strip(\"'\"))\n\n\n    def pull_singularity_image(self, container):\n        \"\"\"Uses a local installation of singularity to pull an image from Docker Hub.\n\n        Args:\n            container (str): A pipeline's container name. Usually it is of similar format\n                to `nfcore/name:dev`.\n\n        Raises:\n            Various exceptions possible from `subprocess` execution of Singularity.\n        \"\"\"\n        out_name = '{}.simg'.format(container.replace('nfcore', 'nf-core').replace('/','-').replace(':', '-'))\n        out_path = os.path.abspath(os.path.join(self.outdir, 'singularity-images', out_name))\n        address = 'docker://{}'.format(container.replace('docker://', ''))\n        singularity_command = [\"singularity\", \"pull\", \"--name\", out_path, address]\n        logging.info(\"Building singularity image from dockerhub: {}\".format(address))\n        logging.debug(\"Singularity command: {}\".format(' '.join(singularity_command)))\n\n        # Try to use singularity to pull image\n        try:\n            subprocess.call(singularity_command)\n        except OSError as e:\n            if e.errno == errno.ENOENT:\n                # Singularity is not installed\n                logging.error('Singularity is not installed!')\n            else:\n                # Something else went wrong with singularity command\n                raise e\n\n    def validate_md5(self, fname, expected):\n        \"\"\"Calculates the md5sum for a file on the disk and validate with expected.\n\n        Args:\n            fname (str): Path to a local file.\n            expected (str): The expected md5sum.\n\n        Raises:\n            IOError, if the md5sum does not match the remote sum.\n        \"\"\"\n        logging.debug(\"Validating image hash: {}\".format(fname))\n\n        # Calculate the md5 for the file on disk\n        hash_md5 = hashlib.md5()\n        with open(fname, \"rb\") as f:\n            for chunk in iter(lambda: f.read(4096), b\"\"):\n                hash_md5.update(chunk)\n        file_hash = hash_md5.hexdigest()\n\n        if file_hash == expected:\n            logging.debug('md5 sum of image matches expected: {}'.format(expected))\n        else:\n            raise IOError (\"{} md5 does not match remote: {} - {}\".format(fname, expected, file_hash))\n", "path": "nf_core/download.py"}]}