We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
arguments passed using TOIL_SLURM_ARGS not recognized
Hi, 

I am trying to modify the default Slurm parameter via TOIL_SLURM_ARGS. As far as I can tell the env variable looks OK. 

[]export TOIL_SLURM_ARGS="-t 1:00:00 --qos=lassance"
[]env | grep TOIL
TOIL_SLURM_ARGS=-t 1:00:00 --qos=lassance

However, from the error message I get, things do not seem to be parsed properly ( I don't undertand why the valueError is raised as neither "--mem" or "--cpus-per-task" are present in the args )
  
Exception in thread Thread-200:
Traceback (most recent call last):
  File "/n/home01/lassance/.conda/envs/ENV_PROGRESSIVECACTUS/lib/python2.7/threading.py", line 801, in __bootstrap_inner
    self.run()
  File "/n/home01/lassance/.local/lib/python2.7/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py", line 209, in run
    activity |= self.createJobs(newJob)
  File "/n/home01/lassance/.local/lib/python2.7/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py", line 115, in createJobs
    subLine = self.prepareSubmission(cpu, memory, jobID, command)
  File "/n/home01/lassance/.local/lib/python2.7/site-packages/toil/batchSystems/slurm.py", line 61, in prepareSubmission
    return self.prepareSbatch(cpu, memory, jobID) + ['--wrap={}'.format(command)]
  File "/n/home01/lassance/.local/lib/python2.7/site-packages/toil/batchSystems/slurm.py", line 188, in prepareSbatch
    raise ValueError("Some resource arguments are incompatible: {}".format(nativeConfig))
ValueError: Some resource arguments are incompatible: -t 1:00:00 --qos=lassance

Any help troubleshooting this would be greatly appreciated. 

Thanks

JML



arguments passed using TOIL_SLURM_ARGS not recognized
Hi, 

I am trying to modify the default Slurm parameter via TOIL_SLURM_ARGS. As far as I can tell the env variable looks OK. 

[]export TOIL_SLURM_ARGS="-t 1:00:00 --qos=lassance"
[]env | grep TOIL
TOIL_SLURM_ARGS=-t 1:00:00 --qos=lassance

However, from the error message I get, things do not seem to be parsed properly ( I don't undertand why the valueError is raised as neither "--mem" or "--cpus-per-task" are present in the args )
  
Exception in thread Thread-200:
Traceback (most recent call last):
  File "/n/home01/lassance/.conda/envs/ENV_PROGRESSIVECACTUS/lib/python2.7/threading.py", line 801, in __bootstrap_inner
    self.run()
  File "/n/home01/lassance/.local/lib/python2.7/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py", line 209, in run
    activity |= self.createJobs(newJob)
  File "/n/home01/lassance/.local/lib/python2.7/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py", line 115, in createJobs
    subLine = self.prepareSubmission(cpu, memory, jobID, command)
  File "/n/home01/lassance/.local/lib/python2.7/site-packages/toil/batchSystems/slurm.py", line 61, in prepareSubmission
    return self.prepareSbatch(cpu, memory, jobID) + ['--wrap={}'.format(command)]
  File "/n/home01/lassance/.local/lib/python2.7/site-packages/toil/batchSystems/slurm.py", line 188, in prepareSbatch
    raise ValueError("Some resource arguments are incompatible: {}".format(nativeConfig))
ValueError: Some resource arguments are incompatible: -t 1:00:00 --qos=lassance

Any help troubleshooting this would be greatly appreciated. 

Thanks

JML




--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `src/toil/batchSystems/slurm.py`
Content:
```
1 # Copyright (c) 2016 Duke Center for Genomic and Computational Biology
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 from __future__ import absolute_import
16 import logging
17 import os
18 from pipes import quote
19 import subprocess
20 import time
21 import math
22 
23 # Python 3 compatibility imports
24 from six.moves.queue import Empty, Queue
25 from six import iteritems
26 
27 from toil.batchSystems import MemoryString
28 from toil.batchSystems.abstractGridEngineBatchSystem import AbstractGridEngineBatchSystem
29 
30 logger = logging.getLogger(__name__)
31 
32 class SlurmBatchSystem(AbstractGridEngineBatchSystem):
33 
34     class Worker(AbstractGridEngineBatchSystem.Worker):
35 
36         def getRunningJobIDs(self):
37             # Should return a dictionary of Job IDs and number of seconds
38             times = {}
39             currentjobs = dict((str(self.batchJobIDs[x][0]), x) for x in self.runningJobs)
40             # currentjobs is a dictionary that maps a slurm job id (string) to our own internal job id
41             # squeue arguments:
42             # -h for no header
43             # --format to get jobid i, state %t and time days-hours:minutes:seconds
44 
45             lines = subprocess.check_output(['squeue', '-h', '--format', '%i %t %M']).split('\n')
46             for line in lines:
47                 values = line.split()
48                 if len(values) < 3:
49                     continue
50                 slurm_jobid, state, elapsed_time = values
51                 if slurm_jobid in currentjobs and state == 'R':
52                     seconds_running = self.parse_elapsed(elapsed_time)
53                     times[currentjobs[slurm_jobid]] = seconds_running
54 
55             return times
56 
57         def killJob(self, jobID):
58             subprocess.check_call(['scancel', self.getBatchSystemID(jobID)])
59 
60         def prepareSubmission(self, cpu, memory, jobID, command):
61             return self.prepareSbatch(cpu, memory, jobID) + ['--wrap={}'.format(command)]
62 
63         def submitJob(self, subLine):
64             try:
65                 output = subprocess.check_output(subLine, stderr=subprocess.STDOUT)
66                 # sbatch prints a line like 'Submitted batch job 2954103'
67                 result = int(output.strip().split()[-1])
68                 logger.debug("sbatch submitted job %d", result)
69                 return result
70             except subprocess.CalledProcessError as e:
71                 logger.error("sbatch command failed with code %d: %s", e.returncode, e.output)
72                 raise e
73             except OSError as e:
74                 logger.error("sbatch command failed")
75                 raise e
76 
77         def getJobExitCode(self, slurmJobID):
78             logger.debug("Getting exit code for slurm job %d", int(slurmJobID))
79             
80             state, rc = self._getJobDetailsFromSacct(slurmJobID)
81             
82             if rc == -999:
83                 state, rc = self._getJobDetailsFromScontrol(slurmJobID)
84             
85             logger.debug("s job state is %s", state)
86             # If Job is in a running state, return None to indicate we don't have an update                                 
87             if state in ('PENDING', 'RUNNING', 'CONFIGURING', 'COMPLETING', 'RESIZING', 'SUSPENDED'):
88                 return None
89             
90             return rc
91             
92         def _getJobDetailsFromSacct(self, slurmJobID):
93             # SLURM job exit codes are obtained by running sacct.
94             args = ['sacct',
95                     '-n', # no header
96                     '-j', str(slurmJobID), # job
97                     '--format', 'State,ExitCode', # specify output columns
98                     '-P', # separate columns with pipes
99                     '-S', '1970-01-01'] # override start time limit
100             
101             process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
102             rc = process.returncode
103             
104             if rc != 0:
105                 # no accounting system or some other error
106                 return (None, -999)
107             
108             for line in process.stdout:
109                 values = line.strip().split('|')
110                 if len(values) < 2:
111                     continue
112                 state, exitcode = values
113                 logger.debug("sacct job state is %s", state)
114                 # If Job is in a running state, return None to indicate we don't have an update
115                 status, _ = exitcode.split(':')
116                 logger.debug("sacct exit code is %s, returning status %s", exitcode, status)
117                 return (state, int(status))
118             logger.debug("Did not find exit code for job in sacct output")
119             return None
120 
121         def _getJobDetailsFromScontrol(self, slurmJobID):
122             args = ['scontrol',
123                     'show',
124                     'job',
125                     str(slurmJobID)]
126     
127             process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
128     
129             job = dict()
130             for line in process.stdout:
131                 values = line.strip().split()
132     
133                 # If job information is not available an error is issued:
134                 # slurm_load_jobs error: Invalid job id specified
135                 # There is no job information, so exit.
136                 if len(values)>0 and values[0] == 'slurm_load_jobs':
137                     return None
138                 
139                 # Output is in the form of many key=value pairs, multiple pairs on each line
140                 # and multiple lines in the output. Each pair is pulled out of each line and
141                 # added to a dictionary
142                 for v in values:
143                     bits = v.split('=')
144                     job[bits[0]] = bits[1]
145     
146             state = job['JobState']
147             try:
148                 exitcode = job['ExitCode']
149                 if exitcode is not None:
150                     status, _ = exitcode.split(':')
151                     logger.debug("scontrol exit code is %s, returning status %s", exitcode, status)
152                     rc = int(status)
153                 else:
154                     rc = None
155             except KeyError:
156                 rc = None
157             
158             return (state, rc)
159 
160         """
161         Implementation-specific helper methods
162         """
163 
164         def prepareSbatch(self, cpu, mem, jobID):
165             #  Returns the sbatch command line before the script to run
166             sbatch_line = ['sbatch', '-Q', '-J', 'toil_job_{}'.format(jobID)]
167 
168             if self.boss.environment:
169                 argList = []
170                 
171                 for k, v in self.boss.environment.iteritems():
172                     quoted_value = quote(os.environ[k] if v is None else v)
173                     argList.append('{}={}'.format(k, quoted_value))
174                     
175                 sbatch_line.append('--export=' + ','.join(argList))
176             
177             if mem is not None:
178                 # memory passed in is in bytes, but slurm expects megabytes
179                 sbatch_line.append('--mem={}'.format(int(mem) / 2 ** 20))
180             if cpu is not None:
181                 sbatch_line.append('--cpus-per-task={}'.format(int(math.ceil(cpu))))
182 
183             # "Native extensions" for SLURM (see DRMAA or SAGA)
184             nativeConfig = os.getenv('TOIL_SLURM_ARGS')
185             if nativeConfig is not None:
186                 logger.debug("Native SLURM options appended to sbatch from TOIL_SLURM_RESOURCES env. variable: {}".format(nativeConfig))
187                 if "--mem" or "--cpus-per-task" in nativeConfig:
188                     raise ValueError("Some resource arguments are incompatible: {}".format(nativeConfig))
189 
190                 sbatch_line.extend([nativeConfig])
191 
192             return sbatch_line
193 
194         def parse_elapsed(self, elapsed):
195             # slurm returns elapsed time in days-hours:minutes:seconds format
196             # Sometimes it will only return minutes:seconds, so days may be omitted
197             # For ease of calculating, we'll make sure all the delimeters are ':'
198             # Then reverse the list so that we're always counting up from seconds -> minutes -> hours -> days
199             total_seconds = 0
200             try:
201                 elapsed = elapsed.replace('-', ':').split(':')
202                 elapsed.reverse()
203                 seconds_per_unit = [1, 60, 3600, 86400]
204                 for index, multiplier in enumerate(seconds_per_unit):
205                     if index < len(elapsed):
206                         total_seconds += multiplier * int(elapsed[index])
207             except ValueError:
208                 pass  # slurm may return INVALID instead of a time
209             return total_seconds
210 
211     """
212     The interface for SLURM
213     """
214 
215     @classmethod
216     def getWaitDuration(cls):
217         return 1.0
218 
219     @classmethod
220     def obtainSystemConstants(cls):
221         # sinfo -Ne --format '%m,%c'
222         # sinfo arguments:
223         # -N for node-oriented
224         # -h for no header
225         # -e for exact values (e.g. don't return 32+)
226         # --format to get memory, cpu
227         max_cpu = 0
228         max_mem = MemoryString('0')
229         lines = subprocess.check_output(['sinfo', '-Nhe', '--format', '%m %c']).split('\n')
230         for line in lines:
231             values = line.split()
232             if len(values) < 2:
233                 continue
234             mem, cpu = values
235             max_cpu = max(max_cpu, int(cpu))
236             max_mem = max(max_mem, MemoryString(mem + 'M'))
237         if max_cpu == 0 or max_mem.byteVal() == 0:
238             RuntimeError('sinfo did not return memory or cpu info')
239         return max_cpu, max_mem
240
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/src/toil/batchSystems/slurm.py b/src/toil/batchSystems/slurm.py
--- a/src/toil/batchSystems/slurm.py
+++ b/src/toil/batchSystems/slurm.py
@@ -184,10 +184,10 @@
             nativeConfig = os.getenv('TOIL_SLURM_ARGS')
             if nativeConfig is not None:
                 logger.debug("Native SLURM options appended to sbatch from TOIL_SLURM_RESOURCES env. variable: {}".format(nativeConfig))
-                if "--mem" or "--cpus-per-task" in nativeConfig:
+                if ("--mem" in nativeConfig) or ("--cpus-per-task" in nativeConfig):
                     raise ValueError("Some resource arguments are incompatible: {}".format(nativeConfig))
 
-                sbatch_line.extend([nativeConfig])
+                sbatch_line.extend(nativeConfig.split())
 
             return sbatch_line

{"golden_diff": "diff --git a/src/toil/batchSystems/slurm.py b/src/toil/batchSystems/slurm.py\n--- a/src/toil/batchSystems/slurm.py\n+++ b/src/toil/batchSystems/slurm.py\n@@ -184,10 +184,10 @@\n             nativeConfig = os.getenv('TOIL_SLURM_ARGS')\n             if nativeConfig is not None:\n                 logger.debug(\"Native SLURM options appended to sbatch from TOIL_SLURM_RESOURCES env. variable: {}\".format(nativeConfig))\n-                if \"--mem\" or \"--cpus-per-task\" in nativeConfig:\n+                if (\"--mem\" in nativeConfig) or (\"--cpus-per-task\" in nativeConfig):\n                     raise ValueError(\"Some resource arguments are incompatible: {}\".format(nativeConfig))\n \n-                sbatch_line.extend([nativeConfig])\n+                sbatch_line.extend(nativeConfig.split())\n \n             return sbatch_line\n", "issue": "arguments passed using TOIL_SLURM_ARGS not recognized\nHi, \r\n\r\nI am trying to modify the default Slurm parameter via TOIL_SLURM_ARGS. As far as I can tell the env variable looks OK. \r\n\r\n[]export TOIL_SLURM_ARGS=\"-t 1:00:00 --qos=lassance\"\r\n[]env | grep TOIL\r\nTOIL_SLURM_ARGS=-t 1:00:00 --qos=lassance\r\n\r\nHowever, from the error message I get, things do not seem to be parsed properly ( I don't undertand why the valueError is raised as neither \"--mem\" or \"--cpus-per-task\" are present in the args )\r\n  \r\nException in thread Thread-200:\r\nTraceback (most recent call last):\r\n  File \"/n/home01/lassance/.conda/envs/ENV_PROGRESSIVECACTUS/lib/python2.7/threading.py\", line 801, in __bootstrap_inner\r\n    self.run()\r\n  File \"/n/home01/lassance/.local/lib/python2.7/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py\", line 209, in run\r\n    activity |= self.createJobs(newJob)\r\n  File \"/n/home01/lassance/.local/lib/python2.7/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py\", line 115, in createJobs\r\n    subLine = self.prepareSubmission(cpu, memory, jobID, command)\r\n  File \"/n/home01/lassance/.local/lib/python2.7/site-packages/toil/batchSystems/slurm.py\", line 61, in prepareSubmission\r\n    return self.prepareSbatch(cpu, memory, jobID) + ['--wrap={}'.format(command)]\r\n  File \"/n/home01/lassance/.local/lib/python2.7/site-packages/toil/batchSystems/slurm.py\", line 188, in prepareSbatch\r\n    raise ValueError(\"Some resource arguments are incompatible: {}\".format(nativeConfig))\r\nValueError: Some resource arguments are incompatible: -t 1:00:00 --qos=lassance\r\n\r\nAny help troubleshooting this would be greatly appreciated. \r\n\r\nThanks\r\n\r\nJML\r\n\r\n\r\n\narguments passed using TOIL_SLURM_ARGS not recognized\nHi, \r\n\r\nI am trying to modify the default Slurm parameter via TOIL_SLURM_ARGS. As far as I can tell the env variable looks OK. \r\n\r\n[]export TOIL_SLURM_ARGS=\"-t 1:00:00 --qos=lassance\"\r\n[]env | grep TOIL\r\nTOIL_SLURM_ARGS=-t 1:00:00 --qos=lassance\r\n\r\nHowever, from the error message I get, things do not seem to be parsed properly ( I don't undertand why the valueError is raised as neither \"--mem\" or \"--cpus-per-task\" are present in the args )\r\n  \r\nException in thread Thread-200:\r\nTraceback (most recent call last):\r\n  File \"/n/home01/lassance/.conda/envs/ENV_PROGRESSIVECACTUS/lib/python2.7/threading.py\", line 801, in __bootstrap_inner\r\n    self.run()\r\n  File \"/n/home01/lassance/.local/lib/python2.7/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py\", line 209, in run\r\n    activity |= self.createJobs(newJob)\r\n  File \"/n/home01/lassance/.local/lib/python2.7/site-packages/toil/batchSystems/abstractGridEngineBatchSystem.py\", line 115, in createJobs\r\n    subLine = self.prepareSubmission(cpu, memory, jobID, command)\r\n  File \"/n/home01/lassance/.local/lib/python2.7/site-packages/toil/batchSystems/slurm.py\", line 61, in prepareSubmission\r\n    return self.prepareSbatch(cpu, memory, jobID) + ['--wrap={}'.format(command)]\r\n  File \"/n/home01/lassance/.local/lib/python2.7/site-packages/toil/batchSystems/slurm.py\", line 188, in prepareSbatch\r\n    raise ValueError(\"Some resource arguments are incompatible: {}\".format(nativeConfig))\r\nValueError: Some resource arguments are incompatible: -t 1:00:00 --qos=lassance\r\n\r\nAny help troubleshooting this would be greatly appreciated. \r\n\r\nThanks\r\n\r\nJML\r\n\r\n\r\n\n", "before_files": [{"content": "# Copyright (c) 2016 Duke Center for Genomic and Computational Biology\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\nimport logging\nimport os\nfrom pipes import quote\nimport subprocess\nimport time\nimport math\n\n# Python 3 compatibility imports\nfrom six.moves.queue import Empty, Queue\nfrom six import iteritems\n\nfrom toil.batchSystems import MemoryString\nfrom toil.batchSystems.abstractGridEngineBatchSystem import AbstractGridEngineBatchSystem\n\nlogger = logging.getLogger(__name__)\n\nclass SlurmBatchSystem(AbstractGridEngineBatchSystem):\n\n    class Worker(AbstractGridEngineBatchSystem.Worker):\n\n        def getRunningJobIDs(self):\n            # Should return a dictionary of Job IDs and number of seconds\n            times = {}\n            currentjobs = dict((str(self.batchJobIDs[x][0]), x) for x in self.runningJobs)\n            # currentjobs is a dictionary that maps a slurm job id (string) to our own internal job id\n            # squeue arguments:\n            # -h for no header\n            # --format to get jobid i, state %t and time days-hours:minutes:seconds\n\n            lines = subprocess.check_output(['squeue', '-h', '--format', '%i %t %M']).split('\\n')\n            for line in lines:\n                values = line.split()\n                if len(values) < 3:\n                    continue\n                slurm_jobid, state, elapsed_time = values\n                if slurm_jobid in currentjobs and state == 'R':\n                    seconds_running = self.parse_elapsed(elapsed_time)\n                    times[currentjobs[slurm_jobid]] = seconds_running\n\n            return times\n\n        def killJob(self, jobID):\n            subprocess.check_call(['scancel', self.getBatchSystemID(jobID)])\n\n        def prepareSubmission(self, cpu, memory, jobID, command):\n            return self.prepareSbatch(cpu, memory, jobID) + ['--wrap={}'.format(command)]\n\n        def submitJob(self, subLine):\n            try:\n                output = subprocess.check_output(subLine, stderr=subprocess.STDOUT)\n                # sbatch prints a line like 'Submitted batch job 2954103'\n                result = int(output.strip().split()[-1])\n                logger.debug(\"sbatch submitted job %d\", result)\n                return result\n            except subprocess.CalledProcessError as e:\n                logger.error(\"sbatch command failed with code %d: %s\", e.returncode, e.output)\n                raise e\n            except OSError as e:\n                logger.error(\"sbatch command failed\")\n                raise e\n\n        def getJobExitCode(self, slurmJobID):\n            logger.debug(\"Getting exit code for slurm job %d\", int(slurmJobID))\n            \n            state, rc = self._getJobDetailsFromSacct(slurmJobID)\n            \n            if rc == -999:\n                state, rc = self._getJobDetailsFromScontrol(slurmJobID)\n            \n            logger.debug(\"s job state is %s\", state)\n            # If Job is in a running state, return None to indicate we don't have an update                                 \n            if state in ('PENDING', 'RUNNING', 'CONFIGURING', 'COMPLETING', 'RESIZING', 'SUSPENDED'):\n                return None\n            \n            return rc\n            \n        def _getJobDetailsFromSacct(self, slurmJobID):\n            # SLURM job exit codes are obtained by running sacct.\n            args = ['sacct',\n                    '-n', # no header\n                    '-j', str(slurmJobID), # job\n                    '--format', 'State,ExitCode', # specify output columns\n                    '-P', # separate columns with pipes\n                    '-S', '1970-01-01'] # override start time limit\n            \n            process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)\n            rc = process.returncode\n            \n            if rc != 0:\n                # no accounting system or some other error\n                return (None, -999)\n            \n            for line in process.stdout:\n                values = line.strip().split('|')\n                if len(values) < 2:\n                    continue\n                state, exitcode = values\n                logger.debug(\"sacct job state is %s\", state)\n                # If Job is in a running state, return None to indicate we don't have an update\n                status, _ = exitcode.split(':')\n                logger.debug(\"sacct exit code is %s, returning status %s\", exitcode, status)\n                return (state, int(status))\n            logger.debug(\"Did not find exit code for job in sacct output\")\n            return None\n\n        def _getJobDetailsFromScontrol(self, slurmJobID):\n            args = ['scontrol',\n                    'show',\n                    'job',\n                    str(slurmJobID)]\n    \n            process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)\n    \n            job = dict()\n            for line in process.stdout:\n                values = line.strip().split()\n    \n                # If job information is not available an error is issued:\n                # slurm_load_jobs error: Invalid job id specified\n                # There is no job information, so exit.\n                if len(values)>0 and values[0] == 'slurm_load_jobs':\n                    return None\n                \n                # Output is in the form of many key=value pairs, multiple pairs on each line\n                # and multiple lines in the output. Each pair is pulled out of each line and\n                # added to a dictionary\n                for v in values:\n                    bits = v.split('=')\n                    job[bits[0]] = bits[1]\n    \n            state = job['JobState']\n            try:\n                exitcode = job['ExitCode']\n                if exitcode is not None:\n                    status, _ = exitcode.split(':')\n                    logger.debug(\"scontrol exit code is %s, returning status %s\", exitcode, status)\n                    rc = int(status)\n                else:\n                    rc = None\n            except KeyError:\n                rc = None\n            \n            return (state, rc)\n\n        \"\"\"\n        Implementation-specific helper methods\n        \"\"\"\n\n        def prepareSbatch(self, cpu, mem, jobID):\n            #  Returns the sbatch command line before the script to run\n            sbatch_line = ['sbatch', '-Q', '-J', 'toil_job_{}'.format(jobID)]\n\n            if self.boss.environment:\n                argList = []\n                \n                for k, v in self.boss.environment.iteritems():\n                    quoted_value = quote(os.environ[k] if v is None else v)\n                    argList.append('{}={}'.format(k, quoted_value))\n                    \n                sbatch_line.append('--export=' + ','.join(argList))\n            \n            if mem is not None:\n                # memory passed in is in bytes, but slurm expects megabytes\n                sbatch_line.append('--mem={}'.format(int(mem) / 2 ** 20))\n            if cpu is not None:\n                sbatch_line.append('--cpus-per-task={}'.format(int(math.ceil(cpu))))\n\n            # \"Native extensions\" for SLURM (see DRMAA or SAGA)\n            nativeConfig = os.getenv('TOIL_SLURM_ARGS')\n            if nativeConfig is not None:\n                logger.debug(\"Native SLURM options appended to sbatch from TOIL_SLURM_RESOURCES env. variable: {}\".format(nativeConfig))\n                if \"--mem\" or \"--cpus-per-task\" in nativeConfig:\n                    raise ValueError(\"Some resource arguments are incompatible: {}\".format(nativeConfig))\n\n                sbatch_line.extend([nativeConfig])\n\n            return sbatch_line\n\n        def parse_elapsed(self, elapsed):\n            # slurm returns elapsed time in days-hours:minutes:seconds format\n            # Sometimes it will only return minutes:seconds, so days may be omitted\n            # For ease of calculating, we'll make sure all the delimeters are ':'\n            # Then reverse the list so that we're always counting up from seconds -> minutes -> hours -> days\n            total_seconds = 0\n            try:\n                elapsed = elapsed.replace('-', ':').split(':')\n                elapsed.reverse()\n                seconds_per_unit = [1, 60, 3600, 86400]\n                for index, multiplier in enumerate(seconds_per_unit):\n                    if index < len(elapsed):\n                        total_seconds += multiplier * int(elapsed[index])\n            except ValueError:\n                pass  # slurm may return INVALID instead of a time\n            return total_seconds\n\n    \"\"\"\n    The interface for SLURM\n    \"\"\"\n\n    @classmethod\n    def getWaitDuration(cls):\n        return 1.0\n\n    @classmethod\n    def obtainSystemConstants(cls):\n        # sinfo -Ne --format '%m,%c'\n        # sinfo arguments:\n        # -N for node-oriented\n        # -h for no header\n        # -e for exact values (e.g. don't return 32+)\n        # --format to get memory, cpu\n        max_cpu = 0\n        max_mem = MemoryString('0')\n        lines = subprocess.check_output(['sinfo', '-Nhe', '--format', '%m %c']).split('\\n')\n        for line in lines:\n            values = line.split()\n            if len(values) < 2:\n                continue\n            mem, cpu = values\n            max_cpu = max(max_cpu, int(cpu))\n            max_mem = max(max_mem, MemoryString(mem + 'M'))\n        if max_cpu == 0 or max_mem.byteVal() == 0:\n            RuntimeError('sinfo did not return memory or cpu info')\n        return max_cpu, max_mem\n", "path": "src/toil/batchSystems/slurm.py"}], "after_files": [{"content": "# Copyright (c) 2016 Duke Center for Genomic and Computational Biology\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\nimport logging\nimport os\nfrom pipes import quote\nimport subprocess\nimport time\nimport math\n\n# Python 3 compatibility imports\nfrom six.moves.queue import Empty, Queue\nfrom six import iteritems\n\nfrom toil.batchSystems import MemoryString\nfrom toil.batchSystems.abstractGridEngineBatchSystem import AbstractGridEngineBatchSystem\n\nlogger = logging.getLogger(__name__)\n\nclass SlurmBatchSystem(AbstractGridEngineBatchSystem):\n\n    class Worker(AbstractGridEngineBatchSystem.Worker):\n\n        def getRunningJobIDs(self):\n            # Should return a dictionary of Job IDs and number of seconds\n            times = {}\n            currentjobs = dict((str(self.batchJobIDs[x][0]), x) for x in self.runningJobs)\n            # currentjobs is a dictionary that maps a slurm job id (string) to our own internal job id\n            # squeue arguments:\n            # -h for no header\n            # --format to get jobid i, state %t and time days-hours:minutes:seconds\n\n            lines = subprocess.check_output(['squeue', '-h', '--format', '%i %t %M']).split('\\n')\n            for line in lines:\n                values = line.split()\n                if len(values) < 3:\n                    continue\n                slurm_jobid, state, elapsed_time = values\n                if slurm_jobid in currentjobs and state == 'R':\n                    seconds_running = self.parse_elapsed(elapsed_time)\n                    times[currentjobs[slurm_jobid]] = seconds_running\n\n            return times\n\n        def killJob(self, jobID):\n            subprocess.check_call(['scancel', self.getBatchSystemID(jobID)])\n\n        def prepareSubmission(self, cpu, memory, jobID, command):\n            return self.prepareSbatch(cpu, memory, jobID) + ['--wrap={}'.format(command)]\n\n        def submitJob(self, subLine):\n            try:\n                output = subprocess.check_output(subLine, stderr=subprocess.STDOUT)\n                # sbatch prints a line like 'Submitted batch job 2954103'\n                result = int(output.strip().split()[-1])\n                logger.debug(\"sbatch submitted job %d\", result)\n                return result\n            except subprocess.CalledProcessError as e:\n                logger.error(\"sbatch command failed with code %d: %s\", e.returncode, e.output)\n                raise e\n            except OSError as e:\n                logger.error(\"sbatch command failed\")\n                raise e\n\n        def getJobExitCode(self, slurmJobID):\n            logger.debug(\"Getting exit code for slurm job %d\", int(slurmJobID))\n            \n            state, rc = self._getJobDetailsFromSacct(slurmJobID)\n            \n            if rc == -999:\n                state, rc = self._getJobDetailsFromScontrol(slurmJobID)\n            \n            logger.debug(\"s job state is %s\", state)\n            # If Job is in a running state, return None to indicate we don't have an update                                 \n            if state in ('PENDING', 'RUNNING', 'CONFIGURING', 'COMPLETING', 'RESIZING', 'SUSPENDED'):\n                return None\n            \n            return rc\n            \n        def _getJobDetailsFromSacct(self, slurmJobID):\n            # SLURM job exit codes are obtained by running sacct.\n            args = ['sacct',\n                    '-n', # no header\n                    '-j', str(slurmJobID), # job\n                    '--format', 'State,ExitCode', # specify output columns\n                    '-P', # separate columns with pipes\n                    '-S', '1970-01-01'] # override start time limit\n            \n            process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)\n            rc = process.returncode\n            \n            if rc != 0:\n                # no accounting system or some other error\n                return (None, -999)\n            \n            for line in process.stdout:\n                values = line.strip().split('|')\n                if len(values) < 2:\n                    continue\n                state, exitcode = values\n                logger.debug(\"sacct job state is %s\", state)\n                # If Job is in a running state, return None to indicate we don't have an update\n                status, _ = exitcode.split(':')\n                logger.debug(\"sacct exit code is %s, returning status %s\", exitcode, status)\n                return (state, int(status))\n            logger.debug(\"Did not find exit code for job in sacct output\")\n            return None\n\n        def _getJobDetailsFromScontrol(self, slurmJobID):\n            args = ['scontrol',\n                    'show',\n                    'job',\n                    str(slurmJobID)]\n    \n            process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)\n    \n            job = dict()\n            for line in process.stdout:\n                values = line.strip().split()\n    \n                # If job information is not available an error is issued:\n                # slurm_load_jobs error: Invalid job id specified\n                # There is no job information, so exit.\n                if len(values)>0 and values[0] == 'slurm_load_jobs':\n                    return None\n                \n                # Output is in the form of many key=value pairs, multiple pairs on each line\n                # and multiple lines in the output. Each pair is pulled out of each line and\n                # added to a dictionary\n                for v in values:\n                    bits = v.split('=')\n                    job[bits[0]] = bits[1]\n    \n            state = job['JobState']\n            try:\n                exitcode = job['ExitCode']\n                if exitcode is not None:\n                    status, _ = exitcode.split(':')\n                    logger.debug(\"scontrol exit code is %s, returning status %s\", exitcode, status)\n                    rc = int(status)\n                else:\n                    rc = None\n            except KeyError:\n                rc = None\n            \n            return (state, rc)\n\n        \"\"\"\n        Implementation-specific helper methods\n        \"\"\"\n\n        def prepareSbatch(self, cpu, mem, jobID):\n            #  Returns the sbatch command line before the script to run\n            sbatch_line = ['sbatch', '-Q', '-J', 'toil_job_{}'.format(jobID)]\n\n            if self.boss.environment:\n                argList = []\n                \n                for k, v in self.boss.environment.iteritems():\n                    quoted_value = quote(os.environ[k] if v is None else v)\n                    argList.append('{}={}'.format(k, quoted_value))\n                    \n                sbatch_line.append('--export=' + ','.join(argList))\n            \n            if mem is not None:\n                # memory passed in is in bytes, but slurm expects megabytes\n                sbatch_line.append('--mem={}'.format(int(mem) / 2 ** 20))\n            if cpu is not None:\n                sbatch_line.append('--cpus-per-task={}'.format(int(math.ceil(cpu))))\n\n            # \"Native extensions\" for SLURM (see DRMAA or SAGA)\n            nativeConfig = os.getenv('TOIL_SLURM_ARGS')\n            if nativeConfig is not None:\n                logger.debug(\"Native SLURM options appended to sbatch from TOIL_SLURM_RESOURCES env. variable: {}\".format(nativeConfig))\n                if (\"--mem\" in nativeConfig) or (\"--cpus-per-task\" in nativeConfig):\n                    raise ValueError(\"Some resource arguments are incompatible: {}\".format(nativeConfig))\n\n                sbatch_line.extend(nativeConfig.split())\n\n            return sbatch_line\n\n        def parse_elapsed(self, elapsed):\n            # slurm returns elapsed time in days-hours:minutes:seconds format\n            # Sometimes it will only return minutes:seconds, so days may be omitted\n            # For ease of calculating, we'll make sure all the delimeters are ':'\n            # Then reverse the list so that we're always counting up from seconds -> minutes -> hours -> days\n            total_seconds = 0\n            try:\n                elapsed = elapsed.replace('-', ':').split(':')\n                elapsed.reverse()\n                seconds_per_unit = [1, 60, 3600, 86400]\n                for index, multiplier in enumerate(seconds_per_unit):\n                    if index < len(elapsed):\n                        total_seconds += multiplier * int(elapsed[index])\n            except ValueError:\n                pass  # slurm may return INVALID instead of a time\n            return total_seconds\n\n    \"\"\"\n    The interface for SLURM\n    \"\"\"\n\n    @classmethod\n    def getWaitDuration(cls):\n        return 1.0\n\n    @classmethod\n    def obtainSystemConstants(cls):\n        # sinfo -Ne --format '%m,%c'\n        # sinfo arguments:\n        # -N for node-oriented\n        # -h for no header\n        # -e for exact values (e.g. don't return 32+)\n        # --format to get memory, cpu\n        max_cpu = 0\n        max_mem = MemoryString('0')\n        lines = subprocess.check_output(['sinfo', '-Nhe', '--format', '%m %c']).split('\\n')\n        for line in lines:\n            values = line.split()\n            if len(values) < 2:\n                continue\n            mem, cpu = values\n            max_cpu = max(max_cpu, int(cpu))\n            max_mem = max(max_mem, MemoryString(mem + 'M'))\n        if max_cpu == 0 or max_mem.byteVal() == 0:\n            RuntimeError('sinfo did not return memory or cpu info')\n        return max_cpu, max_mem\n", "path": "src/toil/batchSystems/slurm.py"}]}