def _launch_process_group(self, process_commands, streams_path):
processes = {}<EOL>def check_complete_processes(wait=False):<EOL><INDENT>"""<STR_LIT>"""<EOL>result = False<EOL>for proc in list(processes):<EOL><INDENT>if wait: proc.wait()<EOL>if proc.poll() is not None:<EOL><INDENT>self.debug("<STR_LIT>"<EOL>% (processes[proc]['<STR_LIT>'], proc.poll()))<EOL>processes[proc]['<STR_LIT>'].close()<EOL>processes[proc]['<STR_LIT>'].close()<EOL>del processes[proc]<EOL>result = True<EOL><DEDENT><DEDENT>return result<EOL><DEDENT>for cmd, tid in process_commands:<EOL><INDENT>self.debug("<STR_LIT>" % tid)<EOL>job_timestamp = time.strftime('<STR_LIT>')<EOL>basename = "<STR_LIT>" % (self.batch_name, job_timestamp, tid)<EOL>stdout_handle = open(os.path.join(streams_path, "<STR_LIT>"<EOL>% (basename, tid)), "<STR_LIT:wb>")<EOL>stderr_handle = open(os.path.join(streams_path, "<STR_LIT>"<EOL>% (basename, tid)), "<STR_LIT:wb>")<EOL>proc = subprocess.Popen(cmd, stdout=stdout_handle, stderr=stderr_handle)<EOL>processes[proc] = { '<STR_LIT>' : tid,<EOL>'<STR_LIT>' : stdout_handle,<EOL>'<STR_LIT>' : stderr_handle }<EOL>if self.max_concurrency:<EOL><INDENT>while len(processes) >= self.max_concurrency:<EOL><INDENT>if not check_complete_processes(len(processes)==<NUM_LIT:1>):<EOL><INDENT>time.sleep(<NUM_LIT:0.1>)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>while len(processes) > <NUM_LIT:0>:<EOL><INDENT>if not check_complete_processes(True):<EOL><INDENT>time.sleep(<NUM_LIT:0.1>)<EOL><DEDENT><DEDENT>
Launches processes defined by process_commands, but only
executes max_concurrency processes at a time; if a process
completes and there are still outstanding processes to be
executed, the next processes are run until max_concurrency is
reached again.