def _runPermutationsCluster(self, jobParams, loggingLevel=logging.INFO,<EOL>maxNumWorkers=<NUM_LIT:4>, env=None,<EOL>waitForCompletion=True, ignoreErrModels=False,<EOL>timeoutSec=DEFAULT_JOB_TIMEOUT_SEC):
print()<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>print("<STR_LIT>")<EOL>if env is not None and len(env) > <NUM_LIT:0>:<EOL><INDENT>envItems = []<EOL>for (key, value) in env.items():<EOL><INDENT>if (sys.platform.startswith('<STR_LIT>')):<EOL><INDENT>envItems.append("<STR_LIT>" % (key, value))<EOL><DEDENT>else:<EOL><INDENT>envItems.append("<STR_LIT>" % (key, value))<EOL><DEDENT><DEDENT>if (sys.platform.startswith('<STR_LIT>')):<EOL><INDENT>envStr = "<STR_LIT>" % ('<STR_LIT>'.join(envItems))<EOL><DEDENT>else:<EOL><INDENT>envStr = "<STR_LIT>" % ('<STR_LIT:;>'.join(envItems))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>envStr = '<STR_LIT>'<EOL><DEDENT>cmdLine = '<STR_LIT>''<STR_LIT>'% (envStr, loggingLevel)<EOL>cjDAO = ClientJobsDAO.get()<EOL>jobID = cjDAO.jobInsert(client='<STR_LIT:test>', cmdLine=cmdLine,<EOL>params=json.dumps(jobParams),<EOL>minimumWorkers=<NUM_LIT:1>, maximumWorkers=maxNumWorkers,<EOL>jobType = cjDAO.JOB_TYPE_HS)<EOL>workerCmdLine = '<STR_LIT>''<STR_LIT>'% (envStr, jobID, loggingLevel)<EOL>workers = self._launchWorkers(cmdLine=workerCmdLine, numWorkers=maxNumWorkers)<EOL>print("<STR_LIT>" % (jobID))<EOL>print("<STR_LIT>" % (maxNumWorkers),cmdLine)<EOL>if not waitForCompletion:<EOL><INDENT>return (jobID, None, None)<EOL><DEDENT>if timeoutSec is None:<EOL><INDENT>timeout=DEFAULT_JOB_TIMEOUT_SEC<EOL><DEDENT>else:<EOL><INDENT>timeout=timeoutSec<EOL><DEDENT>startTime = time.time()<EOL>lastUpdate = time.time()<EOL>lastCompleted = <NUM_LIT:0><EOL>lastCompletedWithError = <NUM_LIT:0><EOL>lastCompletedAsOrphan = <NUM_LIT:0><EOL>lastStarted = <NUM_LIT:0><EOL>lastJobStatus = "<STR_LIT>"<EOL>lastJobResults = None<EOL>lastActiveSwarms = None<EOL>lastEngStatus = None<EOL>modelIDs = []<EOL>print("<STR_LIT>" % ("<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>", "<STR_LIT>"))<EOL>print("<STR_LIT>")<EOL>while (lastJobStatus != ClientJobsDAO.STATUS_COMPLETED)and (time.time() - lastUpdate < timeout):<EOL><INDENT>printUpdate = False<EOL>if g_myEnv.options.verbosity == <NUM_LIT:0>:<EOL><INDENT>time.sleep(<NUM_LIT:0.5>)<EOL><DEDENT>jobInfo = self._getJobInfo(cjDAO, workers, jobID)<EOL>if jobInfo.status != lastJobStatus:<EOL><INDENT>if jobInfo.status == ClientJobsDAO.STATUS_RUNNINGand lastJobStatus != ClientJobsDAO.STATUS_RUNNING:<EOL><INDENT>print("<STR_LIT>"% (jobInfo.jobId))<EOL><DEDENT>lastJobStatus = jobInfo.status<EOL>printUpdate = True<EOL><DEDENT>if g_myEnv.options.verbosity >= <NUM_LIT:1>:<EOL><INDENT>if jobInfo.engWorkerState is not None:<EOL><INDENT>activeSwarms = json.loads(jobInfo.engWorkerState)['<STR_LIT>']<EOL>if activeSwarms != lastActiveSwarms:<EOL><INDENT>print("<STR_LIT>", '<STR_LIT>'.join(activeSwarms))<EOL>lastActiveSwarms = activeSwarms<EOL>print()<EOL><DEDENT><DEDENT>if jobInfo.results != lastJobResults:<EOL><INDENT>print("<STR_LIT>", jobInfo.results, "<STR_LIT>")<EOL>lastJobResults = jobInfo.results<EOL><DEDENT>if jobInfo.engStatus != lastEngStatus:<EOL><INDENT>print('<STR_LIT>' % jobInfo.engStatus)<EOL>print()<EOL>lastEngStatus = jobInfo.engStatus<EOL><DEDENT><DEDENT>modelCounters = cjDAO.modelsGetUpdateCounters(jobID)<EOL>if len(modelCounters) != lastStarted:<EOL><INDENT>modelIDs = [x.modelId for x in modelCounters]<EOL>lastStarted = len(modelCounters)<EOL>printUpdate = True<EOL><DEDENT>if len(modelIDs) > <NUM_LIT:0>:<EOL><INDENT>completed = <NUM_LIT:0><EOL>completedWithError = <NUM_LIT:0><EOL>completedAsOrphan = <NUM_LIT:0><EOL>infos = cjDAO.modelsGetResultAndStatus(modelIDs)<EOL>for info in infos:<EOL><INDENT>if info.status == ClientJobsDAO.STATUS_COMPLETED:<EOL><INDENT>completed += <NUM_LIT:1><EOL>if info.completionReason == ClientJobsDAO.CMPL_REASON_ERROR:<EOL><INDENT>completedWithError += <NUM_LIT:1><EOL><DEDENT>if info.completionReason == ClientJobsDAO.CMPL_REASON_ORPHAN:<EOL><INDENT>completedAsOrphan += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>if completed != lastCompletedor completedWithError != lastCompletedWithErroror completedAsOrphan != lastCompletedAsOrphan:<EOL><INDENT>lastCompleted = completed<EOL>lastCompletedWithError = completedWithError<EOL>lastCompletedAsOrphan = completedAsOrphan<EOL>printUpdate = True<EOL><DEDENT><DEDENT>if printUpdate:<EOL><INDENT>lastUpdate = time.time()<EOL>if g_myEnv.options.verbosity >= <NUM_LIT:1>:<EOL><INDENT>print("<STR_LIT>", end='<STR_LIT:U+0020>')<EOL><DEDENT>print("<STR_LIT>" % (lastJobStatus, lastStarted,<EOL>lastCompleted,<EOL>lastCompletedWithError,<EOL>lastCompletedAsOrphan))<EOL><DEDENT><DEDENT>print("<STR_LIT>" % (lastJobStatus, lastStarted,<EOL>lastCompleted,<EOL>lastCompletedWithError,<EOL>lastCompletedAsOrphan))<EOL>jobInfo = self._getJobInfo(cjDAO, workers, jobID)<EOL>if not ignoreErrModels:<EOL><INDENT>self.assertEqual (jobInfo.completionReason,<EOL>ClientJobsDAO.CMPL_REASON_SUCCESS)<EOL><DEDENT>models = cjDAO.modelsGetUpdateCounters(jobID)<EOL>modelIDs = [model.modelId for model in models]<EOL>if len(modelIDs) > <NUM_LIT:0>:<EOL><INDENT>results = cjDAO.modelsGetResultAndStatus(modelIDs)<EOL><DEDENT>else:<EOL><INDENT>results = []<EOL><DEDENT>metricResults = []<EOL>for result in results:<EOL><INDENT>if result.results is not None:<EOL><INDENT>metricResults.append(list(json.loads(result.results)[<NUM_LIT:1>].values())[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>metricResults.append(None)<EOL><DEDENT>if not ignoreErrModels:<EOL><INDENT>self.assertNotEqual(result.completionReason, cjDAO.CMPL_REASON_ERROR,<EOL>"<STR_LIT>" % (result.completionMsg))<EOL><DEDENT><DEDENT>return (jobID, jobInfo, results, metricResults)<EOL>
Given a prepared, filled in jobParams for a hypersearch, this starts
        the job, waits for it to complete, and returns the results for all
        models.

        Parameters:
        -------------------------------------------------------------------
        jobParams:        filled in job params for a hypersearch
        loggingLevel:    logging level to use in the Hypersearch worker
        maxNumWorkers:    max # of worker processes to use
        env:             if not None, this is a dict of environment variables
                            that should be sent to each worker process. These can
                            aid in re-using the same description/permutations file
                            for different tests.
        waitForCompletion: If True, wait for job to complete before returning
                           If False, then return resultsInfoForAllModels and
                           metricResults will be None
        ignoreErrModels:  If true, ignore erred models
        retval:          (jobID, jobInfo, resultsInfoForAllModels, metricResults)