'Called repeatedly by `monitor` to process waiting jobs. Gets any new
jobs (either from the database or from its own queue), then iterates
over all new and waiting jobs to check the state of the jobs each
depends on. If the job has dependencies that have not finished, it
goes to the waiting queue. If the job has dependencies with errors,
it is marked as having errors and removed from the queue. If the job
belongs to an inactive user it is ignored.
Otherwise, the job is dispatched.'

def __monitor_step(self):

    jobs_to_check = []
    resubmit_jobs = []
    if self.track_jobs_in_database:
        self.sa_session.expunge_all()
        hda_not_ready = self.sa_session.query(model.Job.id).enable_eagerloads(False).join(model.JobToInputDatasetAssociation).join(model.HistoryDatasetAssociation).join(model.Dataset).filter(and_((model.Job.state == model.Job.states.NEW), or_((model.HistoryDatasetAssociation._state == model.HistoryDatasetAssociation.states.FAILED_METADATA), (model.HistoryDatasetAssociation.deleted == true()), (model.Dataset.state != model.Dataset.states.OK), (model.Dataset.deleted == true())))).subquery()
        ldda_not_ready = self.sa_session.query(model.Job.id).enable_eagerloads(False).join(model.JobToInputLibraryDatasetAssociation).join(model.LibraryDatasetDatasetAssociation).join(model.Dataset).filter(and_((model.Job.state == model.Job.states.NEW), or_((model.LibraryDatasetDatasetAssociation._state != null()), (model.LibraryDatasetDatasetAssociation.deleted == true()), (model.Dataset.state != model.Dataset.states.OK), (model.Dataset.deleted == true())))).subquery()
        if self.app.config.user_activation_on:
            jobs_to_check = self.sa_session.query(model.Job).enable_eagerloads(False).outerjoin(model.User).filter(and_((model.Job.state == model.Job.states.NEW), or_((model.Job.user_id == null()), (model.User.active == true())), (model.Job.handler == self.app.config.server_name), (~ model.Job.table.c.id.in_(hda_not_ready)), (~ model.Job.table.c.id.in_(ldda_not_ready)))).order_by(model.Job.id).all()
        else:
            jobs_to_check = self.sa_session.query(model.Job).enable_eagerloads(False).filter(and_((model.Job.state == model.Job.states.NEW), (model.Job.handler == self.app.config.server_name), (~ model.Job.table.c.id.in_(hda_not_ready)), (~ model.Job.table.c.id.in_(ldda_not_ready)))).order_by(model.Job.id).all()
        resubmit_jobs = self.sa_session.query(model.Job).enable_eagerloads(False).filter(and_((model.Job.state == model.Job.states.RESUBMITTED), (model.Job.handler == self.app.config.server_name))).order_by(model.Job.id).all()
    else:
        for job_id in self.waiting_jobs:
            jobs_to_check.append(self.sa_session.query(model.Job).get(job_id))
        try:
            while 1:
                message = self.queue.get_nowait()
                if (message is self.STOP_SIGNAL):
                    return
                (job_id, tool_id) = message
                jobs_to_check.append(self.sa_session.query(model.Job).get(job_id))
        except Empty:
            pass
    self.__clear_job_count()
    for job in resubmit_jobs:
        log.debug('(%s)    Job    was    resubmitted    and    is    being    dispatched    immediately', job.id)
        jw = self.__recover_job_wrapper(job)
        if jw.is_ready_for_resubmission(job):
            self.increase_running_job_count(job.user_id, jw.job_destination.id)
            self.dispatcher.put(jw)
    new_waiting_jobs = []
    for job in jobs_to_check:
        try:
            job_state = self.__check_job_state(job)
            if (job_state == JOB_WAIT):
                new_waiting_jobs.append(job.id)
            elif (job_state == JOB_INPUT_ERROR):
                log.info(('(%d)    Job    unable    to    run:    one    or    more    inputs    in    error    state' % job.id))
            elif (job_state == JOB_INPUT_DELETED):
                log.info(('(%d)    Job    unable    to    run:    one    or    more    inputs    deleted' % job.id))
            elif (job_state == JOB_READY):
                self.dispatcher.put(self.job_wrappers.pop(job.id))
                log.info(('(%d)    Job    dispatched' % job.id))
            elif (job_state == JOB_DELETED):
                log.info(('(%d)    Job    deleted    by    user    while    still    queued' % job.id))
            elif (job_state == JOB_ADMIN_DELETED):
                log.info(('(%d)    Job    deleted    by    admin    while    still    queued' % job.id))
            elif (job_state in (JOB_USER_OVER_QUOTA, JOB_USER_OVER_TOTAL_WALLTIME)):
                if (job_state == JOB_USER_OVER_QUOTA):
                    log.info(('(%d)    User    (%s)    is    over    quota:    job    paused' % (job.id, job.user_id)))
                else:
                    log.info(('(%d)    User    (%s)    is    over    total    walltime    limit:    job    paused' % (job.id, job.user_id)))
                job.set_state(model.Job.states.PAUSED)
                for dataset_assoc in (job.output_datasets + job.output_library_datasets):
                    dataset_assoc.dataset.dataset.state = model.Dataset.states.PAUSED
                    dataset_assoc.dataset.info = "Execution    of    this    dataset's    job    is    paused    because    you    were    over    your    disk    quota    at    the    time    it    was    ready    to    run"
                    self.sa_session.add(dataset_assoc.dataset.dataset)
                self.sa_session.add(job)
            elif (job_state == JOB_ERROR):
                log.error(('(%d)    Error    checking    job    readiness' % job.id))
            else:
                log.error(("(%d)    Job    in    unknown    state    '%s'" % (job.id, job_state)))
                new_waiting_jobs.append(job.id)
        except Exception:
            log.exception('failure    running    job    %d', job.id)
    if (not self.track_jobs_in_database):
        self.waiting_jobs = new_waiting_jobs
    for id in self.job_wrappers.keys():
        if (id not in new_waiting_jobs):
            del self.job_wrappers[id]
    self.sa_session.flush()
    self.sa_session.remove()
