'Called by the monitor thread to look at each watched job and deal
with state changes.'

def check_watched_items(self):

    new_watched = []
    for ajs in self.watched:
        external_job_id = ajs.job_id
        galaxy_id_tag = ajs.job_wrapper.get_id_tag()
        old_state = ajs.old_state
        try:
            assert (external_job_id not in (None, 'None')), ('(%s/%s)    Invalid    job    id' % (galaxy_id_tag, external_job_id))
            state = self.ds.job_status(external_job_id)
            for retry_exception in RETRY_EXCEPTIONS_LOWER:
                setattr(ajs, (retry_exception + '_retries'), 0)
        except (drmaa.InternalException, drmaa.InvalidJobException) as e:
            ecn = type(e).__name__
            retry_param = (ecn.lower() + '_retries')
            state_param = (ecn.lower() + '_state')
            retries = getattr(ajs, retry_param, 0)
            log.warning('(%s/%s)    unable    to    check    job    status    because    of    %s    exception    for    %d    consecutive    tries:    %s', galaxy_id_tag, external_job_id, ecn, (retries + 1), e)
            if (self.runner_params[retry_param] > 0):
                if (retries < self.runner_params[retry_param]):
                    setattr(ajs, retry_param, (retries + 1))
                    new_watched.append(ajs)
                    continue
            if (self.runner_params[state_param] == model.Job.states.OK):
                log.warning('(%s/%s)    job    will    now    be    finished    OK', galaxy_id_tag, external_job_id)
                self.work_queue.put((self.finish_job, ajs))
            elif (self.runner_params[state_param] == model.Job.states.ERROR):
                log.warning('(%s/%s)    job    will    now    be    errored', galaxy_id_tag, external_job_id)
                self.work_queue.put((self.fail_job, ajs))
            else:
                raise Exception('%s    is    set    to    an    invalid    value    (%s),    this    should    not    be    possible.    See    galaxy.jobs.drmaa.__init__()', state_param, self.runner_params[state_param])
            continue
        except drmaa.DrmCommunicationException as e:
            log.warning('(%s/%s)    unable    to    communicate    with    DRM:    %s', galaxy_id_tag, external_job_id, e)
            new_watched.append(ajs)
            continue
        except Exception as e:
            log.exception(('(%s/%s)    unable    to    check    job    status:    %s' % (galaxy_id_tag, external_job_id, e)))
            log.warning(('(%s/%s)    job    will    now    be    errored' % (galaxy_id_tag, external_job_id)))
            ajs.fail_message = 'Cluster    could    not    complete    job'
            self.work_queue.put((self.fail_job, ajs))
            continue
        if (state != old_state):
            log.debug(('(%s/%s)    state    change:    %s' % (galaxy_id_tag, external_job_id, self.drmaa_job_state_strings[state])))
        if ((state == drmaa.JobState.RUNNING) and (not ajs.running)):
            ajs.running = True
            ajs.job_wrapper.change_state(model.Job.states.RUNNING)
        if (state in (drmaa.JobState.FAILED, drmaa.JobState.DONE)):
            if (self._complete_terminal_job(ajs, drmaa_state=state) is not None):
                state = ajs.old_state
            else:
                continue
        if ajs.check_limits():
            self.work_queue.put((self.fail_job, ajs))
            continue
        ajs.old_state = state
        new_watched.append(ajs)
    self.watched = new_watched
