'Called by the monitor thread to look at each watched job and deal
with state changes.'

def check_watched_items(self):

    new_watched = []
    (failures, statuses) = self.check_all_jobs()
    for pbs_job_state in self.watched:
        job_id = pbs_job_state.job_id
        galaxy_job_id = pbs_job_state.job_wrapper.get_id_tag()
        old_state = pbs_job_state.old_state
        pbs_server_name = self.__get_pbs_server(pbs_job_state.job_destination.params)
        if (pbs_server_name in failures):
            log.debug(('(%s/%s)    Skipping    state    check    because    PBS    server    connection    failed' % (galaxy_job_id, job_id)))
            new_watched.append(pbs_job_state)
            continue
        try:
            status = statuses[job_id]
        except KeyError:
            if (pbs_job_state.job_wrapper.get_state() == model.Job.states.DELETED):
                continue
            try:
                self.check_single_job(pbs_server_name, job_id)
                log.warning(('(%s/%s)    PBS    job    was    not    in    state    check    list,    but    was    found    with    individual    state    check' % (galaxy_job_id, job_id)))
                new_watched.append(pbs_job_state)
            except:
                (errno, text) = pbs.error()
                if (errno == 15001):
                    log.debug(('(%s/%s)    PBS    job    has    left    queue' % (galaxy_job_id, job_id)))
                    self.work_queue.put((self.finish_job, pbs_job_state))
                else:
                    log.info(('(%s/%s)    PBS    state    check    resulted    in    error    (%d):    %s' % (galaxy_job_id, job_id, errno, text)))
                    new_watched.append(pbs_job_state)
            continue
        if (status.job_state != old_state):
            log.debug(('(%s/%s)    PBS    job    state    changed    from    %s    to    %s' % (galaxy_job_id, job_id, old_state, status.job_state)))
        if ((status.job_state == 'R') and (not pbs_job_state.running)):
            pbs_job_state.running = True
            pbs_job_state.job_wrapper.change_state(model.Job.states.RUNNING)
        if ((status.job_state == 'R') and status.get('resources_used', False)):
            (h, m, s) = [int(i) for i in status.resources_used.walltime.split(':')]
            runtime = timedelta(0, s, 0, 0, m, h)
            if pbs_job_state.check_limits(runtime=runtime):
                self.work_queue.put((self.fail_job, pbs_job_state))
                continue
        elif (status.job_state == 'C'):
            try:
                assert (int(status.exit_status) == 0)
                log.debug(('(%s/%s)    PBS    job    has    completed    successfully' % (galaxy_job_id, job_id)))
            except AssertionError:
                exit_status = int(status.exit_status)
                error_message = JOB_EXIT_STATUS.get(exit_status, ('Unknown    error:    %s' % status.exit_status))
                pbs_job_state.fail_message = (CLUSTER_ERROR_MESSAGE % error_message)
                log.error(('(%s/%s)    PBS    job    failed:    %s' % (galaxy_job_id, job_id, error_message)))
                pbs_job_state.stop_job = False
                self.work_queue.put((self.fail_job, pbs_job_state))
                continue
            except AttributeError:
                log.debug(('(%s/%s)    PBS    job    has    completed' % (galaxy_job_id, job_id)))
            self.work_queue.put((self.finish_job, pbs_job_state))
            continue
        pbs_job_state.old_state = status.job_state
        new_watched.append(pbs_job_state)
    self.watched = new_watched
