'This function inspects the current state of the SGE queue and decides
whether or not to add nodes to the cluster. Returns the number of nodes
to add.'

def _eval_add_node(self):

    num_nodes = len(self._cluster.nodes)
    if (num_nodes >= self.max_nodes):
        log.info(('Not    adding    nodes:    already    at    or    above    maximum    (%d)' % self.max_nodes))
        return
    queued_jobs = self.stat.get_queued_jobs()
    if ((not queued_jobs) and (num_nodes >= self.min_nodes)):
        log.info('Not    adding    nodes:    at    or    above    minimum    nodes    and    no    queued    jobs...')
        return
    total_slots = self.stat.count_total_slots()
    if ((not self.has_cluster_stabilized()) and (total_slots > 0)):
        return
    running_jobs = self.stat.get_running_jobs()
    used_slots = sum([int(j['slots']) for j in running_jobs])
    qw_slots = sum([int(j['slots']) for j in queued_jobs])
    slots_per_host = self.stat.slots_per_host()
    avail_slots = (total_slots - used_slots)
    need_to_add = 0
    if (num_nodes < self.min_nodes):
        log.info(('Adding    node:    below    minimum    (%d)' % self.min_nodes))
        need_to_add = (self.min_nodes - num_nodes)
    elif (total_slots == 0):
        need_to_add = 1
    elif (qw_slots > avail_slots):
        log.info(('Queued    jobs    need    more    slots    (%d)    than    available    (%d)' % (qw_slots, avail_slots)))
        oldest_job_dt = self.stat.oldest_queued_job_age()
        now = self.get_remote_time()
        age_delta = (now - oldest_job_dt)
        if (age_delta.seconds > self.longest_allowed_queue_time):
            log.info(('A    job    has    been    waiting    for    %d    seconds    longer    than    max:    %d' % (age_delta.seconds, self.longest_allowed_queue_time)))
            if (slots_per_host != 0):
                need_to_add = (qw_slots / slots_per_host)
            else:
                need_to_add = 1
        else:
            log.info(('No    queued    jobs    older    than    %d    seconds' % self.longest_allowed_queue_time))
    max_add = (self.max_nodes - len(self._cluster.running_nodes))
    need_to_add = min(self.add_nodes_per_iteration, need_to_add, max_add)
    if (need_to_add > 0):
        log.warn(('Adding    %d    nodes    at    %s' % (need_to_add, str(utils.get_utc_now()))))
        try:
            self._cluster.add_nodes(need_to_add)
            self.__last_cluster_mod_time = utils.get_utc_now()
            log.info(('Done    adding    nodes    at    %s' % str(self.__last_cluster_mod_time)))
        except Exception:
            log.error('Failed    to    add    new    host', exc_info=True)
