'This function will loop indefinitely, using SGELoadBalancer.get_stats()
to get the clusters status. It looks at the job queue and tries to
decide whether to add or remove a node.  It should later look at job
durations (currently doesn\'t)'

def run(self, cluster):

    self._cluster = cluster
    if (self.max_nodes is None):
        self.max_nodes = cluster.cluster_size
    if (self.min_nodes is None):
        self.min_nodes = 1
    if self.kill_cluster:
        self.min_nodes = 0
    if (self.min_nodes > self.max_nodes):
        raise exception.BaseException('min_nodes    cannot    be    greater    than    max_nodes')
    use_default_stats_file = (self.dump_stats and (not self.stats_file))
    use_default_plots_dir = (self.plot_stats and (not self.plot_output_dir))
    if (use_default_stats_file or use_default_plots_dir):
        self._mkdir((DEFAULT_STATS_DIR % cluster.cluster_tag), makedirs=True)
    if (not self.stats_file):
        self.stats_file = (DEFAULT_STATS_FILE % cluster.cluster_tag)
    if (not self.plot_output_dir):
        self.plot_output_dir = (DEFAULT_STATS_DIR % cluster.cluster_tag)
    if (not cluster.is_cluster_up()):
        raise exception.ClusterNotRunning(cluster.cluster_tag)
    if self.dump_stats:
        if os.path.isdir(self.stats_file):
            raise exception.BaseException(("stats    file    destination    '%s'    is    a    directory" % self.stats_file))
        sfdir = os.path.dirname(os.path.abspath(self.stats_file))
        self._validate_dir(sfdir, msg_prefix='stats    file    destination')
    if self.plot_stats:
        if os.path.isfile(self.plot_output_dir):
            raise exception.BaseException(("plot    output    destination    '%s'    is    a    file" % self.plot_output_dir))
        self._validate_dir(self.plot_output_dir, msg_prefix='plot    output    destination')
    raw = dict(__raw__=True)
    log.info('Starting    load    balancer    (Use    ctrl-c    to    exit)')
    log.info(('Maximum    cluster    size:    %d' % self.max_nodes), extra=raw)
    log.info(('Minimum    cluster    size:    %d' % self.min_nodes), extra=raw)
    log.info(('Cluster    growth    rate:    %d    nodes/iteration\n' % self.add_nodes_per_iteration), extra=raw)
    if self.dump_stats:
        log.info(('Writing    stats    to    file:    %s' % self.stats_file))
    if self.plot_stats:
        log.info(('Plotting    stats    to    directory:    %s' % self.plot_output_dir))
    while self._keep_polling:
        if (not cluster.is_cluster_up()):
            log.info('Waiting    for    all    nodes    to    come    up...')
            time.sleep(self.polling_interval)
            continue
        self.get_stats()
        log.info(('Execution    hosts:    %d' % len(self.stat.hosts)), extra=raw)
        log.info(('Queued    jobs:    %d' % len(self.stat.get_queued_jobs())), extra=raw)
        oldest_queued_job_age = self.stat.oldest_queued_job_age()
        if oldest_queued_job_age:
            log.info(('Oldest    queued    job:    %s' % oldest_queued_job_age), extra=raw)
        log.info(('Avg    job    duration:    %d    secs' % self.stat.avg_job_duration()), extra=raw)
        log.info(('Avg    job    wait    time:    %d    secs' % self.stat.avg_wait_time()), extra=raw)
        log.info(('Last    cluster    modification    time:    %s' % self.__last_cluster_mod_time.strftime('%Y-%m-%d    %X%z')), extra=dict(__raw__=True))
        self._eval_add_node()
        self._eval_remove_node()
        if (self.dump_stats or self.plot_stats):
            self.stat.write_stats_to_csv(self.stats_file)
        if self.plot_stats:
            try:
                self.visualizer.graph_all()
            except IOError as e:
                raise exception.BaseException(str(e))
        if self.kill_cluster:
            if self._eval_terminate_cluster():
                log.info('Terminating    cluster    and    exiting...')
                return self._cluster.terminate_cluster()
        log.info(('Sleeping...(looping    again    in    %d    secs)\n' % self.polling_interval))
        time.sleep(self.polling_interval)
