def _strategy_simple(self, tasks, *args, kind=None, **kwargs):
        """Peek at the DFK and the executors specified.

        We assume here that tasks are not held in a runnable
        state, and that all tasks from an app would be sent to
        a single specific executor, i.e tasks cannot be specified
        to go to one of more executors.

        Args:
            - tasks (task_ids): Not used here.

        KWargs:
            - kind (Not used)
        """

        for label, executor in self.dfk.executors.items():
            if not executor.scaling_enabled:
                continue

            # Tasks that are either pending completion
            active_tasks = executor.outstanding

            status = executor.status()
            self.unset_logging()

            # FIXME we need to handle case where provider does not define these
            # FIXME probably more of this logic should be moved to the provider
            min_blocks = executor.provider.min_blocks
            max_blocks = executor.provider.max_blocks
            if isinstance(executor, IPyParallelExecutor):
                tasks_per_node = executor.workers_per_node
            elif isinstance(executor, HighThroughputExecutor):
                # This is probably wrong calculation, we need this to come from the executor
                # since we can't know slots ahead of time.
                tasks_per_node = 1
            elif isinstance(executor, ExtremeScaleExecutor):
                tasks_per_node = executor.ranks_per_node

            nodes_per_block = executor.provider.nodes_per_block
            parallelism = executor.provider.parallelism

            running = sum([1 for x in status if x == 'RUNNING'])
            submitting = sum([1 for x in status if x == 'SUBMITTING'])
            pending = sum([1 for x in status if x == 'PENDING'])
            active_blocks = running + submitting + pending
            active_slots = active_blocks * tasks_per_node * nodes_per_block

            if hasattr(executor, 'connected_workers'):
                logger.debug('Executor {} has {} active tasks, {}/{}/{} running/submitted/pending blocks, and {} connected workers'.format(
                    label, active_tasks, running, submitting, pending, executor.connected_workers))
            else:
                logger.debug('Executor {} has {} active tasks and {}/{}/{} running/submitted/pending blocks'.format(
                    label, active_tasks, running, submitting, pending))

            # reset kill timer if executor has active tasks
            if active_tasks > 0 and self.executors[executor.label]['idle_since']:
                self.executors[executor.label]['idle_since'] = None

            # Case 1
            # No tasks.
            if active_tasks == 0:
                # Case 1a
                # Fewer blocks that min_blocks
                if active_blocks <= min_blocks:
                    # Ignore
                    # logger.debug("Strategy: Case.1a")
                    pass

                # Case 1b
                # More blocks than min_blocks. Scale down
                else:
                    # We want to make sure that max_idletime is reached
                    # before killing off resources
                    if not self.executors[executor.label]['idle_since']:
                        logger.debug("Executor {} has 0 active tasks; starting kill timer (if idle time exceeds {}s, resources will be removed)".format(
                            label, self.max_idletime)
                        )
                        self.executors[executor.label]['idle_since'] = time.time()

                    idle_since = self.executors[executor.label]['idle_since']
                    if (time.time() - idle_since) > self.max_idletime:
                        # We have resources idle for the max duration,
                        # we have to scale_in now.
                        logger.debug("Idle time has reached {}s for executor {}; removing resources".format(
                            self.max_idletime, label)
                        )
                        executor.scale_in(active_blocks - min_blocks)

                    else:
                        pass
                        # logger.debug("Strategy: Case.1b. Waiting for timer : {0}".format(idle_since))

            # Case 2
            # More tasks than the available slots.
            elif (float(active_slots) / active_tasks) < parallelism:
                # Case 2a
                # We have the max blocks possible
                if active_blocks >= max_blocks:
                    # Ignore since we already have the max nodes
                    # logger.debug("Strategy: Case.2a")
                    pass

                # Case 2b
                else:
                    # logger.debug("Strategy: Case.2b")
                    excess = math.ceil((active_tasks * parallelism) - active_slots)
                    excess_blocks = math.ceil(float(excess) / (tasks_per_node * nodes_per_block))
                    logger.debug("Requesting {} more blocks".format(excess_blocks))
                    executor.scale_out(excess_blocks)

            elif active_slots == 0 and active_tasks > 0:
                # Case 4
                # Check if slots are being lost quickly ?
                logger.debug("Requesting single slot")
                executor.scale_out(1)
            # Case 3
            # tasks ~ slots
            else:
                # logger.debug("Strategy: Case 3")
                pass
def _strategy_simple(self, tasks, *args, kind=None, **kwargs):
        """Peek at the DFK and the executors specified.

        We assume here that tasks are not held in a runnable
        state, and that all tasks from an app would be sent to
        a single specific executor, i.e tasks cannot be specified
        to go to one of more executors.

        Args:
            - tasks (task_ids): Not used here.

        KWargs:
            - kind (Not used)
        """

        for label, executor in self.dfk.executors.items():
            if not executor.scaling_enabled:
                continue

            # Tasks that are either pending completion
            active_tasks = executor.outstanding

            status = executor.status()
            self.unset_logging()

            # FIXME we need to handle case where provider does not define these
            # FIXME probably more of this logic should be moved to the provider
            min_blocks = executor.provider.min_blocks
            max_blocks = executor.provider.max_blocks
            if isinstance(executor, IPyParallelExecutor):
                tasks_per_node = executor.workers_per_node
            elif isinstance(executor, HighThroughputExecutor):
                # This is probably wrong calculation, we need this to come from the executor
                # since we can't know slots ahead of time.
                tasks_per_node = 1
            elif isinstance(executor, ExtremeScaleExecutor):
                tasks_per_node = executor.ranks_per_node

            nodes_per_block = executor.provider.nodes_per_block
            parallelism = executor.provider.parallelism

            running = sum([1 for x in status if x == 'RUNNING'])
            submitting = sum([1 for x in status if x == 'SUBMITTING'])
            pending = sum([1 for x in status if x == 'PENDING'])
            active_blocks = running + submitting + pending
            active_slots = active_blocks * tasks_per_node * nodes_per_block

            if hasattr(executor, 'connected_workers'):
                logger.debug('Executor {} has {} active tasks, {}/{}/{} running/submitted/pending blocks, and {} connected workers'.format(
                    label, active_tasks, running, submitting, pending, executor.connected_workers))
            else:
                logger.debug('Executor {} has {} active tasks and {}/{}/{} running/submitted/pending blocks'.format(
                    label, active_tasks, running, submitting, pending))

            # reset kill timer if executor has active tasks
            if active_tasks > 0 and self.executors[executor.label]['idle_since']:
                self.executors[executor.label]['idle_since'] = None

            # Case 1
            # No tasks.
            if active_tasks == 0:
                # Case 1a
                # Fewer blocks that min_blocks
                if active_blocks <= min_blocks:
                    # Ignore
                    # logger.debug("Strategy: Case.1a")
                    pass

                # Case 1b
                # More blocks than min_blocks. Scale down
                else:
                    # We want to make sure that max_idletime is reached
                    # before killing off resources
                    if not self.executors[executor.label]['idle_since']:
                        logger.debug("Executor {} has 0 active tasks; starting kill timer (if idle time exceeds {}s, resources will be removed)".format(
                            label, self.max_idletime)
                        )
                        self.executors[executor.label]['idle_since'] = time.time()

                    idle_since = self.executors[executor.label]['idle_since']
                    if (time.time() - idle_since) > self.max_idletime:
                        # We have resources idle for the max duration,
                        # we have to scale_in now.
                        logger.debug("Idle time has reached {}s for executor {}; removing resources".format(
                            self.max_idletime, label)
                        )
                        executor.scale_in(active_blocks - min_blocks)

                    else:
                        pass
                        # logger.debug("Strategy: Case.1b. Waiting for timer : {0}".format(idle_since))

            # Case 2
            # More tasks than the available slots.
            elif (float(active_slots) / active_tasks) < parallelism:
                # Case 2a
                # We have the max blocks possible
                if active_blocks >= max_blocks:
                    # Ignore since we already have the max nodes
                    # logger.debug("Strategy: Case.2a")
                    pass

                # Case 2b
                else:
                    # logger.debug("Strategy: Case.2b")
                    excess = math.ceil((active_tasks * parallelism) - active_slots)
                    excess_blocks = math.ceil(float(excess) / (tasks_per_node * nodes_per_block))
                    logger.debug("Requesting {} more blocks".format(excess_blocks))
                    executor.scale_out(excess_blocks)

            elif active_slots == 0 and active_tasks > 0:
                # Case 4
                # Check if slots are being lost quickly ?
                logger.debug("Requesting single slot")
                executor.scale_out(1)
            # Case 3
            # tasks ~ slots
            else:
                # logger.debug("Strategy: Case 3")
                pass
def _strategy_simple ( self , tasks , * args , kind = None , * * kwargs ) : for label , executor in self . dfk . executors . items ( ) : if not executor . scaling_enabled : continue # Tasks that are either pending completion active_tasks = executor . outstanding status = executor . status ( ) self . unset_logging ( ) # FIXME we need to handle case where provider does not define these # FIXME probably more of this logic should be moved to the provider min_blocks = executor . provider . min_blocks max_blocks = executor . provider . max_blocks if isinstance ( executor , IPyParallelExecutor ) : tasks_per_node = executor . workers_per_node elif isinstance ( executor , HighThroughputExecutor ) : # This is probably wrong calculation, we need this to come from the executor # since we can't know slots ahead of time. tasks_per_node = 1 elif isinstance ( executor , ExtremeScaleExecutor ) : tasks_per_node = executor . ranks_per_node nodes_per_block = executor . provider . nodes_per_block parallelism = executor . provider . parallelism running = sum ( [ 1 for x in status if x == 'RUNNING' ] ) submitting = sum ( [ 1 for x in status if x == 'SUBMITTING' ] ) pending = sum ( [ 1 for x in status if x == 'PENDING' ] ) active_blocks = running + submitting + pending active_slots = active_blocks * tasks_per_node * nodes_per_block if hasattr ( executor , 'connected_workers' ) : logger . debug ( 'Executor {} has {} active tasks, {}/{}/{} running/submitted/pending blocks, and {} connected workers' . format ( label , active_tasks , running , submitting , pending , executor . connected_workers ) ) else : logger . debug ( 'Executor {} has {} active tasks and {}/{}/{} running/submitted/pending blocks' . format ( label , active_tasks , running , submitting , pending ) ) # reset kill timer if executor has active tasks if active_tasks > 0 and self . executors [ executor . label ] [ 'idle_since' ] : self . executors [ executor . label ] [ 'idle_since' ] = None # Case 1 # No tasks. if active_tasks == 0 : # Case 1a # Fewer blocks that min_blocks if active_blocks <= min_blocks : # Ignore # logger.debug("Strategy: Case.1a") pass # Case 1b # More blocks than min_blocks. Scale down else : # We want to make sure that max_idletime is reached # before killing off resources if not self . executors [ executor . label ] [ 'idle_since' ] : logger . debug ( "Executor {} has 0 active tasks; starting kill timer (if idle time exceeds {}s, resources will be removed)" . format ( label , self . max_idletime ) ) self . executors [ executor . label ] [ 'idle_since' ] = time . time ( ) idle_since = self . executors [ executor . label ] [ 'idle_since' ] if ( time . time ( ) - idle_since ) > self . max_idletime : # We have resources idle for the max duration, # we have to scale_in now. logger . debug ( "Idle time has reached {}s for executor {}; removing resources" . format ( self . max_idletime , label ) ) executor . scale_in ( active_blocks - min_blocks ) else : pass # logger.debug("Strategy: Case.1b. Waiting for timer : {0}".format(idle_since)) # Case 2 # More tasks than the available slots. elif ( float ( active_slots ) / active_tasks ) < parallelism : # Case 2a # We have the max blocks possible if active_blocks >= max_blocks : # Ignore since we already have the max nodes # logger.debug("Strategy: Case.2a") pass # Case 2b else : # logger.debug("Strategy: Case.2b") excess = math . ceil ( ( active_tasks * parallelism ) - active_slots ) excess_blocks = math . ceil ( float ( excess ) / ( tasks_per_node * nodes_per_block ) ) logger . debug ( "Requesting {} more blocks" . format ( excess_blocks ) ) executor . scale_out ( excess_blocks ) elif active_slots == 0 and active_tasks > 0 : # Case 4 # Check if slots are being lost quickly ? logger . debug ( "Requesting single slot" ) executor . scale_out ( 1 ) # Case 3 # tasks ~ slots else : # logger.debug("Strategy: Case 3") pass
Peek at the DFK and the executors specified.

        We assume here that tasks are not held in a runnable
        state, and that all tasks from an app would be sent to
        a single specific executor, i.e tasks cannot be specified
        to go to one of more executors.

        Args:
            - tasks (task_ids): Not used here.

        KWargs:
            - kind (Not used)
Peek at the DFK and the executors specified .