sregistry/main/workers/worker.py
def run(self, func, tasks, func2=None):
        '''run will send a list of tasks,
        a tuple with arguments, through a function.
        the arguments should be ordered correctly.
        :param func: the function to run with multiprocessing.pool
        :param tasks: a list of tasks, each a tuple
                      of arguments to process
        :param func2: filter function to run result
                      from func through (optional)
        '''

        # Keep track of some progress for the user
        progress = 1
        total = len(tasks)

        # if we don't have tasks, don't run
        if len(tasks) == 0:
            return

        # If two functions are run per task, double total jobs
        if func2 is not None:
            total = total * 2

        finished = []
        level1 = []
        results = []

        try:
            prefix = "[%s/%s]" % (progress, total)
            bot.show_progress(0, total, length=35, prefix=prefix)
            pool = multiprocessing.Pool(self.workers, init_worker)

            self.start()
            for task in tasks:
                result = pool.apply_async(multi_wrapper,
                                          multi_package(func, [task]))
                results.append(result)
                level1.append(result._job)

            while len(results) > 0:
                result = results.pop()
                result.wait()
                bot.show_progress(progress, total, length=35, prefix=prefix)
                progress += 1
                prefix = "[%s/%s]" % (progress, total)

                # Pass the result through a second function?
                if func2 is not None and result._job in level1:
                    result = pool.apply_async(multi_wrapper,
                                              multi_package(func2,
                                                            [(result.get(),)]))
                    results.append(result)
                else:
                    finished.append(result.get())

            self.end()
            pool.close()
            pool.join()

        except (KeyboardInterrupt, SystemExit):
            bot.error("Keyboard interrupt detected, terminating workers!")
            pool.terminate()
            sys.exit(1)

        except Exception as e:
            bot.error(e)

        return finished
def run(self, func, tasks, func2=None):
        '''run will send a list of tasks,
        a tuple with arguments, through a function.
        the arguments should be ordered correctly.
        :param func: the function to run with multiprocessing.pool
        :param tasks: a list of tasks, each a tuple
                      of arguments to process
        :param func2: filter function to run result
                      from func through (optional)
        '''

        # Keep track of some progress for the user
        progress = 1
        total = len(tasks)

        # if we don't have tasks, don't run
        if len(tasks) == 0:
            return

        # If two functions are run per task, double total jobs
        if func2 is not None:
            total = total * 2

        finished = []
        level1 = []
        results = []

        try:
            prefix = "[%s/%s]" % (progress, total)
            bot.show_progress(0, total, length=35, prefix=prefix)
            pool = multiprocessing.Pool(self.workers, init_worker)

            self.start()
            for task in tasks:
                result = pool.apply_async(multi_wrapper,
                                          multi_package(func, [task]))
                results.append(result)
                level1.append(result._job)

            while len(results) > 0:
                result = results.pop()
                result.wait()
                bot.show_progress(progress, total, length=35, prefix=prefix)
                progress += 1
                prefix = "[%s/%s]" % (progress, total)

                # Pass the result through a second function?
                if func2 is not None and result._job in level1:
                    result = pool.apply_async(multi_wrapper,
                                              multi_package(func2,
                                                            [(result.get(),)]))
                    results.append(result)
                else:
                    finished.append(result.get())

            self.end()
            pool.close()
            pool.join()

        except (KeyboardInterrupt, SystemExit):
            bot.error("Keyboard interrupt detected, terminating workers!")
            pool.terminate()
            sys.exit(1)

        except Exception as e:
            bot.error(e)

        return finished
def run ( self , func , tasks , func2 = None ) : # Keep track of some progress for the user progress = 1 total = len ( tasks ) # if we don't have tasks, don't run if len ( tasks ) == 0 : return # If two functions are run per task, double total jobs if func2 is not None : total = total * 2 finished = [ ] level1 = [ ] results = [ ] try : prefix = "[%s/%s]" % ( progress , total ) bot . show_progress ( 0 , total , length = 35 , prefix = prefix ) pool = multiprocessing . Pool ( self . workers , init_worker ) self . start ( ) for task in tasks : result = pool . apply_async ( multi_wrapper , multi_package ( func , [ task ] ) ) results . append ( result ) level1 . append ( result . _job ) while len ( results ) > 0 : result = results . pop ( ) result . wait ( ) bot . show_progress ( progress , total , length = 35 , prefix = prefix ) progress += 1 prefix = "[%s/%s]" % ( progress , total ) # Pass the result through a second function? if func2 is not None and result . _job in level1 : result = pool . apply_async ( multi_wrapper , multi_package ( func2 , [ ( result . get ( ) , ) ] ) ) results . append ( result ) else : finished . append ( result . get ( ) ) self . end ( ) pool . close ( ) pool . join ( ) except ( KeyboardInterrupt , SystemExit ) : bot . error ( "Keyboard interrupt detected, terminating workers!" ) pool . terminate ( ) sys . exit ( 1 ) except Exception as e : bot . error ( e ) return finished
run will send a list of tasks,
        a tuple with arguments, through a function.
        the arguments should be ordered correctly.
        :param func: the function to run with multiprocessing.pool
        :param tasks: a list of tasks, each a tuple
                      of arguments to process
        :param func2: filter function to run result
                      from func through (optional)
run will send a list of tasks a tuple with arguments through a function . the arguments should be ordered correctly . : param func : the function to run with multiprocessing . pool : param tasks : a list of tasks each a tuple of arguments to process : param func2 : filter function to run result from func through ( optional )