libplanarradpy/planrad.py
def run(self):
        """Distributes the work across the CPUs.  It actually uses _run()"""
        done = False
        dir_list = []
        tic = time.clock()
        lg.info('Starting batch run at :: ' + str(tic))

        if self.run_params.num_cpus == -1:  # user hasn't set a throttle
            self.run_params.num_cpus = os.sysconf("SC_NPROCESSORS_ONLN")
            lg.info('Found ' + str(self.run_params.num_cpus) + ' CPUs')

        # --------------------------------------------------#
        # COUNT THE NUMBER OF DIRECTORIES TO ITERATE THROUGH
        # --------------------------------------------------#
        tmp_dir_list = os.listdir(self.batch_output)
        for direc in tmp_dir_list:
            dir_list.append(os.path.join(self.batch_output, direc))

        num_dirs = len(dir_list)
        lg.info('Found ' + str(num_dirs) + ' directories to process in ' + self.batch_output)

        sub = scipy.floor(num_dirs / self.run_params.num_cpus)
        remainder = num_dirs - (sub * self.run_params.num_cpus)
        if remainder > 0:
            lg.warning('Number of variations not evenly divisible by number of CPUs')
            lg.warning('This is not a problem, last block will not use all available CPUs')
            lg.warning('The remainder is :: ' + str(remainder))

        while not done:
            for l in range(0, int(sub)):
                lg.info('Starting processing block of :: ' + str(self.run_params.num_cpus) + ' processes')

                for m in range(0, self.run_params.num_cpus):
                    #row = (m * sub) + l
                    _dir = dir_list.pop()

                    #--------------------------------------------------#
                    # CHECK TO SEE IF REPORT HAS BEEN GENERATED AND DON'T
                    # BOTHER RUNNING AGAIN IF THEY DO EXIST
                    #--------------------------------------------------#

                    report_dir, report_file_name = os.path.split(self.run_params.report_file)
                    lg.debug(report_file_name)
                    lg.debug(os.path.join(_dir, report_file_name))

                    try:
                        rep_size = os.path.getsize(os.path.join(_dir, report_file_name.strip('\n')))
                        lg.debug('report size is :: ' + str(rep_size))
                    except:
                        rep_size = 0

                    if rep_size < 1.0:  # TODO this is a spoof!
                        lg.info('No report file found, running process')
                        p = Process(target=self._run, args=(_dir,))
                    else:
                        lg.warning('Report file found :: ' + os.path.join(_dir, report_file_name.strip(
                            '\n')) + ' not redoing run ')
                        p = Process(target=self._dummy, args=(_dir,))

                    # !! for testing
                    #p = Process(target=self._dummy, args=(_dir,))
                    p.start()
                    lg.info('Starting Process :: Process ID :: ' + str(p.pid))

                p.join()

            self.run_params.num_cpus = remainder
            remainder = 0
            lg.info('Processing remainder')
            sub = 1
            if remainder == 0:
                done = True

        toc = time.clock()  # this isn't working
        lg.info('Ending batch run at :: ' + str(toc))
        timeTaken = toc - tic
        lg.info('Time taken ::' + str(timeTaken))
def run(self):
        """Distributes the work across the CPUs.  It actually uses _run()"""
        done = False
        dir_list = []
        tic = time.clock()
        lg.info('Starting batch run at :: ' + str(tic))

        if self.run_params.num_cpus == -1:  # user hasn't set a throttle
            self.run_params.num_cpus = os.sysconf("SC_NPROCESSORS_ONLN")
            lg.info('Found ' + str(self.run_params.num_cpus) + ' CPUs')

        # --------------------------------------------------#
        # COUNT THE NUMBER OF DIRECTORIES TO ITERATE THROUGH
        # --------------------------------------------------#
        tmp_dir_list = os.listdir(self.batch_output)
        for direc in tmp_dir_list:
            dir_list.append(os.path.join(self.batch_output, direc))

        num_dirs = len(dir_list)
        lg.info('Found ' + str(num_dirs) + ' directories to process in ' + self.batch_output)

        sub = scipy.floor(num_dirs / self.run_params.num_cpus)
        remainder = num_dirs - (sub * self.run_params.num_cpus)
        if remainder > 0:
            lg.warning('Number of variations not evenly divisible by number of CPUs')
            lg.warning('This is not a problem, last block will not use all available CPUs')
            lg.warning('The remainder is :: ' + str(remainder))

        while not done:
            for l in range(0, int(sub)):
                lg.info('Starting processing block of :: ' + str(self.run_params.num_cpus) + ' processes')

                for m in range(0, self.run_params.num_cpus):
                    #row = (m * sub) + l
                    _dir = dir_list.pop()

                    #--------------------------------------------------#
                    # CHECK TO SEE IF REPORT HAS BEEN GENERATED AND DON'T
                    # BOTHER RUNNING AGAIN IF THEY DO EXIST
                    #--------------------------------------------------#

                    report_dir, report_file_name = os.path.split(self.run_params.report_file)
                    lg.debug(report_file_name)
                    lg.debug(os.path.join(_dir, report_file_name))

                    try:
                        rep_size = os.path.getsize(os.path.join(_dir, report_file_name.strip('\n')))
                        lg.debug('report size is :: ' + str(rep_size))
                    except:
                        rep_size = 0

                    if rep_size < 1.0:  # TODO this is a spoof!
                        lg.info('No report file found, running process')
                        p = Process(target=self._run, args=(_dir,))
                    else:
                        lg.warning('Report file found :: ' + os.path.join(_dir, report_file_name.strip(
                            '\n')) + ' not redoing run ')
                        p = Process(target=self._dummy, args=(_dir,))

                    # !! for testing
                    #p = Process(target=self._dummy, args=(_dir,))
                    p.start()
                    lg.info('Starting Process :: Process ID :: ' + str(p.pid))

                p.join()

            self.run_params.num_cpus = remainder
            remainder = 0
            lg.info('Processing remainder')
            sub = 1
            if remainder == 0:
                done = True

        toc = time.clock()  # this isn't working
        lg.info('Ending batch run at :: ' + str(toc))
        timeTaken = toc - tic
        lg.info('Time taken ::' + str(timeTaken))
def run ( self ) : done = False dir_list = [ ] tic = time . clock ( ) lg . info ( 'Starting batch run at :: ' + str ( tic ) ) if self . run_params . num_cpus == - 1 : # user hasn't set a throttle self . run_params . num_cpus = os . sysconf ( "SC_NPROCESSORS_ONLN" ) lg . info ( 'Found ' + str ( self . run_params . num_cpus ) + ' CPUs' ) # --------------------------------------------------# # COUNT THE NUMBER OF DIRECTORIES TO ITERATE THROUGH # --------------------------------------------------# tmp_dir_list = os . listdir ( self . batch_output ) for direc in tmp_dir_list : dir_list . append ( os . path . join ( self . batch_output , direc ) ) num_dirs = len ( dir_list ) lg . info ( 'Found ' + str ( num_dirs ) + ' directories to process in ' + self . batch_output ) sub = scipy . floor ( num_dirs / self . run_params . num_cpus ) remainder = num_dirs - ( sub * self . run_params . num_cpus ) if remainder > 0 : lg . warning ( 'Number of variations not evenly divisible by number of CPUs' ) lg . warning ( 'This is not a problem, last block will not use all available CPUs' ) lg . warning ( 'The remainder is :: ' + str ( remainder ) ) while not done : for l in range ( 0 , int ( sub ) ) : lg . info ( 'Starting processing block of :: ' + str ( self . run_params . num_cpus ) + ' processes' ) for m in range ( 0 , self . run_params . num_cpus ) : #row = (m * sub) + l _dir = dir_list . pop ( ) #--------------------------------------------------# # CHECK TO SEE IF REPORT HAS BEEN GENERATED AND DON'T # BOTHER RUNNING AGAIN IF THEY DO EXIST #--------------------------------------------------# report_dir , report_file_name = os . path . split ( self . run_params . report_file ) lg . debug ( report_file_name ) lg . debug ( os . path . join ( _dir , report_file_name ) ) try : rep_size = os . path . getsize ( os . path . join ( _dir , report_file_name . strip ( '\n' ) ) ) lg . debug ( 'report size is :: ' + str ( rep_size ) ) except : rep_size = 0 if rep_size < 1.0 : # TODO this is a spoof! lg . info ( 'No report file found, running process' ) p = Process ( target = self . _run , args = ( _dir , ) ) else : lg . warning ( 'Report file found :: ' + os . path . join ( _dir , report_file_name . strip ( '\n' ) ) + ' not redoing run ' ) p = Process ( target = self . _dummy , args = ( _dir , ) ) # !! for testing #p = Process(target=self._dummy, args=(_dir,)) p . start ( ) lg . info ( 'Starting Process :: Process ID :: ' + str ( p . pid ) ) p . join ( ) self . run_params . num_cpus = remainder remainder = 0 lg . info ( 'Processing remainder' ) sub = 1 if remainder == 0 : done = True toc = time . clock ( ) # this isn't working lg . info ( 'Ending batch run at :: ' + str ( toc ) ) timeTaken = toc - tic lg . info ( 'Time taken ::' + str ( timeTaken ) )
Distributes the work across the CPUs.  It actually uses _run()
Distributes the work across the CPUs . It actually uses _run ()