src/dirbs/listgen/generator.py
def _populate_new_notifications_lists(self, executor):
        """Top-level job function to populate per-MNO notification lists."""
        with create_db_connection(self._config.db_config) as conn, CodeProfiler() as cp:
            # First generate the table of IMEIs to notify and commit so that other tables can see results
            with conn:
                self._run_intermediate_table_job(conn,
                                                 self._populate_new_notifications_imei_table,
                                                 description='IMEIS to notify')

            # Queue jobs to calculate the triplets to notify. This is done in parallel as this tends to be the
            # slowest part of the list generation process
            num_phys_shards = partition_utils.num_physical_imei_shards(conn)
            virt_imei_shard_ranges = partition_utils.virt_imei_shard_bounds(num_phys_shards)
            per_shard_jobs = {}
            for shard_num, shard_range in enumerate(virt_imei_shard_ranges, start=1):
                virt_imei_range_start, virt_imei_range_end = shard_range
                self._queue_intermediate_table_job(executor,
                                                   per_shard_jobs,
                                                   partial(self._populate_new_notifications_triplets_single_shard,
                                                           virt_imei_range_start,
                                                           virt_imei_range_end),
                                                   'triplets to notify (shard {shard_num} of {num_phys_shards})'
                                                   .format(shard_num=shard_num, num_phys_shards=num_phys_shards))

            self._wait_for_futures(per_shard_jobs)

            # Now that we have calculate all the prerequisites, populate each MNO's notification list in parallel
            per_mno_jobs = {}
            for op in self._operators:
                self._queue_intermediate_table_job(executor,
                                                   per_mno_jobs,
                                                   partial(self._populate_new_notifications_list, op.id),
                                                   'per-MNO notifications for {0}'.format(op.id))
            self._wait_for_futures(per_mno_jobs)

            # ANALYZE parent table, which analyzes children as well
            with conn.cursor() as cursor:
                self._analyze_helper(cursor, self._notifications_lists_new_tblname)

        return -1, cp.duration
Top-level job function to populate per-MNO notification lists.
Top-level job function to populate per-MNO notification lists.
Top - level job function to populate per - MNO notification lists .
def _populate_new_notifications_lists(self, executor):
        with create_db_connection(self._config.db_config) as conn, CodeProfiler() as cp:
            with conn:
                self._run_intermediate_table_job(conn,
                                                 self._populate_new_notifications_imei_table,
                                                 description='IMEIS to notify')
            num_phys_shards = partition_utils.num_physical_imei_shards(conn)
            virt_imei_shard_ranges = partition_utils.virt_imei_shard_bounds(num_phys_shards)
            per_shard_jobs = {}
            for shard_num, shard_range in enumerate(virt_imei_shard_ranges, start=1):
                virt_imei_range_start, virt_imei_range_end = shard_range
                self._queue_intermediate_table_job(executor,
                                                   per_shard_jobs,
                                                   partial(self._populate_new_notifications_triplets_single_shard,
                                                           virt_imei_range_start,
                                                           virt_imei_range_end),
                                                   'triplets to notify (shard {shard_num} of {num_phys_shards})'
                                                   .format(shard_num=shard_num, num_phys_shards=num_phys_shards))
            self._wait_for_futures(per_shard_jobs)
            per_mno_jobs = {}
            for op in self._operators:
                self._queue_intermediate_table_job(executor,
                                                   per_mno_jobs,
                                                   partial(self._populate_new_notifications_list, op.id),
                                                   'per-MNO notifications for {0}'.format(op.id))
            self._wait_for_futures(per_mno_jobs)
            with conn.cursor() as cursor:
                self._analyze_helper(cursor, self._notifications_lists_new_tblname)
        return -1, cp.duration
def _populate_new_notifications_lists ( self , executor ) : with create_db_connection ( self . _config . db_config ) as conn , CodeProfiler ( ) as cp : with conn : self . _run_intermediate_table_job ( conn , self . _populate_new_notifications_imei_table , description = 'IMEIS to notify' ) num_phys_shards = partition_utils . num_physical_imei_shards ( conn ) virt_imei_shard_ranges = partition_utils . virt_imei_shard_bounds ( num_phys_shards ) per_shard_jobs = { } for shard_num , shard_range in enumerate ( virt_imei_shard_ranges , start = 1 ) : virt_imei_range_start , virt_imei_range_end = shard_range self . _queue_intermediate_table_job ( executor , per_shard_jobs , partial ( self . _populate_new_notifications_triplets_single_shard , virt_imei_range_start , virt_imei_range_end ) , 'triplets to notify (shard {shard_num} of {num_phys_shards})' . format ( shard_num = shard_num , num_phys_shards = num_phys_shards ) ) self . _wait_for_futures ( per_shard_jobs ) per_mno_jobs = { } for op in self . _operators : self . _queue_intermediate_table_job ( executor , per_mno_jobs , partial ( self . _populate_new_notifications_list , op . id ) , 'per-MNO notifications for {0}' . format ( op . id ) ) self . _wait_for_futures ( per_mno_jobs ) with conn . cursor ( ) as cursor : self . _analyze_helper ( cursor , self . _notifications_lists_new_tblname ) return - 1 , cp . duration
Top-level job function to populate per-MNO notification lists.
Top - level job function to populate per - MNO notification lists .
"""Top-level job function to populate per-MNO notification lists.""" # First generate the table of IMEIs to notify and commit so that other tables can see results # Queue jobs to calculate the triplets to notify. This is done in parallel as this tends to be the # slowest part of the list generation process # Now that we have calculate all the prerequisites, populate each MNO's notification list in parallel # ANALYZE parent table, which analyzes children as well
self
executor