src/dirbs/reports/stats_generator.py
def _calc_compliance_data(db_config, month, year, condition_tuples, *, run_id, operator=None):
    """Helper functions to calculate the compliance data for each operator."""
    # We fetch results 100000 at a time from cursors in here
    durations = []
    with utils.create_db_connection(db_config) as conn, utils.CodeProfiler() as cp, conn.cursor() as cursor:
        with utils.CodeProfiler() as scp:
            cursor.execute("""CREATE TEMP TABLE network_triplet_counts(
                                  imei_norm TEXT NOT NULL,
                                  per_imei_imsi_counts BIGINT NOT NULL,
                                  per_imei_msisdn_counts BIGINT NOT NULL,
                                  per_imei_triplet_counts BIGINT NOT NULL
                              )""")

            network_triplets_partition_id = _monthly_network_triplets_partition(conn=conn, month=month, year=year,
                                                                                operator=operator)
            cursor.execute(sql.SQL("""INSERT INTO network_triplet_counts(imei_norm,
                                                                         per_imei_imsi_counts,
                                                                         per_imei_msisdn_counts,
                                                                         per_imei_triplet_counts)
                                           SELECT imei_norm,
                                                  COUNT(DISTINCT imsi),
                                                  COUNT(DISTINCT msisdn),
                                                  COUNT(DISTINCT triplet_hash)
                                                        filter(WHERE imei_norm IS NOT NULL
                                                                 AND imsi IS NOT NULL
                                                                 AND msisdn IS NOT NULL)
                                             FROM {0}
                                            WHERE imei_norm IS NOT NULL
                                         GROUP BY imei_norm""").format(network_triplets_partition_id))  # noqa: Q447
            cursor.execute('CREATE UNIQUE INDEX ON network_triplet_counts(imei_norm)')
            cursor.execute('ANALYZE network_triplet_counts')
        durations.append(scp.duration)

        start_date, end_date = _calc_date_range(month, year)
        with utils.CodeProfiler() as scp:
            cursor.execute('CREATE TEMP TABLE network_gross_adds(imei_norm TEXT NOT NULL)')
            cursor.execute(sql.SQL("""INSERT INTO network_gross_adds(imei_norm)
                                           SELECT imei_norm
                                             FROM network_imeis
                                            WHERE network_imeis.first_seen >= %s
                                              AND network_imeis.first_seen < %s
                                              AND EXISTS (SELECT 1
                                                            FROM {network_triplets_partition_id}
                                                           WHERE imei_norm = network_imeis.imei_norm)
                                   """).format(network_triplets_partition_id=network_triplets_partition_id),
                           [start_date, end_date])
            cursor.execute('CREATE UNIQUE INDEX ON network_gross_adds(imei_norm)')
            cursor.execute('ANALYZE network_gross_adds')
        durations.append(scp.duration)

        # Generate the per-TAC compliance data
        per_tac_results, results, per_condition_counts, table_durations = \
            _generate_compliance_data_table(conn, year, month, operator, condition_tuples, run_id=run_id)
        durations.extend(table_durations)

        # Generate compliance breakdown
        with utils.CodeProfiler() as scp:
            compliance_breakdown = _generate_compliance_breakdown(per_tac_results)
        durations.append(scp.duration)

    return per_tac_results, results, per_condition_counts, compliance_breakdown, cp.duration, durations
Helper functions to calculate the compliance data for each operator.
Helper functions to calculate the compliance data for each operator.
Helper functions to calculate the compliance data for each operator .
def _calc_compliance_data(db_config, month, year, condition_tuples, *, run_id, operator=None):
    durations = []
    with utils.create_db_connection(db_config) as conn, utils.CodeProfiler() as cp, conn.cursor() as cursor:
        with utils.CodeProfiler() as scp:
            cursor.execute("""CREATE TEMP TABLE network_triplet_counts(
                                  imei_norm TEXT NOT NULL,
                                  per_imei_imsi_counts BIGINT NOT NULL,
                                  per_imei_msisdn_counts BIGINT NOT NULL,
                                  per_imei_triplet_counts BIGINT NOT NULL
                              )""")
            network_triplets_partition_id = _monthly_network_triplets_partition(conn=conn, month=month, year=year,
                                                                                operator=operator)
            cursor.execute(sql.SQL("""INSERT INTO network_triplet_counts(imei_norm,
                                                                         per_imei_imsi_counts,
                                                                         per_imei_msisdn_counts,
                                                                         per_imei_triplet_counts)
                                           SELECT imei_norm,
                                                  COUNT(DISTINCT imsi),
                                                  COUNT(DISTINCT msisdn),
                                                  COUNT(DISTINCT triplet_hash)
                                                        filter(WHERE imei_norm IS NOT NULL
                                                                 AND imsi IS NOT NULL
                                                                 AND msisdn IS NOT NULL)
                                             FROM {0}
                                            WHERE imei_norm IS NOT NULL
                                         GROUP BY imei_norm""").format(network_triplets_partition_id))  
            cursor.execute('CREATE UNIQUE INDEX ON network_triplet_counts(imei_norm)')
            cursor.execute('ANALYZE network_triplet_counts')
        durations.append(scp.duration)
        start_date, end_date = _calc_date_range(month, year)
        with utils.CodeProfiler() as scp:
            cursor.execute('CREATE TEMP TABLE network_gross_adds(imei_norm TEXT NOT NULL)')
            cursor.execute(sql.SQL("""INSERT INTO network_gross_adds(imei_norm)
                                           SELECT imei_norm
                                             FROM network_imeis
                                            WHERE network_imeis.first_seen >= %s
                                              AND network_imeis.first_seen < %s
                                              AND EXISTS (SELECT 1
                                                            FROM {network_triplets_partition_id}
                                                           WHERE imei_norm = network_imeis.imei_norm)
                                   """).format(network_triplets_partition_id=network_triplets_partition_id),
                           [start_date, end_date])
            cursor.execute('CREATE UNIQUE INDEX ON network_gross_adds(imei_norm)')
            cursor.execute('ANALYZE network_gross_adds')
        durations.append(scp.duration)
        per_tac_results, results, per_condition_counts, table_durations = \
            _generate_compliance_data_table(conn, year, month, operator, condition_tuples, run_id=run_id)
        durations.extend(table_durations)
        with utils.CodeProfiler() as scp:
            compliance_breakdown = _generate_compliance_breakdown(per_tac_results)
        durations.append(scp.duration)
    return per_tac_results, results, per_condition_counts, compliance_breakdown, cp.duration, durations
def _calc_compliance_data ( db_config , month , year , condition_tuples , * , run_id , operator = None ) : durations = [ ] with utils . create_db_connection ( db_config ) as conn , utils . CodeProfiler ( ) as cp , conn . cursor ( ) as cursor : with utils . CodeProfiler ( ) as scp : cursor . execute ( """CREATE TEMP TABLE network_triplet_counts(
                                  imei_norm TEXT NOT NULL,
                                  per_imei_imsi_counts BIGINT NOT NULL,
                                  per_imei_msisdn_counts BIGINT NOT NULL,
                                  per_imei_triplet_counts BIGINT NOT NULL
                              )""" ) network_triplets_partition_id = _monthly_network_triplets_partition ( conn = conn , month = month , year = year , operator = operator ) cursor . execute ( sql . SQL ( """INSERT INTO network_triplet_counts(imei_norm,
                                                                         per_imei_imsi_counts,
                                                                         per_imei_msisdn_counts,
                                                                         per_imei_triplet_counts)
                                           SELECT imei_norm,
                                                  COUNT(DISTINCT imsi),
                                                  COUNT(DISTINCT msisdn),
                                                  COUNT(DISTINCT triplet_hash)
                                                        filter(WHERE imei_norm IS NOT NULL
                                                                 AND imsi IS NOT NULL
                                                                 AND msisdn IS NOT NULL)
                                             FROM {0}
                                            WHERE imei_norm IS NOT NULL
                                         GROUP BY imei_norm""" ) . format ( network_triplets_partition_id ) ) cursor . execute ( 'CREATE UNIQUE INDEX ON network_triplet_counts(imei_norm)' ) cursor . execute ( 'ANALYZE network_triplet_counts' ) durations . append ( scp . duration ) start_date , end_date = _calc_date_range ( month , year ) with utils . CodeProfiler ( ) as scp : cursor . execute ( 'CREATE TEMP TABLE network_gross_adds(imei_norm TEXT NOT NULL)' ) cursor . execute ( sql . SQL ( """INSERT INTO network_gross_adds(imei_norm)
                                           SELECT imei_norm
                                             FROM network_imeis
                                            WHERE network_imeis.first_seen >= %s
                                              AND network_imeis.first_seen < %s
                                              AND EXISTS (SELECT 1
                                                            FROM {network_triplets_partition_id}
                                                           WHERE imei_norm = network_imeis.imei_norm)
                                   """ ) . format ( network_triplets_partition_id = network_triplets_partition_id ) , [ start_date , end_date ] ) cursor . execute ( 'CREATE UNIQUE INDEX ON network_gross_adds(imei_norm)' ) cursor . execute ( 'ANALYZE network_gross_adds' ) durations . append ( scp . duration ) per_tac_results , results , per_condition_counts , table_durations = _generate_compliance_data_table ( conn , year , month , operator , condition_tuples , run_id = run_id ) durations . extend ( table_durations ) with utils . CodeProfiler ( ) as scp : compliance_breakdown = _generate_compliance_breakdown ( per_tac_results ) durations . append ( scp . duration ) return per_tac_results , results , per_condition_counts , compliance_breakdown , cp . duration , durations
Helper functions to calculate the compliance data for each operator.
Helper functions to calculate the compliance data for each operator .
"""Helper functions to calculate the compliance data for each operator.""" # We fetch results 100000 at a time from cursors in here # noqa: Q447 # Generate the per-TAC compliance data # Generate compliance breakdown
db_config
month
year
condition_tuples
run_id
operator