src/dirbs/reports/csv_reports.py
def _validate_data_partitions(config: callable, conn: callable, month: int, year: int, logger: callable,
                              disable_data_check: bool) -> None:
    """
    Validate that data is present for all configured operators and only configured operators.

    Arguments:
        config: DIRBS config object
        conn: DIRBS postgresql connection object
        month: data partition month
        year: data partition year
        logger: DIRBS logger object
        disable_data_check: boolean to disable data check
    Returns:
        None
    Raises:
        MissingOperatorDataException: if monthly_network_triplets_per_mno partition is missing for any operator
        ExtraOperatorDataException: if monthly_network_triplets_per_mno partition is detected for unconfigured mno
    """
    operators = config.region_config.operators
    assert len(operators) > 0

    operator_partitions = utils.child_table_names(conn, 'monthly_network_triplets_per_mno')
    observed_operator_ids = {x for x in utils.table_invariants_list(conn, operator_partitions, ['operator_id'])}
    required_operator_ids = {(o.id,) for o in operators}
    missing_operator_ids = required_operator_ids - observed_operator_ids
    if len(missing_operator_ids) > 0:
        msg = 'Missing monthly_network_triplets_per_mno partitions for operators: {0}' \
              .format(', '.join([x[0] for x in missing_operator_ids]))
        if disable_data_check:
            logger.warning(msg)
        else:
            logger.error(msg)
            raise exceptions.MissingOperatorDataException(msg)

    extra_operator_ids = observed_operator_ids - required_operator_ids
    if len(extra_operator_ids) > 0:
        msg = 'Extra monthly_network_triplets_per_mno partitions detected for unconfigured operators: {0}' \
              .format(', '.join([x[0] for x in extra_operator_ids]))
        if disable_data_check:
            logger.warning(msg)
        else:
            logger.error(msg)
            raise exceptions.ExtraOperatorDataException(msg)

    operator_monthly_partitions = set()
    for op_partition in operator_partitions:
        operator_monthly_partitions.update(utils.child_table_names(conn, op_partition))
    observed_invariants = {x for x in utils.table_invariants_list(conn,
                                                                  operator_monthly_partitions,
                                                                  ['operator_id', 'triplet_year', 'triplet_month'])}
    observed_invariants = {x for x in observed_invariants if x.triplet_year == year and x.triplet_month == month}
    required_invariants = {(o.id, year, month) for o in operators}
    missing_invariants = required_invariants - observed_invariants
    if len(missing_invariants) > 0:
        msg = 'Missing monthly_network_triplets_per_mno partitions for the requested reporting ' \
              'month for the following configured operators: {0}' \
              .format(', '.join([x[0] for x in missing_invariants]))
        if disable_data_check:
            logger.warning(msg)
        else:
            logger.error(msg)
            raise exceptions.MissingOperatorDataException(msg)

    extra_invariants = observed_invariants - required_invariants
    if len(extra_invariants) > 0:
        msg = 'Extra monthly_network_triplets_per_mno partitions detected for the requested ' \
              'reporting month for the following unconfigured operators: {0}' \
              .format(', '.join([x[0] for x in extra_invariants]))
        if disable_data_check:
            logger.warning(msg)
        else:
            logger.error(msg)
            raise exceptions.ExtraOperatorDataException(msg)

    country_imei_shard_name = partition_utils.monthly_network_triplets_country_partition(month=month, year=year)
    with conn.cursor() as cursor:
        cursor.execute(utils.table_exists_sql(), [country_imei_shard_name])
        partition_exists = cursor.fetchone()[0]
        if not partition_exists:
            msg = 'Missing monthly_network_triplets_country partition for year and month'
            if disable_data_check:
                logger.warning(msg)
            else:
                logger.error(msg)
                raise exceptions.ExtraOperatorDataException(msg)

    Validate that data is present for all configured operators and only configured operators.

    Arguments:
        config: DIRBS config object
        conn: DIRBS postgresql connection object
        month: data partition month
        year: data partition year
        logger: DIRBS logger object
        disable_data_check: boolean to disable data check
    Returns:
        None
    Raises:
        MissingOperatorDataException: if monthly_network_triplets_per_mno partition is missing for any operator
        ExtraOperatorDataException: if monthly_network_triplets_per_mno partition is detected for unconfigured mno
    
Validate that data is present for all configured operators and only configured operators.
Validate that data is present for all configured operators and only configured operators .
def _validate_data_partitions(config: callable, conn: callable, month: int, year: int, logger: callable,
                              disable_data_check: bool) -> None:
    operators = config.region_config.operators
    assert len(operators) > 0
    operator_partitions = utils.child_table_names(conn, 'monthly_network_triplets_per_mno')
    observed_operator_ids = {x for x in utils.table_invariants_list(conn, operator_partitions, ['operator_id'])}
    required_operator_ids = {(o.id,) for o in operators}
    missing_operator_ids = required_operator_ids - observed_operator_ids
    if len(missing_operator_ids) > 0:
        msg = 'Missing monthly_network_triplets_per_mno partitions for operators: {0}' \
              .format(', '.join([x[0] for x in missing_operator_ids]))
        if disable_data_check:
            logger.warning(msg)
        else:
            logger.error(msg)
            raise exceptions.MissingOperatorDataException(msg)
    extra_operator_ids = observed_operator_ids - required_operator_ids
    if len(extra_operator_ids) > 0:
        msg = 'Extra monthly_network_triplets_per_mno partitions detected for unconfigured operators: {0}' \
              .format(', '.join([x[0] for x in extra_operator_ids]))
        if disable_data_check:
            logger.warning(msg)
        else:
            logger.error(msg)
            raise exceptions.ExtraOperatorDataException(msg)
    operator_monthly_partitions = set()
    for op_partition in operator_partitions:
        operator_monthly_partitions.update(utils.child_table_names(conn, op_partition))
    observed_invariants = {x for x in utils.table_invariants_list(conn,
                                                                  operator_monthly_partitions,
                                                                  ['operator_id', 'triplet_year', 'triplet_month'])}
    observed_invariants = {x for x in observed_invariants if x.triplet_year == year and x.triplet_month == month}
    required_invariants = {(o.id, year, month) for o in operators}
    missing_invariants = required_invariants - observed_invariants
    if len(missing_invariants) > 0:
        msg = 'Missing monthly_network_triplets_per_mno partitions for the requested reporting ' \
              'month for the following configured operators: {0}' \
              .format(', '.join([x[0] for x in missing_invariants]))
        if disable_data_check:
            logger.warning(msg)
        else:
            logger.error(msg)
            raise exceptions.MissingOperatorDataException(msg)
    extra_invariants = observed_invariants - required_invariants
    if len(extra_invariants) > 0:
        msg = 'Extra monthly_network_triplets_per_mno partitions detected for the requested ' \
              'reporting month for the following unconfigured operators: {0}' \
              .format(', '.join([x[0] for x in extra_invariants]))
        if disable_data_check:
            logger.warning(msg)
        else:
            logger.error(msg)
            raise exceptions.ExtraOperatorDataException(msg)
    country_imei_shard_name = partition_utils.monthly_network_triplets_country_partition(month=month, year=year)
    with conn.cursor() as cursor:
        cursor.execute(utils.table_exists_sql(), [country_imei_shard_name])
        partition_exists = cursor.fetchone()[0]
        if not partition_exists:
            msg = 'Missing monthly_network_triplets_country partition for year and month'
            if disable_data_check:
                logger.warning(msg)
            else:
                logger.error(msg)
                raise exceptions.ExtraOperatorDataException(msg)
def _validate_data_partitions ( config : callable , conn : callable , month : int , year : int , logger : callable , disable_data_check : bool ) -> None : operators = config . region_config . operators assert len ( operators ) > 0 operator_partitions = utils . child_table_names ( conn , 'monthly_network_triplets_per_mno' ) observed_operator_ids = { x for x in utils . table_invariants_list ( conn , operator_partitions , [ 'operator_id' ] ) } required_operator_ids = { ( o . id , ) for o in operators } missing_operator_ids = required_operator_ids - observed_operator_ids if len ( missing_operator_ids ) > 0 : msg = 'Missing monthly_network_triplets_per_mno partitions for operators: {0}' . format ( ', ' . join ( [ x [ 0 ] for x in missing_operator_ids ] ) ) if disable_data_check : logger . warning ( msg ) else : logger . error ( msg ) raise exceptions . MissingOperatorDataException ( msg ) extra_operator_ids = observed_operator_ids - required_operator_ids if len ( extra_operator_ids ) > 0 : msg = 'Extra monthly_network_triplets_per_mno partitions detected for unconfigured operators: {0}' . format ( ', ' . join ( [ x [ 0 ] for x in extra_operator_ids ] ) ) if disable_data_check : logger . warning ( msg ) else : logger . error ( msg ) raise exceptions . ExtraOperatorDataException ( msg ) operator_monthly_partitions = set ( ) for op_partition in operator_partitions : operator_monthly_partitions . update ( utils . child_table_names ( conn , op_partition ) ) observed_invariants = { x for x in utils . table_invariants_list ( conn , operator_monthly_partitions , [ 'operator_id' , 'triplet_year' , 'triplet_month' ] ) } observed_invariants = { x for x in observed_invariants if x . triplet_year == year and x . triplet_month == month } required_invariants = { ( o . id , year , month ) for o in operators } missing_invariants = required_invariants - observed_invariants if len ( missing_invariants ) > 0 : msg = 'Missing monthly_network_triplets_per_mno partitions for the requested reporting ' 'month for the following configured operators: {0}' . format ( ', ' . join ( [ x [ 0 ] for x in missing_invariants ] ) ) if disable_data_check : logger . warning ( msg ) else : logger . error ( msg ) raise exceptions . MissingOperatorDataException ( msg ) extra_invariants = observed_invariants - required_invariants if len ( extra_invariants ) > 0 : msg = 'Extra monthly_network_triplets_per_mno partitions detected for the requested ' 'reporting month for the following unconfigured operators: {0}' . format ( ', ' . join ( [ x [ 0 ] for x in extra_invariants ] ) ) if disable_data_check : logger . warning ( msg ) else : logger . error ( msg ) raise exceptions . ExtraOperatorDataException ( msg ) country_imei_shard_name = partition_utils . monthly_network_triplets_country_partition ( month = month , year = year ) with conn . cursor ( ) as cursor : cursor . execute ( utils . table_exists_sql ( ) , [ country_imei_shard_name ] ) partition_exists = cursor . fetchone ( ) [ 0 ] if not partition_exists : msg = 'Missing monthly_network_triplets_country partition for year and month' if disable_data_check : logger . warning ( msg ) else : logger . error ( msg ) raise exceptions . ExtraOperatorDataException ( msg )
Validate that data is present for all configured operators and only configured operators.
Validate that data is present for all configured operators and only configured operators .
"""
    Validate that data is present for all configured operators and only configured operators.

    Arguments:
        config: DIRBS config object
        conn: DIRBS postgresql connection object
        month: data partition month
        year: data partition year
        logger: DIRBS logger object
        disable_data_check: boolean to disable data check
    Returns:
        None
    Raises:
        MissingOperatorDataException: if monthly_network_triplets_per_mno partition is missing for any operator
        ExtraOperatorDataException: if monthly_network_triplets_per_mno partition is detected for unconfigured mno
    """
None
if monthly_network_triplets_per_mno partition is missing for any operator
if monthly_network_triplets_per_mno partition is missing for any operator
MissingOperatorDataException
if monthly_network_triplets_per_mno partition is detected for unconfigured mno
if monthly_network_triplets_per_mno partition is detected for unconfigured mno
ExtraOperatorDataException
config
callable
DIRBS config object
DIRBS config object
conn
callable
DIRBS postgresql connection object
DIRBS postgresql connection object
month
int
data partition month
data partition month
year
int
data partition year
data partition year
logger
callable
DIRBS logger object
DIRBS logger object
disable_data_check
bool
boolean to disable data check
boolean to disable data check