src/dirbs/cli/prune.py
def blacklist(ctx, config, statsd, logger, run_id, conn, metadata_conn, command,
              metrics_root, metrics_run_root, condition_name, prune_all):
    """Expire IMEIs outside the blacklist retention period from blacklist."""
    current_date = datetime.date.today()
    retention_days = config.retention_config.blacklist_retention

    if condition_name is None and prune_all is False:
        logger.info('Error: one of the arguments "condition_name" or "--prune-all" is required')
        metadata.add_optional_job_metadata(metadata_conn, command, run_id,
                                           curr_date=current_date.isoformat(),
                                           retention_days=retention_days,
                                           job_executed=False)
    elif condition_name is not None and prune_all is True:
        logger.info('Error: only one of the arguments "condition_name" or "--prune-all" is required')
        metadata.add_optional_job_metadata(metadata_conn, command, run_id,
                                           curr_date=current_date.isoformat(),
                                           retention_days=retention_days,
                                           job_executed=False)
    elif retention_days == 0:
        logger.info('Blacklist will not be prune, as retention value is set to {0}'.format(retention_days))
        metadata.add_optional_job_metadata(metadata_conn, command, run_id,
                                           curr_date=current_date.isoformat(),
                                           retention_days=retention_days,
                                           job_executed=False)
    else:
        _warn_about_prune_all(prune_all, logger)
        logger.info('Pruning blacklist to remove any data related to specified condition '
                    'outside the retention window.')
        last_retention_date = datetime.date(current_date.year,
                                            current_date.month,
                                            current_date.day) - datetime.timedelta(retention_days)

        # store metadata
        logger.info('Blacklist entries with start_date earlier than {0} will be pruned'.format(last_retention_date))
        metadata.add_optional_job_metadata(metadata_conn, command, run_id,
                                           curr_date=current_date.isoformat(),
                                           retention_days=retention_days,
                                           job_executed=True,
                                           last_retention_date=last_retention_date.isoformat())

        with utils.db_role_setter(conn, role_name='dirbs_core_power_user'), conn.cursor() as cursor:
            logger.debug('Calculating original number of rows with block_date in classification_state table...')

            cursor.execute("""SELECT COUNT(*)
                                FROM classification_state
                               WHERE block_date IS NOT NULL
                                 AND end_date IS NULL""")
            rows_before = cursor.fetchone()[0]

            logger.debug('Calculated original number of rows (having block_date) in classification_state table')
            statsd.gauge('{0}rows_before'.format(metrics_run_root), rows_before)
            metadata.add_optional_job_metadata(metadata_conn, command, run_id, rows_before=rows_before)

            # if its a condition based pruning
            if not prune_all:
                cursor.execute(sql.SQL("""SELECT COUNT(*)
                                            FROM classification_state
                                           WHERE start_date < %s
                                             AND cond_name = %s
                                             AND end_date IS NULL
                                             AND block_date IS NOT NULL"""),
                               [last_retention_date, condition_name[0].label])
                total_rows_to_prune = cursor.fetchone()[0]

                logger.info('Found {0:d} rows of classification_state table '
                            'with start_date for {1} dimension outside the blacklist '
                            'retention window.'.format(total_rows_to_prune, condition_name[0].label))

                if total_rows_to_prune > 0:
                    cursor.execute(sql.SQL("""UPDATE classification_state
                                                 SET end_date = '{0}'
                                               WHERE start_date < '{1}'
                                                 AND cond_name = '{2}'
                                                 AND end_date IS NULL
                                                 AND block_date IS NOT NULL""".format(current_date.isoformat(),
                                                                                      last_retention_date,
                                                                                      condition_name[0].label)))

                logger.info('Pruned {0:d} rows from blacklist for {1} dimension'.format(
                    total_rows_to_prune, condition_name[0].label))

            # prune without any condition
            else:
                cursor.execute(sql.SQL("""SELECT COUNT(*)
                                            FROM classification_state
                                           WHERE start_date < %s
                                             AND end_date IS NULL
                                             AND block_date IS NOT NULL"""), [last_retention_date])
                total_rows_to_prune = cursor.fetchone()[0]

                logger.info('Found {0:d} rows of classification_state table '
                            'with start_date outside the blacklist retention window.'.format(total_rows_to_prune))

                if total_rows_to_prune > 0:
                    cursor.execute(sql.SQL("""UPDATE classification_state
                                                 SET end_date = '{0}'
                                               WHERE start_date < '{1}'
                                                 AND end_date IS NULL
                                                 AND block_date IS NOT NULL""".format(current_date.isoformat(),
                                                                                      last_retention_date)))
                logger.info('Pruned {0:d} rows from blacklist'.format(total_rows_to_prune))

            logger.debug('Calculating remaining number of rows with block_date (end_date is null) '
                         'in classification_state table...')
            cursor.execute("""SELECT COUNT(*)
                                FROM classification_state
                               WHERE block_date IS NOT NULL
                                 AND end_date IS NULL""")
            rows_after = cursor.fetchone()[0]

            logger.debug('Calculated remaining number of rows (having block_date and end_date null) '
                         'in classification_state table')
            statsd.gauge('{0}rows_after'.format(metrics_run_root), rows_after)
            metadata.add_optional_job_metadata(metadata_conn, command, run_id, rows_after=rows_after)
Expire IMEIs outside the blacklist retention period from blacklist.
Expire IMEIs outside the blacklist retention period from blacklist.
Expire IMEIs outside the blacklist retention period from blacklist .
def blacklist(ctx, config, statsd, logger, run_id, conn, metadata_conn, command,
              metrics_root, metrics_run_root, condition_name, prune_all):
    current_date = datetime.date.today()
    retention_days = config.retention_config.blacklist_retention
    if condition_name is None and prune_all is False:
        logger.info('Error: one of the arguments "condition_name" or "--prune-all" is required')
        metadata.add_optional_job_metadata(metadata_conn, command, run_id,
                                           curr_date=current_date.isoformat(),
                                           retention_days=retention_days,
                                           job_executed=False)
    elif condition_name is not None and prune_all is True:
        logger.info('Error: only one of the arguments "condition_name" or "--prune-all" is required')
        metadata.add_optional_job_metadata(metadata_conn, command, run_id,
                                           curr_date=current_date.isoformat(),
                                           retention_days=retention_days,
                                           job_executed=False)
    elif retention_days == 0:
        logger.info('Blacklist will not be prune, as retention value is set to {0}'.format(retention_days))
        metadata.add_optional_job_metadata(metadata_conn, command, run_id,
                                           curr_date=current_date.isoformat(),
                                           retention_days=retention_days,
                                           job_executed=False)
    else:
        _warn_about_prune_all(prune_all, logger)
        logger.info('Pruning blacklist to remove any data related to specified condition '
                    'outside the retention window.')
        last_retention_date = datetime.date(current_date.year,
                                            current_date.month,
                                            current_date.day) - datetime.timedelta(retention_days)
        logger.info('Blacklist entries with start_date earlier than {0} will be pruned'.format(last_retention_date))
        metadata.add_optional_job_metadata(metadata_conn, command, run_id,
                                           curr_date=current_date.isoformat(),
                                           retention_days=retention_days,
                                           job_executed=True,
                                           last_retention_date=last_retention_date.isoformat())
        with utils.db_role_setter(conn, role_name='dirbs_core_power_user'), conn.cursor() as cursor:
            logger.debug('Calculating original number of rows with block_date in classification_state table...')
            cursor.execute("""SELECT COUNT(*)
                                FROM classification_state
                               WHERE block_date IS NOT NULL
                                 AND end_date IS NULL""")
            rows_before = cursor.fetchone()[0]
            logger.debug('Calculated original number of rows (having block_date) in classification_state table')
            statsd.gauge('{0}rows_before'.format(metrics_run_root), rows_before)
            metadata.add_optional_job_metadata(metadata_conn, command, run_id, rows_before=rows_before)
            if not prune_all:
                cursor.execute(sql.SQL("""SELECT COUNT(*)
                                            FROM classification_state
                                           WHERE start_date < %s
                                             AND cond_name = %s
                                             AND end_date IS NULL
                                             AND block_date IS NOT NULL"""),
                               [last_retention_date, condition_name[0].label])
                total_rows_to_prune = cursor.fetchone()[0]
                logger.info('Found {0:d} rows of classification_state table '
                            'with start_date for {1} dimension outside the blacklist '
                            'retention window.'.format(total_rows_to_prune, condition_name[0].label))
                if total_rows_to_prune > 0:
                    cursor.execute(sql.SQL("""UPDATE classification_state
                                                 SET end_date = '{0}'
                                               WHERE start_date < '{1}'
                                                 AND cond_name = '{2}'
                                                 AND end_date IS NULL
                                                 AND block_date IS NOT NULL""".format(current_date.isoformat(),
                                                                                      last_retention_date,
                                                                                      condition_name[0].label)))
                logger.info('Pruned {0:d} rows from blacklist for {1} dimension'.format(
                    total_rows_to_prune, condition_name[0].label))
            else:
                cursor.execute(sql.SQL("""SELECT COUNT(*)
                                            FROM classification_state
                                           WHERE start_date < %s
                                             AND end_date IS NULL
                                             AND block_date IS NOT NULL"""), [last_retention_date])
                total_rows_to_prune = cursor.fetchone()[0]
                logger.info('Found {0:d} rows of classification_state table '
                            'with start_date outside the blacklist retention window.'.format(total_rows_to_prune))
                if total_rows_to_prune > 0:
                    cursor.execute(sql.SQL("""UPDATE classification_state
                                                 SET end_date = '{0}'
                                               WHERE start_date < '{1}'
                                                 AND end_date IS NULL
                                                 AND block_date IS NOT NULL""".format(current_date.isoformat(),
                                                                                      last_retention_date)))
                logger.info('Pruned {0:d} rows from blacklist'.format(total_rows_to_prune))
            logger.debug('Calculating remaining number of rows with block_date (end_date is null) '
                         'in classification_state table...')
            cursor.execute("""SELECT COUNT(*)
                                FROM classification_state
                               WHERE block_date IS NOT NULL
                                 AND end_date IS NULL""")
            rows_after = cursor.fetchone()[0]
            logger.debug('Calculated remaining number of rows (having block_date and end_date null) '
                         'in classification_state table')
            statsd.gauge('{0}rows_after'.format(metrics_run_root), rows_after)
            metadata.add_optional_job_metadata(metadata_conn, command, run_id, rows_after=rows_after)
def blacklist ( ctx , config , statsd , logger , run_id , conn , metadata_conn , command , metrics_root , metrics_run_root , condition_name , prune_all ) : current_date = datetime . date . today ( ) retention_days = config . retention_config . blacklist_retention if condition_name is None and prune_all is False : logger . info ( 'Error: one of the arguments "condition_name" or "--prune-all" is required' ) metadata . add_optional_job_metadata ( metadata_conn , command , run_id , curr_date = current_date . isoformat ( ) , retention_days = retention_days , job_executed = False ) elif condition_name is not None and prune_all is True : logger . info ( 'Error: only one of the arguments "condition_name" or "--prune-all" is required' ) metadata . add_optional_job_metadata ( metadata_conn , command , run_id , curr_date = current_date . isoformat ( ) , retention_days = retention_days , job_executed = False ) elif retention_days == 0 : logger . info ( 'Blacklist will not be prune, as retention value is set to {0}' . format ( retention_days ) ) metadata . add_optional_job_metadata ( metadata_conn , command , run_id , curr_date = current_date . isoformat ( ) , retention_days = retention_days , job_executed = False ) else : _warn_about_prune_all ( prune_all , logger ) logger . info ( 'Pruning blacklist to remove any data related to specified condition ' 'outside the retention window.' ) last_retention_date = datetime . date ( current_date . year , current_date . month , current_date . day ) - datetime . timedelta ( retention_days ) logger . info ( 'Blacklist entries with start_date earlier than {0} will be pruned' . format ( last_retention_date ) ) metadata . add_optional_job_metadata ( metadata_conn , command , run_id , curr_date = current_date . isoformat ( ) , retention_days = retention_days , job_executed = True , last_retention_date = last_retention_date . isoformat ( ) ) with utils . db_role_setter ( conn , role_name = 'dirbs_core_power_user' ) , conn . cursor ( ) as cursor : logger . debug ( 'Calculating original number of rows with block_date in classification_state table...' ) cursor . execute ( """SELECT COUNT(*)
                                FROM classification_state
                               WHERE block_date IS NOT NULL
                                 AND end_date IS NULL""" ) rows_before = cursor . fetchone ( ) [ 0 ] logger . debug ( 'Calculated original number of rows (having block_date) in classification_state table' ) statsd . gauge ( '{0}rows_before' . format ( metrics_run_root ) , rows_before ) metadata . add_optional_job_metadata ( metadata_conn , command , run_id , rows_before = rows_before ) if not prune_all : cursor . execute ( sql . SQL ( """SELECT COUNT(*)
                                            FROM classification_state
                                           WHERE start_date < %s
                                             AND cond_name = %s
                                             AND end_date IS NULL
                                             AND block_date IS NOT NULL""" ) , [ last_retention_date , condition_name [ 0 ] . label ] ) total_rows_to_prune = cursor . fetchone ( ) [ 0 ] logger . info ( 'Found {0:d} rows of classification_state table ' 'with start_date for {1} dimension outside the blacklist ' 'retention window.' . format ( total_rows_to_prune , condition_name [ 0 ] . label ) ) if total_rows_to_prune > 0 : cursor . execute ( sql . SQL ( """UPDATE classification_state
                                                 SET end_date = '{0}'
                                               WHERE start_date < '{1}'
                                                 AND cond_name = '{2}'
                                                 AND end_date IS NULL
                                                 AND block_date IS NOT NULL""" . format ( current_date . isoformat ( ) , last_retention_date , condition_name [ 0 ] . label ) ) ) logger . info ( 'Pruned {0:d} rows from blacklist for {1} dimension' . format ( total_rows_to_prune , condition_name [ 0 ] . label ) ) else : cursor . execute ( sql . SQL ( """SELECT COUNT(*)
                                            FROM classification_state
                                           WHERE start_date < %s
                                             AND end_date IS NULL
                                             AND block_date IS NOT NULL""" ) , [ last_retention_date ] ) total_rows_to_prune = cursor . fetchone ( ) [ 0 ] logger . info ( 'Found {0:d} rows of classification_state table ' 'with start_date outside the blacklist retention window.' . format ( total_rows_to_prune ) ) if total_rows_to_prune > 0 : cursor . execute ( sql . SQL ( """UPDATE classification_state
                                                 SET end_date = '{0}'
                                               WHERE start_date < '{1}'
                                                 AND end_date IS NULL
                                                 AND block_date IS NOT NULL""" . format ( current_date . isoformat ( ) , last_retention_date ) ) ) logger . info ( 'Pruned {0:d} rows from blacklist' . format ( total_rows_to_prune ) ) logger . debug ( 'Calculating remaining number of rows with block_date (end_date is null) ' 'in classification_state table...' ) cursor . execute ( """SELECT COUNT(*)
                                FROM classification_state
                               WHERE block_date IS NOT NULL
                                 AND end_date IS NULL""" ) rows_after = cursor . fetchone ( ) [ 0 ] logger . debug ( 'Calculated remaining number of rows (having block_date and end_date null) ' 'in classification_state table' ) statsd . gauge ( '{0}rows_after' . format ( metrics_run_root ) , rows_after ) metadata . add_optional_job_metadata ( metadata_conn , command , run_id , rows_after = rows_after )
Expire IMEIs outside the blacklist retention period from blacklist.
Expire IMEIs outside the blacklist retention period from blacklist .
"""Expire IMEIs outside the blacklist retention period from blacklist.""" # store metadata # if its a condition based pruning # prune without any condition
ctx
config
statsd
logger
run_id
conn
metadata_conn
command
metrics_root
metrics_run_root
condition_name
prune_all