def list_runids(s3_client, full_path):
    """Return list of all run ids inside S3 folder. It does not respect
    S3 pagination (`MaxKeys`) and returns **all** keys from bucket
    and won't list any prefixes with object archived to AWS Glacier

    Arguments:
    s3_client - boto3 S3 client (not service)
    full_path - full valid S3 path to events (such as enriched-archive)
                example: s3://acme-events-bucket/main-pipeline/enriched-archive
    """
    listing_finished = False                 # last response was not truncated
    run_ids_buffer = []
    last_continuation_token = None

    (bucket, prefix) = split_full_path(full_path)

    while not listing_finished:
        options = clean_dict({
            'Bucket': bucket,
            'Prefix': prefix,
            'Delimiter': '/',
            'ContinuationToken': last_continuation_token
        })

        response = s3_client.list_objects_v2(**options)
        keys = [extract_run_id(key['Prefix']) for key
                in response.get('CommonPrefixes', [])]
        run_ids_buffer.extend([key for key in keys if key is not None])
        last_continuation_token = response.get('NextContinuationToken', None)

        if not response['IsTruncated']:
            listing_finished = True

    non_archived_run_ids = [run_id for run_id in run_ids_buffer
                            if not is_glacier(s3_client, bucket, run_id)]
    return non_archived_run_ids
def list_runids(s3_client, full_path):
    """Return list of all run ids inside S3 folder. It does not respect
    S3 pagination (`MaxKeys`) and returns **all** keys from bucket
    and won't list any prefixes with object archived to AWS Glacier

    Arguments:
    s3_client - boto3 S3 client (not service)
    full_path - full valid S3 path to events (such as enriched-archive)
                example: s3://acme-events-bucket/main-pipeline/enriched-archive
    """
    listing_finished = False                 # last response was not truncated
    run_ids_buffer = []
    last_continuation_token = None

    (bucket, prefix) = split_full_path(full_path)

    while not listing_finished:
        options = clean_dict({
            'Bucket': bucket,
            'Prefix': prefix,
            'Delimiter': '/',
            'ContinuationToken': last_continuation_token
        })

        response = s3_client.list_objects_v2(**options)
        keys = [extract_run_id(key['Prefix']) for key
                in response.get('CommonPrefixes', [])]
        run_ids_buffer.extend([key for key in keys if key is not None])
        last_continuation_token = response.get('NextContinuationToken', None)

        if not response['IsTruncated']:
            listing_finished = True

    non_archived_run_ids = [run_id for run_id in run_ids_buffer
                            if not is_glacier(s3_client, bucket, run_id)]
    return non_archived_run_ids
def list_runids ( s3_client , full_path ) : listing_finished = False # last response was not truncated run_ids_buffer = [ ] last_continuation_token = None ( bucket , prefix ) = split_full_path ( full_path ) while not listing_finished : options = clean_dict ( { 'Bucket' : bucket , 'Prefix' : prefix , 'Delimiter' : '/' , 'ContinuationToken' : last_continuation_token } ) response = s3_client . list_objects_v2 ( * * options ) keys = [ extract_run_id ( key [ 'Prefix' ] ) for key in response . get ( 'CommonPrefixes' , [ ] ) ] run_ids_buffer . extend ( [ key for key in keys if key is not None ] ) last_continuation_token = response . get ( 'NextContinuationToken' , None ) if not response [ 'IsTruncated' ] : listing_finished = True non_archived_run_ids = [ run_id for run_id in run_ids_buffer if not is_glacier ( s3_client , bucket , run_id ) ] return non_archived_run_ids
Return list of all run ids inside S3 folder. It does not respect
    S3 pagination (`MaxKeys`) and returns **all** keys from bucket
    and won't list any prefixes with object archived to AWS Glacier

    Arguments:
    s3_client - boto3 S3 client (not service)
    full_path - full valid S3 path to events (such as enriched-archive)
                example: s3://acme-events-bucket/main-pipeline/enriched-archive
Return list of all run ids inside S3 folder . It does not respect S3 pagination ( MaxKeys ) and returns ** all ** keys from bucket and won t list any prefixes with object archived to AWS Glacier