src/whylogs/util/cloudstorage.py
def list_objects(
    bucket: str,
    prefix: str = None,
    suffix: str = None,
    start_after: str = None,
    client=None,
):
    """
    Generator to list objects in an S3 bucket.

    Parameters
    ----------
    bucket : str
        Name of the cloud bucket.
    prefix : str
        Only fetch keys that start with this prefix.
    suffix : str
        Only fetch keys that end with this suffix.
    start_after : str
        Where you want Amazon S3 to start listing from. Amazon S3 starts
        listing after this specified key. StartAfter can be any key in the
        bucket
    client :
        If specified, the boto3 s3 client
    """
    if prefix is None:
        prefix = ""
    if suffix is None:
        suffix = ""
    if client is None:
        client = s3client()
    kwargs = {"Bucket": bucket}

    # If the prefix is a single string (not a tuple of strings), we can
    # do the filtering directly in the S3 API.
    if isinstance(prefix, str):
        kwargs["Prefix"] = prefix
    if start_after:
        kwargs["StartAfter"] = start_after

    while True:

        # The S3 API response is a large blob of metadata.
        # 'Contents' contains information about the listed objects.
        resp = client.list_objects_v2(**kwargs)

        try:
            contents = resp["Contents"]
        except KeyError:
            return

        for obj in contents:
            key = obj["Key"]
            if key.startswith(prefix) and key.endswith(suffix):
                yield obj

        # The S3 API is paginated, returning up to 1000 keys at a time.
        # Pass the continuation token into the next response, until we
        # reach the final page (when this field is missing).
        try:
            kwargs["ContinuationToken"] = resp["NextContinuationToken"]
        except KeyError:
            break

    Generator to list objects in an S3 bucket.

    Parameters
    ----------
    bucket : str
        Name of the cloud bucket.
    prefix : str
        Only fetch keys that start with this prefix.
    suffix : str
        Only fetch keys that end with this suffix.
    start_after : str
        Where you want Amazon S3 to start listing from. Amazon S3 starts
        listing after this specified key. StartAfter can be any key in the
        bucket
    client :
        If specified, the boto3 s3 client
    
Generator to list objects in an S3 bucket.
Parameters

bucket : str
Name of the cloud bucket.
prefix : str
Only fetch keys that start with this prefix.
suffix : str
Only fetch keys that end with this suffix.
start_after : str
Where you want Amazon S3 to start listing from. Amazon S3 starts
listing after this specified key. StartAfter can be any key in the
bucket
client :
If specified, the boto3 s3 client
Generator to list objects in an S3 bucket . Parameters bucket : str Name of the cloud bucket . prefix : str Only fetch keys that start with this prefix . suffix : str Only fetch keys that end with this suffix . start_after : str Where you want Amazon S3 to start listing from . Amazon S3 starts listing after this specified key . StartAfter can be any key in the bucket client : If specified the boto3 s3 client
def list_objects(
    bucket: str,
    prefix: str = None,
    suffix: str = None,
    start_after: str = None,
    client=None,
):
    if prefix is None:
        prefix = ""
    if suffix is None:
        suffix = ""
    if client is None:
        client = s3client()
    kwargs = {"Bucket": bucket}
    if isinstance(prefix, str):
        kwargs["Prefix"] = prefix
    if start_after:
        kwargs["StartAfter"] = start_after
    while True:
        resp = client.list_objects_v2(**kwargs)
        try:
            contents = resp["Contents"]
        except KeyError:
            return
        for obj in contents:
            key = obj["Key"]
            if key.startswith(prefix) and key.endswith(suffix):
                yield obj
        try:
            kwargs["ContinuationToken"] = resp["NextContinuationToken"]
        except KeyError:
            break
def list_objects ( bucket : str , prefix : str = None , suffix : str = None , start_after : str = None , client = None , ) : if prefix is None : prefix = "" if suffix is None : suffix = "" if client is None : client = s3client ( ) kwargs = { "Bucket" : bucket } if isinstance ( prefix , str ) : kwargs [ "Prefix" ] = prefix if start_after : kwargs [ "StartAfter" ] = start_after while True : resp = client . list_objects_v2 ( ** kwargs ) try : contents = resp [ "Contents" ] except KeyError : return for obj in contents : key = obj [ "Key" ] if key . startswith ( prefix ) and key . endswith ( suffix ) : yield obj try : kwargs [ "ContinuationToken" ] = resp [ "NextContinuationToken" ] except KeyError : break
Generator to list objects in an S3 bucket.
Generator to list objects in an S3 bucket .
"""
    Generator to list objects in an S3 bucket.

    Parameters
    ----------
    bucket : str
        Name of the cloud bucket.
    prefix : str
        Only fetch keys that start with this prefix.
    suffix : str
        Only fetch keys that end with this suffix.
    start_after : str
        Where you want Amazon S3 to start listing from. Amazon S3 starts
        listing after this specified key. StartAfter can be any key in the
        bucket
    client :
        If specified, the boto3 s3 client
    """ # If the prefix is a single string (not a tuple of strings), we can # do the filtering directly in the S3 API. # The S3 API response is a large blob of metadata. # 'Contents' contains information about the listed objects. # The S3 API is paginated, returning up to 1000 keys at a time. # Pass the continuation token into the next response, until we # reach the final page (when this field is missing).
bucket
str
prefix
str
suffix
str
start_after
str
client