fsspec/parquet.py
def _get_parquet_byte_ranges(
    paths,
    fs,
    columns=None,
    row_groups=None,
    max_gap=64_000,
    max_block=256_000_000,
    footer_sample_size=1_000_000,
    engine="auto",
):
    """Get a dictionary of the known byte ranges needed
    to read a specific column/row-group selection from a
    Parquet dataset. Each value in the output dictionary
    is intended for use as the `data` argument for the
    `KnownPartsOfAFile` caching strategy of a single path.
    """

    # Set the engine
    engine = _set_engine(engine)

    # Get file sizes asynchronously
    file_sizes = fs.sizes(paths)

    # Populate global paths, starts, & ends
    result = {}
    data_paths = []
    data_starts = []
    data_ends = []
    add_header_magic = True
    if columns is None and row_groups is None:
        # We are NOT selecting specific columns or row-groups.
        #
        # We can avoid sampling the footers, and just transfer
        # all file data with cat_ranges
        for i, path in enumerate(paths):
            result[path] = {}
            for b in range(0, file_sizes[i], max_block):
                data_paths.append(path)
                data_starts.append(b)
                data_ends.append(min(b + max_block, file_sizes[i]))
        add_header_magic = False  # "Magic" should already be included
    else:
        # We ARE selecting specific columns or row-groups.
        #
        # Gather file footers.
        # We just take the last `footer_sample_size` bytes of each
        # file (or the entire file if it is smaller than that)
        footer_starts = []
        footer_ends = []
        for i, path in enumerate(paths):
            footer_ends.append(file_sizes[i])
            sample_size = max(0, file_sizes[i] - footer_sample_size)
            footer_starts.append(sample_size)
        footer_samples = fs.cat_ranges(paths, footer_starts, footer_ends)

        # Check our footer samples and re-sample if necessary.
        missing_footer_starts = footer_starts.copy()
        missing_footer_ends = footer_starts.copy()
        large_footer = 0
        for i, path in enumerate(paths):
            footer_size = int.from_bytes(footer_samples[i][-8:-4], "little")
            real_footer_start = file_sizes[i] - (footer_size + 8)
            if real_footer_start < footer_starts[i]:
                missing_footer_starts[i] = real_footer_start
                missing_footer_ends[i] = footer_starts[i]
                large_footer = max(large_footer, (footer_size + 8))
        if large_footer:
            warnings.warn(
                f"Not enough data was used to sample the parquet footer. "
                f"Try setting footer_sample_size >= {large_footer}."
            )
            for i, block in enumerate(
                fs.cat_ranges(
                    paths,
                    missing_footer_starts,
                    missing_footer_ends,
                )
            ):
                footer_samples[i] = block + footer_samples[i]

        # Calculate required byte ranges for each path
        for i, path in enumerate(paths):

            # Deal with small-file case.
            # Just include all remaining bytes of the file
            # in a single range.
            if file_sizes[i] < max_block:
                if footer_starts[i] > 0:
                    # Only need to transfer the data if the
                    # footer sample isn't already the whole file
                    data_paths.append(path)
                    data_starts.append(0)
                    data_ends.append(footer_starts[i])
                continue

            # Use "engine" to collect data byte ranges
            path_data_starts, path_data_ends = engine._parquet_byte_ranges(
                footer_samples[i], columns, row_groups, footer_starts[i]
            )

            data_paths += [path] * len(path_data_starts)
            data_starts += path_data_starts
            data_ends += path_data_ends

        # Merge adjacent offset ranges
        data_paths, data_starts, data_ends = merge_offset_ranges(
            data_paths,
            data_starts,
            data_ends,
            max_gap=max_gap,
            max_block=max_block,
            sort=False,  # Should already be sorted
        )

        # Start by populating `result` with footer samples
        for i, path in enumerate(paths):
            result[path] = {(footer_starts[i], footer_ends[i]): footer_samples[i]}

    # Use cat_ranges to gather the data byte_ranges
    for i, data in enumerate(fs.cat_ranges(data_paths, data_starts, data_ends)):
        if data_ends[i] > data_starts[i]:
            result[data_paths[i]][(data_starts[i], data_ends[i])] = data

    # Add b"PAR1" to header if necessary
    if add_header_magic:
        for i, path in enumerate(paths):
            add_magic = True
            for k in result[path].keys():
                if k[0] == 0 and k[1] >= 4:
                    add_magic = False
                    break
            if add_magic:
                result[path][(0, 4)] = b"PAR1"

    # Add b"" for reads beyond end of file
    for i, path in enumerate(paths):
        result[path][(file_sizes[i], 2 * file_sizes[i])] = b""

    return result
Get a dictionary of the known byte ranges needed
    to read a specific column/row-group selection from a
    Parquet dataset. Each value in the output dictionary
    is intended for use as the `data` argument for the
    `KnownPartsOfAFile` caching strategy of a single path.
    
Get a dictionary of the known byte ranges needed
to read a specific column/row-group selection from a
Parquet dataset. Each value in the output dictionary
is intended for use as the `data` argument for the
`KnownPartsOfAFile` caching strategy of a single path.
Get a dictionary of the known byte ranges needed to read a specific column / row - group selection from a Parquet dataset . Each value in the output dictionary is intended for use as the ` data ` argument for the ` KnownPartsOfAFile ` caching strategy of a single path .
def _get_parquet_byte_ranges(
    paths,
    fs,
    columns=None,
    row_groups=None,
    max_gap=64_000,
    max_block=256_000_000,
    footer_sample_size=1_000_000,
    engine="auto",
):
    engine = _set_engine(engine)
    file_sizes = fs.sizes(paths)
    result = {}
    data_paths = []
    data_starts = []
    data_ends = []
    add_header_magic = True
    if columns is None and row_groups is None:
         We can avoid sampling the footers, and just transfer
         all file data with cat_ranges
        for i, path in enumerate(paths):
            result[path] = {}
            for b in range(0, file_sizes[i], max_block):
                data_paths.append(path)
                data_starts.append(b)
                data_ends.append(min(b + max_block, file_sizes[i]))
        add_header_magic = False   "Magic" should already be included
    else:
         We ARE selecting specific columns or row-groups.
         Gather file footers.
         We just take the last `footer_sample_size` bytes of each
         file (or the entire file if it is smaller than that)
        footer_starts = []
        footer_ends = []
        for i, path in enumerate(paths):
            footer_ends.append(file_sizes[i])
            sample_size = max(0, file_sizes[i] - footer_sample_size)
            footer_starts.append(sample_size)
        footer_samples = fs.cat_ranges(paths, footer_starts, footer_ends)
         Check our footer samples and re-sample if necessary.
        missing_footer_starts = footer_starts.copy()
        missing_footer_ends = footer_starts.copy()
        large_footer = 0
        for i, path in enumerate(paths):
            footer_size = int.from_bytes(footer_samples[i][-8:-4], "little")
            real_footer_start = file_sizes[i] - (footer_size + 8)
            if real_footer_start < footer_starts[i]:
                missing_footer_starts[i] = real_footer_start
                missing_footer_ends[i] = footer_starts[i]
                large_footer = max(large_footer, (footer_size + 8))
        if large_footer:
            warnings.warn(
                f"Not enough data was used to sample the parquet footer. "
                f"Try setting footer_sample_size >= {large_footer}."
            )
            for i, block in enumerate(
                fs.cat_ranges(
                    paths,
                    missing_footer_starts,
                    missing_footer_ends,
                )
            ):
                footer_samples[i] = block + footer_samples[i]
         Calculate required byte ranges for each path
        for i, path in enumerate(paths):
             Deal with small-file case.
             Just include all remaining bytes of the file
             in a single range.
            if file_sizes[i] < max_block:
                if footer_starts[i] > 0:
                     Only need to transfer the data if the
                     footer sample isn't already the whole file
                    data_paths.append(path)
                    data_starts.append(0)
                    data_ends.append(footer_starts[i])
                continue
             Use "engine" to collect data byte ranges
            path_data_starts, path_data_ends = engine._parquet_byte_ranges(
                footer_samples[i], columns, row_groups, footer_starts[i]
            )
            data_paths += [path] * len(path_data_starts)
            data_starts += path_data_starts
            data_ends += path_data_ends
         Merge adjacent offset ranges
        data_paths, data_starts, data_ends = merge_offset_ranges(
            data_paths,
            data_starts,
            data_ends,
            max_gap=max_gap,
            max_block=max_block,
            sort=False,   Should already be sorted
        )
         Start by populating `result` with footer samples
        for i, path in enumerate(paths):
            result[path] = {(footer_starts[i], footer_ends[i]): footer_samples[i]}
     Use cat_ranges to gather the data byte_ranges
    for i, data in enumerate(fs.cat_ranges(data_paths, data_starts, data_ends)):
        if data_ends[i] > data_starts[i]:
            result[data_paths[i]][(data_starts[i], data_ends[i])] = data
     Add b"PAR1" to header if necessary
    if add_header_magic:
        for i, path in enumerate(paths):
            add_magic = True
            for k in result[path].keys():
                if k[0] == 0 and k[1] >= 4:
                    add_magic = False
                    break
            if add_magic:
                result[path][(0, 4)] = b"PAR1"
     Add b"" for reads beyond end of file
    for i, path in enumerate(paths):
        result[path][(file_sizes[i], 2 * file_sizes[i])] = b""
    return result
def _get_parquet_byte_ranges ( paths , fs , columns = None , row_groups = None , max_gap = 64_000 , max_block = 256_000_000 , footer_sample_size = 1_000_000 , engine = "auto" , ) : engine = _set_engine ( engine ) file_sizes = fs . sizes ( paths ) result = { } data_paths = [ ] data_starts = [ ] data_ends = [ ] add_header_magic = True if columns is None and row_groups is None : for i , path in enumerate ( paths ) : result [ path ] = { } for b in range ( 0 , file_sizes [ i ] , max_block ) : data_paths . append ( path ) data_starts . append ( b ) data_ends . append ( min ( b + max_block , file_sizes [ i ] ) ) add_header_magic = False else : footer_starts = [ ] footer_ends = [ ] for i , path in enumerate ( paths ) : footer_ends . append ( file_sizes [ i ] ) sample_size = max ( 0 , file_sizes [ i ] - footer_sample_size ) footer_starts . append ( sample_size ) footer_samples = fs . cat_ranges ( paths , footer_starts , footer_ends ) missing_footer_starts = footer_starts . copy ( ) missing_footer_ends = footer_starts . copy ( ) large_footer = 0 for i , path in enumerate ( paths ) : footer_size = int . from_bytes ( footer_samples [ i ] [ - 8 : - 4 ] , "little" ) real_footer_start = file_sizes [ i ] - ( footer_size + 8 ) if real_footer_start < footer_starts [ i ] : missing_footer_starts [ i ] = real_footer_start missing_footer_ends [ i ] = footer_starts [ i ] large_footer = max ( large_footer , ( footer_size + 8 ) ) if large_footer : warnings . warn ( f"Not enough data was used to sample the parquet footer. " f"Try setting footer_sample_size >= {large_footer}." ) for i , block in enumerate ( fs . cat_ranges ( paths , missing_footer_starts , missing_footer_ends , ) ) : footer_samples [ i ] = block + footer_samples [ i ] for i , path in enumerate ( paths ) : if file_sizes [ i ] < max_block : if footer_starts [ i ] > 0 : data_paths . append ( path ) data_starts . append ( 0 ) data_ends . append ( footer_starts [ i ] ) continue path_data_starts , path_data_ends = engine . _parquet_byte_ranges ( footer_samples [ i ] , columns , row_groups , footer_starts [ i ] ) data_paths += [ path ] * len ( path_data_starts ) data_starts += path_data_starts data_ends += path_data_ends data_paths , data_starts , data_ends = merge_offset_ranges ( data_paths , data_starts , data_ends , max_gap = max_gap , max_block = max_block , sort = False , ) for i , path in enumerate ( paths ) : result [ path ] = { ( footer_starts [ i ] , footer_ends [ i ] ) : footer_samples [ i ] } for i , data in enumerate ( fs . cat_ranges ( data_paths , data_starts , data_ends ) ) : if data_ends [ i ] > data_starts [ i ] : result [ data_paths [ i ] ] [ ( data_starts [ i ] , data_ends [ i ] ) ] = data if add_header_magic : for i , path in enumerate ( paths ) : add_magic = True for k in result [ path ] . keys ( ) : if k [ 0 ] == 0 and k [ 1 ] >= 4 : add_magic = False break if add_magic : result [ path ] [ ( 0 , 4 ) ] = b"PAR1" for i , path in enumerate ( paths ) : result [ path ] [ ( file_sizes [ i ] , 2 * file_sizes [ i ] ) ] = b"" return result
Get a dictionary of the known byte ranges needed
to read a specific column/row-group selection from a
Parquet dataset.
Get a dictionary of the known byte ranges needed to read a specific column / row - group selection from a Parquet dataset .
"""Get a dictionary of the known byte ranges needed
    to read a specific column/row-group selection from a
    Parquet dataset. Each value in the output dictionary
    is intended for use as the `data` argument for the
    `KnownPartsOfAFile` caching strategy of a single path.
    """ # Set the engine # Get file sizes asynchronously # Populate global paths, starts, & ends # We are NOT selecting specific columns or row-groups. # # We can avoid sampling the footers, and just transfer # all file data with cat_ranges # "Magic" should already be included # We ARE selecting specific columns or row-groups. # # Gather file footers. # We just take the last `footer_sample_size` bytes of each # file (or the entire file if it is smaller than that) # Check our footer samples and re-sample if necessary. # Calculate required byte ranges for each path # Deal with small-file case. # Just include all remaining bytes of the file # in a single range. # Only need to transfer the data if the # footer sample isn't already the whole file # Use "engine" to collect data byte ranges # Merge adjacent offset ranges # Should already be sorted # Start by populating `result` with footer samples # Use cat_ranges to gather the data byte_ranges # Add b"PAR1" to header if necessary # Add b"" for reads beyond end of file
paths
fs
columns
row_groups
max_gap
max_block
footer_sample_size
engine