python/ray/data/impl/lazy_block_list.py
def _get_blocks_with_metadata(
        self,
    ) -> Tuple[List[ObjectRef[Block]], List[BlockMetadata]]:
        """Get all underlying block futures and concrete metadata.

        This will block on the completion of the underlying read tasks and will fetch
        all block metadata outputted by those tasks.
        """
        context = DatasetContext.get_current()
        block_refs, meta_refs = [], []
        for block_ref, meta_ref in self._iter_block_partition_refs():
            block_refs.append(block_ref)
            meta_refs.append(meta_ref)
        if context.block_splitting_enabled:
            # If block splitting is enabled, fetch the partitions.
            parts = ray.get(block_refs)
            block_refs, metadata = [], []
            for part in parts:
                for block_ref, meta in part:
                    block_refs.append(block_ref)
                    metadata.append(meta)
            self._cached_metadata = metadata
            return block_refs, metadata
        if all(meta is not None for meta in self._cached_metadata):
            # Short-circuit on cached metadata.
            return block_refs, self._cached_metadata
        if not meta_refs:
            # Short-circuit on empty set of block partitions.
            assert not block_refs, block_refs
            return [], []
        read_progress_bar = ProgressBar("Read progress", total=len(meta_refs))
        # Fetch the metadata in bulk.
        # Handle duplicates (e.g. due to unioning the same dataset).
        unique_meta_refs = set(meta_refs)
        metadata = read_progress_bar.fetch_until_complete(list(unique_meta_refs))
        ref_to_data = {
            meta_ref: data for meta_ref, data in zip(unique_meta_refs, metadata)
        }
        metadata = [ref_to_data[meta_ref] for meta_ref in meta_refs]
        self._cached_metadata = metadata
        return block_refs, metadata
Get all underlying block futures and concrete metadata.

        This will block on the completion of the underlying read tasks and will fetch
        all block metadata outputted by those tasks.
        
Get all underlying block futures and concrete metadata.
This will block on the completion of the underlying read tasks and will fetch
all block metadata outputted by those tasks.
Get all underlying block futures and concrete metadata . This will block on the completion of the underlying read tasks and will fetch all block metadata outputted by those tasks .
def _get_blocks_with_metadata(
        self,
    ) -> Tuple[List[ObjectRef[Block]], List[BlockMetadata]]:
        context = DatasetContext.get_current()
        block_refs, meta_refs = [], []
        for block_ref, meta_ref in self._iter_block_partition_refs():
            block_refs.append(block_ref)
            meta_refs.append(meta_ref)
        if context.block_splitting_enabled:
            parts = ray.get(block_refs)
            block_refs, metadata = [], []
            for part in parts:
                for block_ref, meta in part:
                    block_refs.append(block_ref)
                    metadata.append(meta)
            self._cached_metadata = metadata
            return block_refs, metadata
        if all(meta is not None for meta in self._cached_metadata):
            return block_refs, self._cached_metadata
        if not meta_refs:
            assert not block_refs, block_refs
            return [], []
        read_progress_bar = ProgressBar("Read progress", total=len(meta_refs))
        unique_meta_refs = set(meta_refs)
        metadata = read_progress_bar.fetch_until_complete(list(unique_meta_refs))
        ref_to_data = {
            meta_ref: data for meta_ref, data in zip(unique_meta_refs, metadata)
        }
        metadata = [ref_to_data[meta_ref] for meta_ref in meta_refs]
        self._cached_metadata = metadata
        return block_refs, metadata
def _get_blocks_with_metadata ( self , ) -> Tuple [ List [ ObjectRef [ Block ] ] , List [ BlockMetadata ] ] : context = DatasetContext . get_current ( ) block_refs , meta_refs = [ ] , [ ] for block_ref , meta_ref in self . _iter_block_partition_refs ( ) : block_refs . append ( block_ref ) meta_refs . append ( meta_ref ) if context . block_splitting_enabled : parts = ray . get ( block_refs ) block_refs , metadata = [ ] , [ ] for part in parts : for block_ref , meta in part : block_refs . append ( block_ref ) metadata . append ( meta ) self . _cached_metadata = metadata return block_refs , metadata if all ( meta is not None for meta in self . _cached_metadata ) : return block_refs , self . _cached_metadata if not meta_refs : assert not block_refs , block_refs return [ ] , [ ] read_progress_bar = ProgressBar ( "Read progress" , total = len ( meta_refs ) ) unique_meta_refs = set ( meta_refs ) metadata = read_progress_bar . fetch_until_complete ( list ( unique_meta_refs ) ) ref_to_data = { meta_ref : data for meta_ref , data in zip ( unique_meta_refs , metadata ) } metadata = [ ref_to_data [ meta_ref ] for meta_ref in meta_refs ] self . _cached_metadata = metadata return block_refs , metadata
Get all underlying block futures and concrete metadata.
Get all underlying block futures and concrete metadata .
"""Get all underlying block futures and concrete metadata.

        This will block on the completion of the underlying read tasks and will fetch
        all block metadata outputted by those tasks.
        """ # If block splitting is enabled, fetch the partitions. # Short-circuit on cached metadata. # Short-circuit on empty set of block partitions. # Fetch the metadata in bulk. # Handle duplicates (e.g. due to unioning the same dataset).
self