python/ray/data/impl/lazy_block_list.py
def iter_blocks_with_metadata(
        self,
        block_for_metadata: bool = False,
    ) -> Iterator[Tuple[ObjectRef[Block], BlockMetadata]]:
        """Iterate over the blocks along with their metadata.

        Note that, if block_for_metadata is False (default), this iterator returns
        pre-read metadata from the ReadTasks given to this LazyBlockList so it doesn't
        have to block on the execution of the read tasks. Therefore, the metadata may be
        under-specified, e.g. missing schema or the number of rows. If fully-specified
        block metadata is required, pass block_for_metadata=True.

        The length of this iterator is not known until execution.

        Args:
            block_for_metadata: Whether we should block on the execution of read tasks
                in order to obtain fully-specified block metadata.

        Returns:
            An iterator of block references and the corresponding block metadata.
        """
        context = DatasetContext.get_current()
        outer = self

        class Iter:
            def __init__(self):
                self._base_iter = outer._iter_block_partition_refs()
                self._pos = -1
                self._buffer = []

            def __iter__(self):
                return self

            def __next__(self):
                while not self._buffer:
                    self._pos += 1
                    if context.block_splitting_enabled:
                        part_ref, _ = next(self._base_iter)
                        partition = ray.get(part_ref)
                    else:
                        block_ref, metadata_ref = next(self._base_iter)
                        if block_for_metadata:
                            # This blocks until the read task completes, returning
                            # fully-specified block metadata.
                            metadata = ray.get(metadata_ref)
                        else:
                            # This does not block, returning (possibly under-specified)
                            # pre-read block metadata.
                            metadata = outer._tasks[self._pos].get_metadata()
                        partition = [(block_ref, metadata)]
                    for block_ref, metadata in partition:
                        self._buffer.append((block_ref, metadata))
                return self._buffer.pop(0)

        return Iter()
Iterate over the blocks along with their metadata.

        Note that, if block_for_metadata is False (default), this iterator returns
        pre-read metadata from the ReadTasks given to this LazyBlockList so it doesn't
        have to block on the execution of the read tasks. Therefore, the metadata may be
        under-specified, e.g. missing schema or the number of rows. If fully-specified
        block metadata is required, pass block_for_metadata=True.

        The length of this iterator is not known until execution.

        Args:
            block_for_metadata: Whether we should block on the execution of read tasks
                in order to obtain fully-specified block metadata.

        Returns:
            An iterator of block references and the corresponding block metadata.
        
Iterate over the blocks along with their metadata.
Note that, if block_for_metadata is False (default), this iterator returns
pre-read metadata from the ReadTasks given to this LazyBlockList so it doesn't
have to block on the execution of the read tasks. Therefore, the metadata may be
under-specified, e.g. missing schema or the number of rows. If fully-specified
block metadata is required, pass block_for_metadata=True.

The length of this iterator is not known until execution.
Iterate over the blocks along with their metadata . Note that if block_for_metadata is False ( default ) this iterator returns pre - read metadata from the ReadTasks given to this LazyBlockList so it doesn ' t have to block on the execution of the read tasks . Therefore the metadata may be under - specified e . g . missing schema or the number of rows . If fully - specified block metadata is required pass block_for_metadata = True . The length of this iterator is not known until execution .
def iter_blocks_with_metadata(
        self,
        block_for_metadata: bool = False,
    ) -> Iterator[Tuple[ObjectRef[Block], BlockMetadata]]:
        context = DatasetContext.get_current()
        outer = self
        class Iter:
            def __init__(self):
                self._base_iter = outer._iter_block_partition_refs()
                self._pos = -1
                self._buffer = []
            def __iter__(self):
                return self
            def __next__(self):
                while not self._buffer:
                    self._pos += 1
                    if context.block_splitting_enabled:
                        part_ref, _ = next(self._base_iter)
                        partition = ray.get(part_ref)
                    else:
                        block_ref, metadata_ref = next(self._base_iter)
                        if block_for_metadata:
                            metadata = ray.get(metadata_ref)
                        else:
                            metadata = outer._tasks[self._pos].get_metadata()
                        partition = [(block_ref, metadata)]
                    for block_ref, metadata in partition:
                        self._buffer.append((block_ref, metadata))
                return self._buffer.pop(0)
        return Iter()
def iter_blocks_with_metadata ( self , block_for_metadata : bool = False , ) -> Iterator [ Tuple [ ObjectRef [ Block ] , BlockMetadata ] ] : context = DatasetContext . get_current ( ) outer = self class Iter : def __init__ ( self ) : self . _base_iter = outer . _iter_block_partition_refs ( ) self . _pos = - 1 self . _buffer = [ ] def __iter__ ( self ) : return self def __next__ ( self ) : while not self . _buffer : self . _pos += 1 if context . block_splitting_enabled : part_ref , _ = next ( self . _base_iter ) partition = ray . get ( part_ref ) else : block_ref , metadata_ref = next ( self . _base_iter ) if block_for_metadata : metadata = ray . get ( metadata_ref ) else : metadata = outer . _tasks [ self . _pos ] . get_metadata ( ) partition = [ ( block_ref , metadata ) ] for block_ref , metadata in partition : self . _buffer . append ( ( block_ref , metadata ) ) return self . _buffer . pop ( 0 ) return Iter ( )
Iterate over the blocks along with their metadata.
Iterate over the blocks along with their metadata .
"""Iterate over the blocks along with their metadata.

        Note that, if block_for_metadata is False (default), this iterator returns
        pre-read metadata from the ReadTasks given to this LazyBlockList so it doesn't
        have to block on the execution of the read tasks. Therefore, the metadata may be
        under-specified, e.g. missing schema or the number of rows. If fully-specified
        block metadata is required, pass block_for_metadata=True.

        The length of this iterator is not known until execution.

        Args:
            block_for_metadata: Whether we should block on the execution of read tasks
                in order to obtain fully-specified block metadata.

        Returns:
            An iterator of block references and the corresponding block metadata.
        """ # This blocks until the read task completes, returning # fully-specified block metadata. # This does not block, returning (possibly under-specified) # pre-read block metadata.
An iterator of block references and the corresponding block metadata.
An iterator of block references and the corresponding block metadata .
self
block_for_metadata
bool
Whether we should block on the execution of read tasks
in order to obtain fully-specified block metadata.
Whether we should block on the execution of read tasks in order to obtain fully - specified block metadata .