python/ray/data/dataset_pipeline.py
def iter_batches(
        self,
        *,
        prefetch_blocks: int = 0,
        batch_size: int = None,
        batch_format: str = "native",
        drop_last: bool = False,
    ) -> Iterator[BatchType]:
        """Return a local batched iterator over the data in the pipeline.

        Examples:
            >>> import ray
            >>> ds = ray.data.range(1000000).repeat(5) # doctest: +SKIP
            >>> for pandas_df in ds.iter_batches(): # doctest: +SKIP
            ...     print(pandas_df) # doctest: +SKIP

        Time complexity: O(1)

        Args:
            prefetch_blocks: The number of blocks to prefetch ahead of the
                current block during the scan.
            batch_size: Record batch size, or None to let the system pick.
            batch_format: The format in which to return each batch.
                Specify "native" to use the current block format (promoting
                Arrow to pandas automatically), "pandas" to
                select ``pandas.DataFrame`` or "pyarrow" to select
                ``pyarrow.Table``. Default is "native".
            drop_last: Whether to drop the last batch if it's incomplete.

        Returns:
            An iterator over record batches.
        """
        time_start = time.perf_counter()
        yield from batch_blocks(
            self._iter_blocks(),
            self._stats,
            prefetch_blocks=prefetch_blocks,
            batch_size=batch_size,
            batch_format=batch_format,
            drop_last=drop_last,
        )
        self._stats.iter_total_s.add(time.perf_counter() - time_start)
Return a local batched iterator over the data in the pipeline.

        Examples:
            >>> import ray
            >>> ds = ray.data.range(1000000).repeat(5) # doctest: +SKIP
            >>> for pandas_df in ds.iter_batches(): # doctest: +SKIP
            ...     print(pandas_df) # doctest: +SKIP

        Time complexity: O(1)

        Args:
            prefetch_blocks: The number of blocks to prefetch ahead of the
                current block during the scan.
            batch_size: Record batch size, or None to let the system pick.
            batch_format: The format in which to return each batch.
                Specify "native" to use the current block format (promoting
                Arrow to pandas automatically), "pandas" to
                select ``pandas.DataFrame`` or "pyarrow" to select
                ``pyarrow.Table``. Default is "native".
            drop_last: Whether to drop the last batch if it's incomplete.

        Returns:
            An iterator over record batches.
        
Return a local batched iterator over the data in the pipeline.
Return a local batched iterator over the data in the pipeline .
def iter_batches(
        self,
        *,
        prefetch_blocks: int = 0,
        batch_size: int = None,
        batch_format: str = "native",
        drop_last: bool = False,
    ) -> Iterator[BatchType]:
        time_start = time.perf_counter()
        yield from batch_blocks(
            self._iter_blocks(),
            self._stats,
            prefetch_blocks=prefetch_blocks,
            batch_size=batch_size,
            batch_format=batch_format,
            drop_last=drop_last,
        )
        self._stats.iter_total_s.add(time.perf_counter() - time_start)
def iter_batches ( self , * , prefetch_blocks : int = 0 , batch_size : int = None , batch_format : str = "native" , drop_last : bool = False , ) -> Iterator [ BatchType ] : time_start = time . perf_counter ( ) yield from batch_blocks ( self . _iter_blocks ( ) , self . _stats , prefetch_blocks = prefetch_blocks , batch_size = batch_size , batch_format = batch_format , drop_last = drop_last , ) self . _stats . iter_total_s . add ( time . perf_counter ( ) - time_start )
Return a local batched iterator over the data in the pipeline.
Return a local batched iterator over the data in the pipeline .
"""Return a local batched iterator over the data in the pipeline.

        Examples:
            >>> import ray
            >>> ds = ray.data.range(1000000).repeat(5) # doctest: +SKIP
            >>> for pandas_df in ds.iter_batches(): # doctest: +SKIP
            ...     print(pandas_df) # doctest: +SKIP

        Time complexity: O(1)

        Args:
            prefetch_blocks: The number of blocks to prefetch ahead of the
                current block during the scan.
            batch_size: Record batch size, or None to let the system pick.
            batch_format: The format in which to return each batch.
                Specify "native" to use the current block format (promoting
                Arrow to pandas automatically), "pandas" to
                select ``pandas.DataFrame`` or "pyarrow" to select
                ``pyarrow.Table``. Default is "native".
            drop_last: Whether to drop the last batch if it's incomplete.

        Returns:
            An iterator over record batches.
        """
An iterator over record batches.
An iterator over record batches .
self
prefetch_blocks
int
The number of blocks to prefetch ahead of the
current block during the scan.
The number of blocks to prefetch ahead of the current block during the scan .
batch_size
int
Record batch size, or None to let the system pick.
Record batch size or None to let the system pick .
batch_format
str
The format in which to return each batch.
Specify "native" to use the current block format (promoting
Arrow to pandas automatically), "pandas" to
select ``pandas.DataFrame`` or "pyarrow" to select
``pyarrow.Table``. Default is "native".
The format in which to return each batch . Specify " native " to use the current block format ( promoting Arrow to pandas automatically ) " pandas " to select ` ` pandas . DataFrame ` ` or " pyarrow " to select ` ` pyarrow . Table ` ` . Default is " native " .
drop_last
bool
Whether to drop the last batch if it's incomplete.
Whether to drop the last batch if it ' s incomplete .
examples
>>> import ray
>>> ds = ray.data.range(1000000).repeat(5) # doctest: +SKIP
>>> for pandas_df in ds.iter_batches(): # doctest: +SKIP
print(pandas_df) # doctest: +SKIP
>>> import ray >>> ds = ray . data . range ( 1000000 ) . repeat ( 5 ) # doctest : + SKIP >>> for pandas_df in ds . iter_batches () : # doctest : + SKIP print ( pandas_df ) # doctest : + SKIP