python/ray/data/read_api.py
def read_csv(
    paths: Union[str, List[str]],
    *,
    filesystem: Optional["pyarrow.fs.FileSystem"] = None,
    parallelism: int = 200,
    ray_remote_args: Dict[str, Any] = None,
    arrow_open_stream_args: Optional[Dict[str, Any]] = None,
    meta_provider: BaseFileMetadataProvider = DefaultFileMetadataProvider(),
    partition_filter: PathPartitionFilter = None,
    **arrow_csv_args,
) -> Dataset[ArrowRow]:
    """Create an Arrow dataset from csv files.

    Examples:
        >>> import ray
        >>> # Read a directory of files in remote storage.
        >>> ray.data.read_csv("s3://bucket/path") # doctest: +SKIP

        >>> # Read multiple local files.
        >>> ray.data.read_csv(["/path/to/file1", "/path/to/file2"]) # doctest: +SKIP

        >>> # Read multiple directories.
        >>> ray.data.read_csv( # doctest: +SKIP
        ...     ["s3://bucket/path1", "s3://bucket/path2"])

    Args:
        paths: A single file/directory path or a list of file/directory paths.
            A list of paths can contain both files and directories.
        filesystem: The filesystem implementation to read from.
        parallelism: The requested parallelism of the read. Parallelism may be
            limited by the number of files of the dataset.
        ray_remote_args: kwargs passed to ray.remote in the read tasks.
        arrow_open_stream_args: kwargs passed to
            pyarrow.fs.FileSystem.open_input_stream
        meta_provider: File metadata provider. Custom metadata providers may
            be able to resolve file metadata more quickly and/or accurately.
        partition_filter: Path-based partition filter, if any. Can be used
            with a custom callback to read only selected partitions of a dataset.
        arrow_csv_args: Other csv read options to pass to pyarrow.

    Returns:
        Dataset holding Arrow records read from the specified paths.
    """
    return read_datasource(
        CSVDatasource(),
        parallelism=parallelism,
        paths=paths,
        filesystem=filesystem,
        ray_remote_args=ray_remote_args,
        open_stream_args=arrow_open_stream_args,
        meta_provider=meta_provider,
        partition_filter=partition_filter,
        **arrow_csv_args,
    )
Create an Arrow dataset from csv files.

    Examples:
        >>> import ray
        >>> # Read a directory of files in remote storage.
        >>> ray.data.read_csv("s3://bucket/path") # doctest: +SKIP

        >>> # Read multiple local files.
        >>> ray.data.read_csv(["/path/to/file1", "/path/to/file2"]) # doctest: +SKIP

        >>> # Read multiple directories.
        >>> ray.data.read_csv( # doctest: +SKIP
        ...     ["s3://bucket/path1", "s3://bucket/path2"])

    Args:
        paths: A single file/directory path or a list of file/directory paths.
            A list of paths can contain both files and directories.
        filesystem: The filesystem implementation to read from.
        parallelism: The requested parallelism of the read. Parallelism may be
            limited by the number of files of the dataset.
        ray_remote_args: kwargs passed to ray.remote in the read tasks.
        arrow_open_stream_args: kwargs passed to
            pyarrow.fs.FileSystem.open_input_stream
        meta_provider: File metadata provider. Custom metadata providers may
            be able to resolve file metadata more quickly and/or accurately.
        partition_filter: Path-based partition filter, if any. Can be used
            with a custom callback to read only selected partitions of a dataset.
        arrow_csv_args: Other csv read options to pass to pyarrow.

    Returns:
        Dataset holding Arrow records read from the specified paths.
    
Create an Arrow dataset from csv files.
Create an Arrow dataset from csv files .
def read_csv(
    paths: Union[str, List[str]],
    *,
    filesystem: Optional["pyarrow.fs.FileSystem"] = None,
    parallelism: int = 200,
    ray_remote_args: Dict[str, Any] = None,
    arrow_open_stream_args: Optional[Dict[str, Any]] = None,
    meta_provider: BaseFileMetadataProvider = DefaultFileMetadataProvider(),
    partition_filter: PathPartitionFilter = None,
    **arrow_csv_args,
) -> Dataset[ArrowRow]:
    return read_datasource(
        CSVDatasource(),
        parallelism=parallelism,
        paths=paths,
        filesystem=filesystem,
        ray_remote_args=ray_remote_args,
        open_stream_args=arrow_open_stream_args,
        meta_provider=meta_provider,
        partition_filter=partition_filter,
        **arrow_csv_args,
    )
def read_csv ( paths : Union [ str , List [ str ] ] , * , filesystem : Optional [ "pyarrow.fs.FileSystem" ] = None , parallelism : int = 200 , ray_remote_args : Dict [ str , Any ] = None , arrow_open_stream_args : Optional [ Dict [ str , Any ] ] = None , meta_provider : BaseFileMetadataProvider = DefaultFileMetadataProvider ( ) , partition_filter : PathPartitionFilter = None , ** arrow_csv_args , ) -> Dataset [ ArrowRow ] : return read_datasource ( CSVDatasource ( ) , parallelism = parallelism , paths = paths , filesystem = filesystem , ray_remote_args = ray_remote_args , open_stream_args = arrow_open_stream_args , meta_provider = meta_provider , partition_filter = partition_filter , ** arrow_csv_args , )
Create an Arrow dataset from csv files.
Create an Arrow dataset from csv files .
"""Create an Arrow dataset from csv files.

    Examples:
        >>> import ray
        >>> # Read a directory of files in remote storage.
        >>> ray.data.read_csv("s3://bucket/path") # doctest: +SKIP

        >>> # Read multiple local files.
        >>> ray.data.read_csv(["/path/to/file1", "/path/to/file2"]) # doctest: +SKIP

        >>> # Read multiple directories.
        >>> ray.data.read_csv( # doctest: +SKIP
        ...     ["s3://bucket/path1", "s3://bucket/path2"])

    Args:
        paths: A single file/directory path or a list of file/directory paths.
            A list of paths can contain both files and directories.
        filesystem: The filesystem implementation to read from.
        parallelism: The requested parallelism of the read. Parallelism may be
            limited by the number of files of the dataset.
        ray_remote_args: kwargs passed to ray.remote in the read tasks.
        arrow_open_stream_args: kwargs passed to
            pyarrow.fs.FileSystem.open_input_stream
        meta_provider: File metadata provider. Custom metadata providers may
            be able to resolve file metadata more quickly and/or accurately.
        partition_filter: Path-based partition filter, if any. Can be used
            with a custom callback to read only selected partitions of a dataset.
        arrow_csv_args: Other csv read options to pass to pyarrow.

    Returns:
        Dataset holding Arrow records read from the specified paths.
    """
Dataset holding Arrow records read from the specified paths.
Dataset holding Arrow records read from the specified paths .
paths
Union[str, List[str]]
A single file/directory path or a list of file/directory paths.
A list of paths can contain both files and directories.
A single file / directory path or a list of file / directory paths . A list of paths can contain both files and directories .
filesystem
Optional["pyarrow.fs.FileSystem"]
The filesystem implementation to read from.
The filesystem implementation to read from .
parallelism
int
The requested parallelism of the read. Parallelism may be
limited by the number of files of the dataset.
The requested parallelism of the read . Parallelism may be limited by the number of files of the dataset .
ray_remote_args
Dict[str, Any]
kwargs passed to ray.remote in the read tasks.
kwargs passed to ray . remote in the read tasks .
arrow_open_stream_args
Optional[Dict[str, Any]]
kwargs passed to
pyarrow.fs.FileSystem.open_input_stream
kwargs passed to pyarrow . fs . FileSystem . open_input_stream
meta_provider
BaseFileMetadataProvider
File metadata provider. Custom metadata providers may
be able to resolve file metadata more quickly and/or accurately.
File metadata provider . Custom metadata providers may be able to resolve file metadata more quickly and / or accurately .
partition_filter
PathPartitionFilter
Path-based partition filter, if any. Can be used
with a custom callback to read only selected partitions of a dataset.
Path - based partition filter if any . Can be used with a custom callback to read only selected partitions of a dataset .
arrow_csv_args
Other csv read options to pass to pyarrow.
Other csv read options to pass to pyarrow .
examples
>>> import ray
>>> # Read a directory of files in remote storage.
>>> ray.data.read_csv("s3://bucket/path") # doctest: +SKIP

>>> # Read multiple local files.

>>> # Read multiple directories.
>>> import ray >>> # Read a directory of files in remote storage . >>> ray . data . read_csv ( " s3 : // bucket / path " ) # doctest : + SKIP >>> # Read multiple local files . >>> # Read multiple directories .