python/ray/data/read_api.py
def read_text(
    paths: Union[str, List[str]],
    *,
    encoding: str = "utf-8",
    errors: str = "ignore",
    drop_empty_lines: bool = True,
    filesystem: Optional["pyarrow.fs.FileSystem"] = None,
    parallelism: int = 200,
    arrow_open_stream_args: Optional[Dict[str, Any]] = None,
    meta_provider: BaseFileMetadataProvider = DefaultFileMetadataProvider(),
    partition_filter: PathPartitionFilter = None,
) -> Dataset[str]:
    """Create a dataset from lines stored in text files.

    Examples:
        >>> import ray
        >>> # Read a directory of files in remote storage.
        >>> ray.data.read_text("s3://bucket/path") # doctest: +SKIP

        >>> # Read multiple local files.
        >>> ray.data.read_text(["/path/to/file1", "/path/to/file2"]) # doctest: +SKIP

    Args:
        paths: A single file path or a list of file paths (or directories).
        encoding: The encoding of the files (e.g., "utf-8" or "ascii").
        errors: What to do with errors on decoding. Specify either "strict",
            "ignore", or "replace". Defaults to "ignore".
        filesystem: The filesystem implementation to read from.
        parallelism: The requested parallelism of the read. Parallelism may be
            limited by the number of files of the dataset.
        arrow_open_stream_args: kwargs passed to
            pyarrow.fs.FileSystem.open_input_stream
        meta_provider: File metadata provider. Custom metadata providers may
            be able to resolve file metadata more quickly and/or accurately.
        partition_filter: Path-based partition filter, if any. Can be used
            with a custom callback to read only selected partitions of a dataset.

    Returns:
        Dataset holding lines of text read from the specified paths.
    """

    def to_text(s):
        lines = s.decode(encoding).split("\n")
        if drop_empty_lines:
            lines = [line for line in lines if line.strip() != ""]
        return lines

    return read_binary_files(
        paths,
        filesystem=filesystem,
        parallelism=parallelism,
        arrow_open_stream_args=arrow_open_stream_args,
        meta_provider=meta_provider,
        partition_filter=partition_filter,
    ).flat_map(to_text)
Create a dataset from lines stored in text files.

    Examples:
        >>> import ray
        >>> # Read a directory of files in remote storage.
        >>> ray.data.read_text("s3://bucket/path") # doctest: +SKIP

        >>> # Read multiple local files.
        >>> ray.data.read_text(["/path/to/file1", "/path/to/file2"]) # doctest: +SKIP

    Args:
        paths: A single file path or a list of file paths (or directories).
        encoding: The encoding of the files (e.g., "utf-8" or "ascii").
        errors: What to do with errors on decoding. Specify either "strict",
            "ignore", or "replace". Defaults to "ignore".
        filesystem: The filesystem implementation to read from.
        parallelism: The requested parallelism of the read. Parallelism may be
            limited by the number of files of the dataset.
        arrow_open_stream_args: kwargs passed to
            pyarrow.fs.FileSystem.open_input_stream
        meta_provider: File metadata provider. Custom metadata providers may
            be able to resolve file metadata more quickly and/or accurately.
        partition_filter: Path-based partition filter, if any. Can be used
            with a custom callback to read only selected partitions of a dataset.

    Returns:
        Dataset holding lines of text read from the specified paths.
    
Create a dataset from lines stored in text files.
Create a dataset from lines stored in text files .
def read_text(
    paths: Union[str, List[str]],
    *,
    encoding: str = "utf-8",
    errors: str = "ignore",
    drop_empty_lines: bool = True,
    filesystem: Optional["pyarrow.fs.FileSystem"] = None,
    parallelism: int = 200,
    arrow_open_stream_args: Optional[Dict[str, Any]] = None,
    meta_provider: BaseFileMetadataProvider = DefaultFileMetadataProvider(),
    partition_filter: PathPartitionFilter = None,
) -> Dataset[str]:
    def to_text(s):
        lines = s.decode(encoding).split("\n")
        if drop_empty_lines:
            lines = [line for line in lines if line.strip() != ""]
        return lines
    return read_binary_files(
        paths,
        filesystem=filesystem,
        parallelism=parallelism,
        arrow_open_stream_args=arrow_open_stream_args,
        meta_provider=meta_provider,
        partition_filter=partition_filter,
    ).flat_map(to_text)
def read_text ( paths : Union [ str , List [ str ] ] , * , encoding : str = "utf-8" , errors : str = "ignore" , drop_empty_lines : bool = True , filesystem : Optional [ "pyarrow.fs.FileSystem" ] = None , parallelism : int = 200 , arrow_open_stream_args : Optional [ Dict [ str , Any ] ] = None , meta_provider : BaseFileMetadataProvider = DefaultFileMetadataProvider ( ) , partition_filter : PathPartitionFilter = None , ) -> Dataset [ str ] : def to_text ( s ) : lines = s . decode ( encoding ) . split ( "\n" ) if drop_empty_lines : lines = [ line for line in lines if line . strip ( ) != "" ] return lines return read_binary_files ( paths , filesystem = filesystem , parallelism = parallelism , arrow_open_stream_args = arrow_open_stream_args , meta_provider = meta_provider , partition_filter = partition_filter , ) . flat_map ( to_text )
Create a dataset from lines stored in text files.
Create a dataset from lines stored in text files .
"""Create a dataset from lines stored in text files.

    Examples:
        >>> import ray
        >>> # Read a directory of files in remote storage.
        >>> ray.data.read_text("s3://bucket/path") # doctest: +SKIP

        >>> # Read multiple local files.
        >>> ray.data.read_text(["/path/to/file1", "/path/to/file2"]) # doctest: +SKIP

    Args:
        paths: A single file path or a list of file paths (or directories).
        encoding: The encoding of the files (e.g., "utf-8" or "ascii").
        errors: What to do with errors on decoding. Specify either "strict",
            "ignore", or "replace". Defaults to "ignore".
        filesystem: The filesystem implementation to read from.
        parallelism: The requested parallelism of the read. Parallelism may be
            limited by the number of files of the dataset.
        arrow_open_stream_args: kwargs passed to
            pyarrow.fs.FileSystem.open_input_stream
        meta_provider: File metadata provider. Custom metadata providers may
            be able to resolve file metadata more quickly and/or accurately.
        partition_filter: Path-based partition filter, if any. Can be used
            with a custom callback to read only selected partitions of a dataset.

    Returns:
        Dataset holding lines of text read from the specified paths.
    """
Dataset holding lines of text read from the specified paths.
Dataset holding lines of text read from the specified paths .
paths
Union[str, List[str]]
A single file path or a list of file paths (or directories).
A single file path or a list of file paths ( or directories ) .
encoding
str
The encoding of the files .
The encoding of the files .
errors
str
None
drop_empty_lines
bool
filesystem
Optional["pyarrow.fs.FileSystem"]
The filesystem implementation to read from.
The filesystem implementation to read from .
parallelism
int
The requested parallelism of the read. Parallelism may be
limited by the number of files of the dataset.
The requested parallelism of the read . Parallelism may be limited by the number of files of the dataset .
arrow_open_stream_args
Optional[Dict[str, Any]]
kwargs passed to
pyarrow.fs.FileSystem.open_input_stream
kwargs passed to pyarrow . fs . FileSystem . open_input_stream
meta_provider
BaseFileMetadataProvider
File metadata provider. Custom metadata providers may
be able to resolve file metadata more quickly and/or accurately.
File metadata provider . Custom metadata providers may be able to resolve file metadata more quickly and / or accurately .
partition_filter
PathPartitionFilter
Path-based partition filter, if any. Can be used
with a custom callback to read only selected partitions of a dataset.
Path - based partition filter if any . Can be used with a custom callback to read only selected partitions of a dataset .
examples
>>> import ray
>>> # Read a directory of files in remote storage.
>>> ray.data.read_text("s3://bucket/path") # doctest: +SKIP

>>> # Read multiple local files.
>>> import ray >>> # Read a directory of files in remote storage . >>> ray . data . read_text ( " s3 : // bucket / path " ) # doctest : + SKIP >>> # Read multiple local files .