# coding=utf-8
# Copyright 2020 The HuggingFace Datasets Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datalabs

logger = datalabs.utils.logging.get_logger(__name__)


@dataclass
class TextConfig(datalabs.BuilderConfig):
    """BuilderConfig for text files."""

    features: Optional[datalabs.Features] = None
    encoding: str = "utf-8"
    chunksize: int = 10 << 20  # 10MB
    keep_linebreaks: bool = False


class Text(datalabs.ArrowBasedBuilder):
    BUILDER_CONFIG_CLASS = TextConfig

    def _info(self):
        return datalabs.DatasetInfo(features=self.config.features)

    def _split_generators(self, dl_manager):
        """The `data_files` kwarg in load_dataset() can
         be a str, List[str], Dict[str,str], or Dict[str,List[str]].

        If str or List[str], then the dataset returns only the 'train' split.
        If dict, then keys should be from the `datalab.Split` enum.
        """
        if not self.config.data_files:
            raise ValueError(
                f"At least one data file must be specified,"
                f" but got data_files={self.config.data_files}"
            )
        data_files = dl_manager.download_and_extract(self.config.data_files)
        if isinstance(data_files, (str, list, tuple)):
            files = data_files
            if isinstance(files, str):
                files = [files]
            return [
                datalabs.SplitGenerator(
                    name=datalabs.Split.TRAIN, gen_kwargs={"files": files}
                )
            ]
        splits = []
        for split_name, files in data_files.items():
            if isinstance(files, str):
                files = [files]
            splits.append(
                datalabs.SplitGenerator(name=split_name, gen_kwargs={"files": files})
            )
        return splits

    def _generate_tables(self, files):
        schema = pa.schema(
            self.config.features.type
            if self.config.features is not None
            else {"text": pa.string()}
        )
        for file_idx, file in enumerate(files):
            batch_idx = 0
            with open(file, "r", encoding=self.config.encoding) as f:
                while True:
                    batch = f.read(self.config.chunksize)
                    if not batch:
                        break
                    batch += f.readline()  # finish current line
                    batch = batch.splitlines(keepends=self.config.keep_linebreaks)
                    pa_table = pa.Table.from_arrays([pa.array(batch)], schema=schema)
                    # Uncomment for debugging (will print the
                    # Arrow table size and elements)
                    # logger.warning(f"pa_table: {pa_table}
                    # num rows: {pa_table.num_rows}")
                    # logger.warning('\n'.join(str(pa_table.slice(i,
                    # 1).to_pydict()) for i in range(pa_table.num_rows)))
                    yield (file_idx, batch_idx), pa_table
                    batch_idx += 1
