




import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


_lowerCamelCase					:     int					  =  datasets.utils.logging.get_logger(__name__)
@dataclass
class 				__snake_case     (datasets.BuilderConfig						):
				lowerCAmelCase__			=   None
				lowerCAmelCase__			=   "utf-8"
				lowerCAmelCase__			=   None
				lowerCAmelCase__			=   1_0 << 2_0  # 10MB
				lowerCAmelCase__			=   False
				lowerCAmelCase__			=   "line"






class 				__snake_case     (datasets.ArrowBasedBuilder						):
				lowerCAmelCase__			=   TextConfig



				def 					SCREAMING_SNAKE_CASE				(					self					:						Dict					)     ->			int:





								'''simple docstring'''




								return datasets.DatasetInfo(features=self.config.features					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						List[Any]						,		_UpperCAmelCase					:						List[str]					)     ->			str:





								'''simple docstring'''




								if not self.config.data_files:
												raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}"					)
								_lowerCAmelCase			:			Union[str, Any]        =					dl_manager.download_and_extract(self.config.data_files					)
								if isinstance(_UpperCAmelCase						,		(str, list, tuple)					):
												_lowerCAmelCase			:			Dict        =					data_files
												if isinstance(_UpperCAmelCase						,		_UpperCAmelCase					):
																_lowerCAmelCase			:			int        =					[files]
												_lowerCAmelCase			:			Tuple        =					[dl_manager.iter_files(_UpperCAmelCase					) for file in files]
												return [datasets.SplitGenerator(name=datasets.Split.TRAIN						,		gen_kwargs={"""files""": files}					)]
								_lowerCAmelCase			:			Dict        =					[]
								for split_name, files in data_files.items():
												if isinstance(_UpperCAmelCase						,		_UpperCAmelCase					):
																_lowerCAmelCase			:			Dict        =					[files]
												_lowerCAmelCase			:			Tuple        =					[dl_manager.iter_files(_UpperCAmelCase					) for file in files]
												splits.append(datasets.SplitGenerator(name=_UpperCAmelCase						,		gen_kwargs={"""files""": files}					)					)
								return splits



				def 					SCREAMING_SNAKE_CASE				(					self					:						List[str]						,		_UpperCAmelCase					:						pa.Table					)     ->			pa.Table:





								'''simple docstring'''




								if self.config.features is not None:
												_lowerCAmelCase			:			Optional[int]        =					self.config.features.arrow_schema
												if all(not require_storage_cast(_UpperCAmelCase					) for feature in self.config.features.values()					):
																# cheaper cast
																_lowerCAmelCase			:			Optional[int]        =					pa_table.cast(_UpperCAmelCase					)
												else:
																# more expensive cast; allows str <-> int/float or str to Audio for example
																_lowerCAmelCase			:			List[str]        =					table_cast(_UpperCAmelCase						,		_UpperCAmelCase					)
												return pa_table
								else:
												return pa_table.cast(pa.schema({"""text""": pa.string()}					)					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						str						,		_UpperCAmelCase					:						Optional[Any]					)     ->			Tuple:





								'''simple docstring'''




								_lowerCAmelCase			:			List[str]        =					list(self.config.features					) if self.config.features is not None else ["""text"""]
								for file_idx, file in enumerate(itertools.chain.from_iterable(_UpperCAmelCase					)					):
												# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
												with open(_UpperCAmelCase						,		encoding=self.config.encoding						,		errors=self.config.errors					) as f:
																if self.config.sample_by == "line":
																				_lowerCAmelCase			:			Optional[int]        =					0
																				while True:
																								_lowerCAmelCase			:			int        =					f.read(self.config.chunksize					)
																								if not batch:
																												break
																								batch += f.readline()  # finish current line
																								# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																								_lowerCAmelCase			:			str        =					StringIO(_UpperCAmelCase					).readlines()
																								if not self.config.keep_linebreaks:
																												_lowerCAmelCase			:			Dict        =					[line.rstrip("""\n"""					) for line in batch]
																								_lowerCAmelCase			:			List[Any]        =					pa.Table.from_arrays([pa.array(_UpperCAmelCase					)]						,		names=_UpperCAmelCase					)
																								# Uncomment for debugging (will print the Arrow table size and elements)
																								# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																								# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																								yield (file_idx, batch_idx), self._cast_table(_UpperCAmelCase					)
																								batch_idx += 1
																elif self.config.sample_by == "paragraph":
																				_lowerCAmelCase			:			Dict        =					0
																				_lowerCAmelCase			:			Dict        =					""""""
																				while True:
																								_lowerCAmelCase			:			str        =					f.read(self.config.chunksize					)
																								if not new_batch:
																												break
																								batch += new_batch
																								batch += f.readline()  # finish current line
																								_lowerCAmelCase			:			Optional[int]        =					batch.split("""\n\n"""					)
																								_lowerCAmelCase			:			Union[str, Any]        =					pa.Table.from_arrays(
																								    [pa.array([example for example in batch[:-1] if example]					)]						,		names=_UpperCAmelCase					)
																								# Uncomment for debugging (will print the Arrow table size and elements)
																								# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																								# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																								yield (file_idx, batch_idx), self._cast_table(_UpperCAmelCase					)
																								batch_idx += 1
																								_lowerCAmelCase			:			Tuple        =					batch[-1]
																				if batch:
																								_lowerCAmelCase			:			str        =					pa.Table.from_arrays([pa.array([batch]					)]						,		names=_UpperCAmelCase					)
																								yield (file_idx, batch_idx), self._cast_table(_UpperCAmelCase					)
																elif self.config.sample_by == "document":
																				_lowerCAmelCase			:			Any        =					f.read()
																				_lowerCAmelCase			:			Optional[int]        =					pa.Table.from_arrays([pa.array([text]					)]						,		names=_UpperCAmelCase					)
																				yield file_idx, self._cast_table(_UpperCAmelCase					)








"""simple docstring"""


from typing import TYPE_CHECKING

from ..utils import OptionalDependencyNotAvailable, _LazyModule, is_flax_available, is_tf_available, is_torch_available


lowerCamelCase__      =      {
    "configuration_utils": ["GenerationConfig"],
    "streamers": ["TextIteratorStreamer", "TextStreamer"],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowerCamelCase__      =      [
      "Constraint",
      "ConstraintListState",
      "DisjunctiveConstraint",
      "PhrasalConstraint",
  ]
  lowerCamelCase__      =      [
      "BeamHypotheses",
      "BeamScorer",
      "BeamSearchScorer",
      "ConstrainedBeamSearchScorer",
  ]
  lowerCamelCase__      =      [
      "EpsilonLogitsWarper",
      "EtaLogitsWarper",
      "ForcedBOSTokenLogitsProcessor",
      "ForcedEOSTokenLogitsProcessor",
      "HammingDiversityLogitsProcessor",
      "InfNanRemoveLogitsProcessor",
      "LogitsProcessor",
      "LogitsProcessorList",
      "LogitsWarper",
      "MinLengthLogitsProcessor",
      "MinNewTokensLengthLogitsProcessor",
      "NoBadWordsLogitsProcessor",
      "NoRepeatNGramLogitsProcessor",
      "PrefixConstrainedLogitsProcessor",
      "RepetitionPenaltyLogitsProcessor",
      "SequenceBiasLogitsProcessor",
      "EncoderRepetitionPenaltyLogitsProcessor",
      "TemperatureLogitsWarper",
      "TopKLogitsWarper",
      "TopPLogitsWarper",
      "TypicalLogitsWarper",
      "EncoderNoRepeatNGramLogitsProcessor",
      "ExponentialDecayLengthPenalty",
      "LogitNormalization",
  ]
  lowerCamelCase__      =      [
      "MaxNewTokensCriteria",
      "MaxLengthCriteria",
      "MaxTimeCriteria",
      "StoppingCriteria",
      "StoppingCriteriaList",
      "validate_stopping_criteria",
  ]
  lowerCamelCase__      =      [
      "GenerationMixin",
      "top_k_top_p_filtering",
      "GreedySearchEncoderDecoderOutput",
      "GreedySearchDecoderOnlyOutput",
      "SampleEncoderDecoderOutput",
      "SampleDecoderOnlyOutput",
      "BeamSearchEncoderDecoderOutput",
      "BeamSearchDecoderOnlyOutput",
      "BeamSampleEncoderDecoderOutput",
      "BeamSampleDecoderOnlyOutput",
      "ContrastiveSearchEncoderDecoderOutput",
      "ContrastiveSearchDecoderOnlyOutput",
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowerCamelCase__      =      [
      "TFForcedBOSTokenLogitsProcessor",
      "TFForcedEOSTokenLogitsProcessor",
      "TFLogitsProcessor",
      "TFLogitsProcessorList",
      "TFLogitsWarper",
      "TFMinLengthLogitsProcessor",
      "TFNoBadWordsLogitsProcessor",
      "TFNoRepeatNGramLogitsProcessor",
      "TFRepetitionPenaltyLogitsProcessor",
      "TFTemperatureLogitsWarper",
      "TFTopKLogitsWarper",
      "TFTopPLogitsWarper",
      "TFForceTokensLogitsProcessor",
      "TFSuppressTokensAtBeginLogitsProcessor",
      "TFSuppressTokensLogitsProcessor",
  ]
  lowerCamelCase__      =      [
      "TFGenerationMixin",
      "tf_top_k_top_p_filtering",
      "TFGreedySearchDecoderOnlyOutput",
      "TFGreedySearchEncoderDecoderOutput",
      "TFSampleEncoderDecoderOutput",
      "TFSampleDecoderOnlyOutput",
      "TFBeamSearchEncoderDecoderOutput",
      "TFBeamSearchDecoderOnlyOutput",
      "TFBeamSampleEncoderDecoderOutput",
      "TFBeamSampleDecoderOnlyOutput",
      "TFContrastiveSearchEncoderDecoderOutput",
      "TFContrastiveSearchDecoderOnlyOutput",
  ]

try:
  if not is_flax_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowerCamelCase__      =      [
      "FlaxForcedBOSTokenLogitsProcessor",
      "FlaxForcedEOSTokenLogitsProcessor",
      "FlaxLogitsProcessor",
      "FlaxLogitsProcessorList",
      "FlaxLogitsWarper",
      "FlaxMinLengthLogitsProcessor",
      "FlaxTemperatureLogitsWarper",
      "FlaxTopKLogitsWarper",
      "FlaxTopPLogitsWarper",
  ]
  lowerCamelCase__      =      [
      "FlaxGenerationMixin",
      "FlaxGreedySearchOutput",
      "FlaxSampleOutput",
      "FlaxBeamSearchOutput",
  ]

if TYPE_CHECKING:
  from .configuration_utils import GenerationConfig
  from .streamers import TextIteratorStreamer, TextStreamer

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .beam_constraints import Constraint, ConstraintListState, DisjunctiveConstraint, PhrasalConstraint
    from .beam_search import BeamHypotheses, BeamScorer, BeamSearchScorer, ConstrainedBeamSearchScorer
    from .logits_process import (
        EncoderNoRepeatNGramLogitsProcessor,
        EncoderRepetitionPenaltyLogitsProcessor,
        EpsilonLogitsWarper,
        EtaLogitsWarper,
        ExponentialDecayLengthPenalty,
        ForcedBOSTokenLogitsProcessor,
        ForcedEOSTokenLogitsProcessor,
        HammingDiversityLogitsProcessor,
        InfNanRemoveLogitsProcessor,
        LogitNormalization,
        LogitsProcessor,
        LogitsProcessorList,
        LogitsWarper,
        MinLengthLogitsProcessor,
        MinNewTokensLengthLogitsProcessor,
        NoBadWordsLogitsProcessor,
        NoRepeatNGramLogitsProcessor,
        PrefixConstrainedLogitsProcessor,
        RepetitionPenaltyLogitsProcessor,
        SequenceBiasLogitsProcessor,
        TemperatureLogitsWarper,
        TopKLogitsWarper,
        TopPLogitsWarper,
        TypicalLogitsWarper,
    )
    from .stopping_criteria import (
        MaxLengthCriteria,
        MaxNewTokensCriteria,
        MaxTimeCriteria,
        StoppingCriteria,
        StoppingCriteriaList,
        validate_stopping_criteria,
    )
    from .utils import (
        BeamSampleDecoderOnlyOutput,
        BeamSampleEncoderDecoderOutput,
        BeamSearchDecoderOnlyOutput,
        BeamSearchEncoderDecoderOutput,
        ContrastiveSearchDecoderOnlyOutput,
        ContrastiveSearchEncoderDecoderOutput,
        GenerationMixin,
        GreedySearchDecoderOnlyOutput,
        GreedySearchEncoderDecoderOutput,
        SampleDecoderOnlyOutput,
        SampleEncoderDecoderOutput,
        top_k_top_p_filtering,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tf_logits_process import (
        TFForcedBOSTokenLogitsProcessor,
        TFForcedEOSTokenLogitsProcessor,
        TFForceTokensLogitsProcessor,
        TFLogitsProcessor,
        TFLogitsProcessorList,
        TFLogitsWarper,
        TFMinLengthLogitsProcessor,
        TFNoBadWordsLogitsProcessor,
        TFNoRepeatNGramLogitsProcessor,
        TFRepetitionPenaltyLogitsProcessor,
        TFSuppressTokensAtBeginLogitsProcessor,
        TFSuppressTokensLogitsProcessor,
        TFTemperatureLogitsWarper,
        TFTopKLogitsWarper,
        TFTopPLogitsWarper,
    )
    from .tf_utils import (
        TFBeamSampleDecoderOnlyOutput,
        TFBeamSampleEncoderDecoderOutput,
        TFBeamSearchDecoderOnlyOutput,
        TFBeamSearchEncoderDecoderOutput,
        TFContrastiveSearchDecoderOnlyOutput,
        TFContrastiveSearchEncoderDecoderOutput,
        TFGenerationMixin,
        TFGreedySearchDecoderOnlyOutput,
        TFGreedySearchEncoderDecoderOutput,
        TFSampleDecoderOnlyOutput,
        TFSampleEncoderDecoderOutput,
        tf_top_k_top_p_filtering,
    )

  try:
    if not is_flax_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .flax_logits_process import (
        FlaxForcedBOSTokenLogitsProcessor,
        FlaxForcedEOSTokenLogitsProcessor,
        FlaxLogitsProcessor,
        FlaxLogitsProcessorList,
        FlaxLogitsWarper,
        FlaxMinLengthLogitsProcessor,
        FlaxTemperatureLogitsWarper,
        FlaxTopKLogitsWarper,
        FlaxTopPLogitsWarper,
    )
    from .flax_utils import FlaxBeamSearchOutput, FlaxGenerationMixin, FlaxGreedySearchOutput, FlaxSampleOutput
else:
  import sys

  lowerCamelCase__      =      _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
