
import os
from typing import BinaryIO, Optional, Union

import numpy as np
import pyarrow.parquet as pq

from .. import Audio, Dataset, Features, Image, NamedSplit, Value, config
from ..features.features import FeatureType, _visit
from ..formatting import query_table
from ..packaged_modules import _PACKAGED_DATASETS_MODULES
from ..packaged_modules.parquet.parquet import Parquet
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader






def      lowerCamelCase__       (       __lowerCAmelCase						:						Features       ):




			"""simple docstring"""


			lowerCAmelCase_      			=			np.inf

			def set_batch_size(__lowerCAmelCase						:						FeatureType       ) -> None:
						nonlocal batch_size
						if isinstance(__lowerCAmelCase					,							__lowerCAmelCase       ):
									lowerCAmelCase_      			=			min(__lowerCAmelCase					,							config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS       )
						elif isinstance(__lowerCAmelCase					,							__lowerCAmelCase       ):
									lowerCAmelCase_      			=			min(__lowerCAmelCase					,							config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS       )
						elif isinstance(__lowerCAmelCase					,							__lowerCAmelCase       ) and feature.dtype == "binary":
									lowerCAmelCase_      			=			min(__lowerCAmelCase					,							config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS       )

			_visit(__lowerCAmelCase					,							__lowerCAmelCase       )

			return None if batch_size is np.inf else batch_size





class 			_lowerCAmelCase      (			__a ):




				def __init__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase = None       ,       _UpperCamelCase = None       ,       _UpperCamelCase = None       ,       _UpperCamelCase = False       ,       _UpperCamelCase = False       ,       _UpperCamelCase = None       ,       **_UpperCamelCase       ,       )       ->   str:
							super().__init__(
							    _UpperCamelCase       ,       split=_UpperCamelCase       ,       features=_UpperCamelCase       ,       cache_dir=_UpperCamelCase       ,       keep_in_memory=_UpperCamelCase       ,       streaming=_UpperCamelCase       ,       num_proc=_UpperCamelCase       ,       **_UpperCamelCase       ,       )
							lowerCAmelCase_      			=			path_or_paths if isinstance(_UpperCamelCase       ,       _UpperCamelCase  ) else {self.split: path_or_paths}
							lowerCAmelCase_      			=			_PACKAGED_DATASETS_MODULES["parquet"][1]
							lowerCAmelCase_      			=			Parquet(
							    cache_dir=_UpperCamelCase       ,       data_files=_UpperCamelCase       ,       features=_UpperCamelCase       ,       hash=_UpperCamelCase       ,       **_UpperCamelCase       ,       )




				def   __a		(      self  )       ->   List[Any]:
							# Build iterable dataset
							if self.streaming:
										lowerCAmelCase_      			=			self.builder.as_streaming_dataset(split=self.split  )
							# Build regular (map-style) dataset
							else:
										lowerCAmelCase_      			=			None
										lowerCAmelCase_      			=			None
										lowerCAmelCase_      			=			None
										lowerCAmelCase_      			=			None

										self.builder.download_and_prepare(
										    download_config=_UpperCamelCase       ,       download_mode=_UpperCamelCase       ,       verification_mode=_UpperCamelCase       ,       base_path=_UpperCamelCase       ,       num_proc=self.num_proc       ,       )
										lowerCAmelCase_      			=			self.builder.as_dataset(
										    split=self.split       ,       verification_mode=_UpperCamelCase       ,       in_memory=self.keep_in_memory  )
							return dataset





class 			_lowerCAmelCase      :




				def __init__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase = None       ,       **_UpperCamelCase       ,       )       ->   Dict:
							lowerCAmelCase_      			=			dataset
							lowerCAmelCase_      			=			path_or_buf
							lowerCAmelCase_      			=			batch_size or get_writer_batch_size(dataset.features  )
							lowerCAmelCase_      			=			parquet_writer_kwargs




				def   __a		(      self  )       ->   int:
							lowerCAmelCase_      			=			self.batch_size if self.batch_size else config.DEFAULT_MAX_BATCH_SIZE

							if isinstance(self.path_or_buf       ,       (str, bytes, os.PathLike)  ):
										with open(self.path_or_buf       ,       "wb+"  ) as buffer:
													lowerCAmelCase_      			=			self._write(file_obj=_UpperCamelCase       ,       batch_size=_UpperCamelCase       ,       **self.parquet_writer_kwargs  )
							else:
										lowerCAmelCase_      			=			self._write(file_obj=self.path_or_buf       ,       batch_size=_UpperCamelCase       ,       **self.parquet_writer_kwargs  )
							return written




				def   __a		(      self       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       **_UpperCamelCase  )       ->   int:
							lowerCAmelCase_      			=			0
							lowerCAmelCase_      			=			parquet_writer_kwargs.pop("path_or_buf"       ,       _UpperCamelCase  )
							lowerCAmelCase_      			=			self.dataset.features.arrow_schema

							lowerCAmelCase_      			=			pq.ParquetWriter(_UpperCamelCase       ,       schema=_UpperCamelCase       ,       **_UpperCamelCase  )

							for offset in logging.tqdm(
							    range(0       ,       len(self.dataset  )       ,       _UpperCamelCase  )       ,       unit="ba"       ,       disable=not logging.is_progress_bar_enabled()       ,       desc="Creating parquet from Arrow format"       ,       ):
										lowerCAmelCase_      			=			query_table(
										    table=self.dataset._data       ,       key=slice(_UpperCamelCase       ,       offset + batch_size  )       ,       indices=self.dataset._indices if self.dataset._indices is not None else None       ,       )
										writer.write_table(_UpperCamelCase  )
										written += batch.nbytes
							writer.close()
							return written





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


_A     =  {}

try:
			if not is_sentencepiece_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A     =  ["NllbTokenizer"]

try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A     =  ["NllbTokenizerFast"]


if TYPE_CHECKING:
			try:
						if not is_sentencepiece_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_nllb import NllbTokenizer

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_nllb_fast import NllbTokenizerFast

else:
			import sys

			_A     =  _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



