


import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


lowerCAmelCase__       :	List[str]     				=datasets.utils.logging.get_logger(__name__)







@dataclass
class      __lowercase  (datasets.BuilderConfig   ):





							"""simple docstring"""







							_UpperCAmelCase     =		None







class      __lowercase  (datasets.ArrowBasedBuilder   ):





							"""simple docstring"""







							_UpperCAmelCase     =		ArrowConfig






							def  UpperCamelCase__			(	self    ):





										"""simple docstring"""




										return datasets.DatasetInfo(features=self.config.features    )






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__    ):





										"""simple docstring"""




										if not self.config.data_files:
													raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''    )
										SCREAMING_SNAKE_CASE_						:			Dict       						= dl_manager.download_and_extract(self.config.data_files    )
										if isinstance(lowerCAmelCase__       ,		(str, list, tuple)    ):
													SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= data_files
													if isinstance(lowerCAmelCase__       ,		lowerCAmelCase__    ):
																SCREAMING_SNAKE_CASE_						:			Optional[int]       						= [files]
													# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
													SCREAMING_SNAKE_CASE_						:			str       						= [dl_manager.iter_files(lowerCAmelCase__    ) for file in files]
													return [datasets.SplitGenerator(name=datasets.Split.TRAIN       ,		gen_kwargs={'files': files}    )]
										SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= []
										for split_name, files in data_files.items():
													if isinstance(lowerCAmelCase__       ,		lowerCAmelCase__    ):
																SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= [files]
													# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
													SCREAMING_SNAKE_CASE_						:			Optional[int]       						= [dl_manager.iter_files(lowerCAmelCase__    ) for file in files]
													# Infer features is they are stoed in the arrow schema
													if self.info.features is None:
																for file in itertools.chain.from_iterable(lowerCAmelCase__    ):
																			with open(lowerCAmelCase__       ,		'rb'    ) as f:
																						SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= datasets.Features.from_arrow_schema(pa.ipc.open_stream(lowerCAmelCase__    ).schema    )
																			break
													splits.append(datasets.SplitGenerator(name=lowerCAmelCase__       ,		gen_kwargs={'files': files}    )    )
										return splits






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__    ):





										"""simple docstring"""




										if self.info.features is not None:
													# more expensive cast to support nested features with keys in a different order
													# allows str <-> int/float or str to Audio for example
													SCREAMING_SNAKE_CASE_						:			Any       						= table_cast(lowerCAmelCase__       ,		self.info.features.arrow_schema    )
										return pa_table






							def  UpperCamelCase__			(	self       ,		lowerCAmelCase__    ):





										"""simple docstring"""




										for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCAmelCase__    )    ):
													with open(lowerCAmelCase__       ,		'rb'    ) as f:
																try:
																			for batch_idx, record_batch in enumerate(pa.ipc.open_stream(lowerCAmelCase__    )    ):
																						SCREAMING_SNAKE_CASE_						:			int       						= pa.Table.from_batches([record_batch]    )
																						# Uncomment for debugging (will print the Arrow table size and elements)
																						# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																						# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																						yield F'''{file_idx}_{batch_idx}''', self._cast_table(lowerCAmelCase__    )
																except ValueError as e:
																			logger.error(F'''Failed to read file \'{file}\' with error {type(lowerCAmelCase__    )}: {e}'''    )
																			raise









from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available, is_tokenizers_available


lowerCAmelCase__       :	List[Any]     				={}

try:
				if not is_sentencepiece_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowerCAmelCase__       :	Optional[Any]     				=['MBart50Tokenizer']

try:
				if not is_tokenizers_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowerCAmelCase__       :	Any     				=['MBart50TokenizerFast']


if TYPE_CHECKING:
				try:
								if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_mbartaa import MBartaaTokenizer

				try:
								if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_mbartaa_fast import MBartaaTokenizerFast

else:
				import sys

				lowerCAmelCase__       :	Optional[int]     				=_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





