




import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


_lowerCamelCase					:     int					  =  datasets.utils.logging.get_logger(__name__)
@dataclass
class 				__snake_case     (datasets.BuilderConfig						):
				lowerCAmelCase__			=   None
				lowerCAmelCase__			=   "utf-8"
				lowerCAmelCase__			=   None
				lowerCAmelCase__			=   1_0 << 2_0  # 10MB
				lowerCAmelCase__			=   False
				lowerCAmelCase__			=   "line"






class 				__snake_case     (datasets.ArrowBasedBuilder						):
				lowerCAmelCase__			=   TextConfig



				def 					SCREAMING_SNAKE_CASE				(					self					:						Dict					)     ->			int:





								'''simple docstring'''




								return datasets.DatasetInfo(features=self.config.features					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						List[Any]						,		_UpperCAmelCase					:						List[str]					)     ->			str:





								'''simple docstring'''




								if not self.config.data_files:
												raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}"					)
								_lowerCAmelCase			:			Union[str, Any]        =					dl_manager.download_and_extract(self.config.data_files					)
								if isinstance(_UpperCAmelCase						,		(str, list, tuple)					):
												_lowerCAmelCase			:			Dict        =					data_files
												if isinstance(_UpperCAmelCase						,		_UpperCAmelCase					):
																_lowerCAmelCase			:			int        =					[files]
												_lowerCAmelCase			:			Tuple        =					[dl_manager.iter_files(_UpperCAmelCase					) for file in files]
												return [datasets.SplitGenerator(name=datasets.Split.TRAIN						,		gen_kwargs={"""files""": files}					)]
								_lowerCAmelCase			:			Dict        =					[]
								for split_name, files in data_files.items():
												if isinstance(_UpperCAmelCase						,		_UpperCAmelCase					):
																_lowerCAmelCase			:			Dict        =					[files]
												_lowerCAmelCase			:			Tuple        =					[dl_manager.iter_files(_UpperCAmelCase					) for file in files]
												splits.append(datasets.SplitGenerator(name=_UpperCAmelCase						,		gen_kwargs={"""files""": files}					)					)
								return splits



				def 					SCREAMING_SNAKE_CASE				(					self					:						List[str]						,		_UpperCAmelCase					:						pa.Table					)     ->			pa.Table:





								'''simple docstring'''




								if self.config.features is not None:
												_lowerCAmelCase			:			Optional[int]        =					self.config.features.arrow_schema
												if all(not require_storage_cast(_UpperCAmelCase					) for feature in self.config.features.values()					):
																# cheaper cast
																_lowerCAmelCase			:			Optional[int]        =					pa_table.cast(_UpperCAmelCase					)
												else:
																# more expensive cast; allows str <-> int/float or str to Audio for example
																_lowerCAmelCase			:			List[str]        =					table_cast(_UpperCAmelCase						,		_UpperCAmelCase					)
												return pa_table
								else:
												return pa_table.cast(pa.schema({"""text""": pa.string()}					)					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						str						,		_UpperCAmelCase					:						Optional[Any]					)     ->			Tuple:





								'''simple docstring'''




								_lowerCAmelCase			:			List[str]        =					list(self.config.features					) if self.config.features is not None else ["""text"""]
								for file_idx, file in enumerate(itertools.chain.from_iterable(_UpperCAmelCase					)					):
												# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
												with open(_UpperCAmelCase						,		encoding=self.config.encoding						,		errors=self.config.errors					) as f:
																if self.config.sample_by == "line":
																				_lowerCAmelCase			:			Optional[int]        =					0
																				while True:
																								_lowerCAmelCase			:			int        =					f.read(self.config.chunksize					)
																								if not batch:
																												break
																								batch += f.readline()  # finish current line
																								# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																								_lowerCAmelCase			:			str        =					StringIO(_UpperCAmelCase					).readlines()
																								if not self.config.keep_linebreaks:
																												_lowerCAmelCase			:			Dict        =					[line.rstrip("""\n"""					) for line in batch]
																								_lowerCAmelCase			:			List[Any]        =					pa.Table.from_arrays([pa.array(_UpperCAmelCase					)]						,		names=_UpperCAmelCase					)
																								# Uncomment for debugging (will print the Arrow table size and elements)
																								# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																								# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																								yield (file_idx, batch_idx), self._cast_table(_UpperCAmelCase					)
																								batch_idx += 1
																elif self.config.sample_by == "paragraph":
																				_lowerCAmelCase			:			Dict        =					0
																				_lowerCAmelCase			:			Dict        =					""""""
																				while True:
																								_lowerCAmelCase			:			str        =					f.read(self.config.chunksize					)
																								if not new_batch:
																												break
																								batch += new_batch
																								batch += f.readline()  # finish current line
																								_lowerCAmelCase			:			Optional[int]        =					batch.split("""\n\n"""					)
																								_lowerCAmelCase			:			Union[str, Any]        =					pa.Table.from_arrays(
																								    [pa.array([example for example in batch[:-1] if example]					)]						,		names=_UpperCAmelCase					)
																								# Uncomment for debugging (will print the Arrow table size and elements)
																								# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																								# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																								yield (file_idx, batch_idx), self._cast_table(_UpperCAmelCase					)
																								batch_idx += 1
																								_lowerCAmelCase			:			Tuple        =					batch[-1]
																				if batch:
																								_lowerCAmelCase			:			str        =					pa.Table.from_arrays([pa.array([batch]					)]						,		names=_UpperCAmelCase					)
																								yield (file_idx, batch_idx), self._cast_table(_UpperCAmelCase					)
																elif self.config.sample_by == "document":
																				_lowerCAmelCase			:			Any        =					f.read()
																				_lowerCAmelCase			:			Optional[int]        =					pa.Table.from_arrays([pa.array([text]					)]						,		names=_UpperCAmelCase					)
																				yield file_idx, self._cast_table(_UpperCAmelCase					)







from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_lowerCamelCase					:     Optional[int]					  =  logging.get_logger(__name__)

_lowerCamelCase					:     Optional[int]					  =  {
    "squeezebert/squeezebert-uncased": (
        "https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/config.json"
    ),
    "squeezebert/squeezebert-mnli": "https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/config.json",
    "squeezebert/squeezebert-mnli-headless": (
        "https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/config.json"
    ),
}
class 				__snake_case     (_a						):
				lowerCAmelCase__			=   SQUEEZEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
				lowerCAmelCase__			=   "squeezebert"



				def __init__(					self					:						Optional[int]						,		_UpperCAmelCase					:						Tuple=3_0522						,		_UpperCAmelCase					:						str=768						,		_UpperCAmelCase					:						int=12						,		_UpperCAmelCase					:						Tuple=12						,		_UpperCAmelCase					:						Tuple=3072						,		_UpperCAmelCase					:						Union[str, Any]="gelu"						,		_UpperCAmelCase					:						Union[str, Any]=0.1						,		_UpperCAmelCase					:						List[Any]=0.1						,		_UpperCAmelCase					:						List[Any]=512						,		_UpperCAmelCase					:						Tuple=2						,		_UpperCAmelCase					:						Optional[Any]=0.02						,		_UpperCAmelCase					:						Any=1E-12						,		_UpperCAmelCase					:						Optional[Any]=0						,		_UpperCAmelCase					:						Union[str, Any]=768						,		_UpperCAmelCase					:						Any=4						,		_UpperCAmelCase					:						Any=4						,		_UpperCAmelCase					:						Optional[int]=4						,		_UpperCAmelCase					:						str=1						,		_UpperCAmelCase					:						Union[str, Any]=4						,		_UpperCAmelCase					:						List[str]=4						,		**_UpperCAmelCase					:						int						,		)     ->			Any:





								'''simple docstring'''




								super().__init__(pad_token_id=_UpperCAmelCase						,		**_UpperCAmelCase					)

								_lowerCAmelCase			:			Optional[Any]        =					vocab_size
								_lowerCAmelCase			:			Optional[Any]        =					hidden_size
								_lowerCAmelCase			:			Tuple        =					num_hidden_layers
								_lowerCAmelCase			:			int        =					num_attention_heads
								_lowerCAmelCase			:			List[Any]        =					hidden_act
								_lowerCAmelCase			:			List[Any]        =					intermediate_size
								_lowerCAmelCase			:			List[str]        =					hidden_dropout_prob
								_lowerCAmelCase			:			Any        =					attention_probs_dropout_prob
								_lowerCAmelCase			:			str        =					max_position_embeddings
								_lowerCAmelCase			:			int        =					type_vocab_size
								_lowerCAmelCase			:			str        =					initializer_range
								_lowerCAmelCase			:			int        =					layer_norm_eps
								_lowerCAmelCase			:			int        =					embedding_size
								_lowerCAmelCase			:			Optional[int]        =					q_groups
								_lowerCAmelCase			:			List[Any]        =					k_groups
								_lowerCAmelCase			:			List[Any]        =					v_groups
								_lowerCAmelCase			:			Dict        =					post_attention_groups
								_lowerCAmelCase			:			Optional[Any]        =					intermediate_groups
								_lowerCAmelCase			:			str        =					output_groups






class 				__snake_case     (_a						):



				@property
				def 					SCREAMING_SNAKE_CASE				(					self					:						Union[str, Any]					)     ->			Mapping[str, Mapping[int, str]]:





								'''simple docstring'''




								if self.task == "multiple-choice":
												_lowerCAmelCase			:			Optional[int]        =					{0: """batch""", 1: """choice""", 2: """sequence"""}
								else:
												_lowerCAmelCase			:			int        =					{0: """batch""", 1: """sequence"""}
								return OrderedDict(
								    [
								        ("""input_ids""", dynamic_axis),
								        ("""attention_mask""", dynamic_axis),
								        ("""token_type_ids""", dynamic_axis),
								    ]					)

