


'''simple docstring'''






import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


__UpperCAmelCase    				=							datasets.utils.logging.get_logger(__name__)




@dataclass
class 		a__				(							datasets.BuilderConfig  ):





							'''simple docstring'''






							lowercase__			:   Optional[datasets.Features]							=      None
							lowercase__			:   str							=      "utf-8"
							lowercase__			:   Optional[str]							=      None
							lowercase__			:   int							=      1_0 << 2_0  # 10MB
							lowercase__			:   bool							=      False
							lowercase__			:   str							=      "line"




class 		a__				(							datasets.ArrowBasedBuilder  ):





							'''simple docstring'''






							lowercase__			:   Dict							=      TextConfig







							def   __SCREAMING_SNAKE_CASE							(		self   )       ->							Union[str, Any]:
														return datasets.DatasetInfo(features=self.config.features   )







							def   __SCREAMING_SNAKE_CASE							(		self					,    lowerCamelCase_   )       ->							Union[str, Any]:
														if not self.config.data_files:
																					raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""   )
														lowerCAmelCase__          =	dl_manager.download_and_extract(self.config.data_files   )
														if isinstance(lowerCamelCase_					,    (str, list, tuple)   ):
																					lowerCAmelCase__          =	data_files
																					if isinstance(lowerCamelCase_					,    lowerCamelCase_   ):
																												lowerCAmelCase__          =	[files]
																					lowerCAmelCase__          =	[dl_manager.iter_files(lowerCamelCase_   ) for file in files]
																					return [datasets.SplitGenerator(name=datasets.Split.TRAIN					,    gen_kwargs={'''files''': files}   )]
														lowerCAmelCase__          =	[]
														for split_name, files in data_files.items():
																					if isinstance(lowerCamelCase_					,    lowerCamelCase_   ):
																												lowerCAmelCase__          =	[files]
																					lowerCAmelCase__          =	[dl_manager.iter_files(lowerCamelCase_   ) for file in files]
																					splits.append(datasets.SplitGenerator(name=lowerCamelCase_					,    gen_kwargs={'''files''': files}   )   )
														return splits







							def   __SCREAMING_SNAKE_CASE							(		self					,    lowerCamelCase_   )       ->							pa.Table:
														if self.config.features is not None:
																					lowerCAmelCase__          =	self.config.features.arrow_schema
																					if all(not require_storage_cast(lowerCamelCase_   ) for feature in self.config.features.values()   ):
																												# cheaper cast
																												lowerCAmelCase__          =	pa_table.cast(lowerCamelCase_   )
																					else:
																												# more expensive cast; allows str <-> int/float or str to Audio for example
																												lowerCAmelCase__          =	table_cast(lowerCamelCase_					,    lowerCamelCase_   )
																					return pa_table
														else:
																					return pa_table.cast(pa.schema({'''text''': pa.string()}   )   )







							def   __SCREAMING_SNAKE_CASE							(		self					,    lowerCamelCase_   )       ->							Dict:
														lowerCAmelCase__          =	list(self.config.features   ) if self.config.features is not None else ['''text''']
														for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCamelCase_   )   ):
																					# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
																					with open(lowerCamelCase_					,    encoding=self.config.encoding					,    errors=self.config.errors   ) as f:
																												if self.config.sample_by == "line":
																																			lowerCAmelCase__          =	0
																																			while True:
																																										lowerCAmelCase__          =	f.read(self.config.chunksize   )
																																										if not batch:
																																																	break
																																										batch += f.readline()  # finish current line
																																										# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																																										lowerCAmelCase__          =	StringIO(lowerCamelCase_   ).readlines()
																																										if not self.config.keep_linebreaks:
																																																	lowerCAmelCase__          =	[line.rstrip('''\n'''   ) for line in batch]
																																										lowerCAmelCase__          =	pa.Table.from_arrays([pa.array(lowerCamelCase_   )]					,    names=lowerCamelCase_   )
																																										# Uncomment for debugging (will print the Arrow table size and elements)
																																										# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																										# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																										yield (file_idx, batch_idx), self._cast_table(lowerCamelCase_   )
																																										batch_idx += 1
																												elif self.config.sample_by == "paragraph":
																																			lowerCAmelCase__          =	0
																																			lowerCAmelCase__          =	''''''
																																			while True:
																																										lowerCAmelCase__          =	f.read(self.config.chunksize   )
																																										if not new_batch:
																																																	break
																																										batch += new_batch
																																										batch += f.readline()  # finish current line
																																										lowerCAmelCase__          =	batch.split('''\n\n'''   )
																																										lowerCAmelCase__          =	pa.Table.from_arrays(
																																										    [pa.array([example for example in batch[:-1] if example]   )]					,    names=lowerCamelCase_   )
																																										# Uncomment for debugging (will print the Arrow table size and elements)
																																										# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																										# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																										yield (file_idx, batch_idx), self._cast_table(lowerCamelCase_   )
																																										batch_idx += 1
																																										lowerCAmelCase__          =	batch[-1]
																																			if batch:
																																										lowerCAmelCase__          =	pa.Table.from_arrays([pa.array([batch]   )]					,    names=lowerCamelCase_   )
																																										yield (file_idx, batch_idx), self._cast_table(lowerCamelCase_   )
																												elif self.config.sample_by == "document":
																																			lowerCAmelCase__          =	f.read()
																																			lowerCAmelCase__          =	pa.Table.from_arrays([pa.array([text]   )]					,    names=lowerCamelCase_   )
																																			yield file_idx, self._cast_table(lowerCamelCase_   )



'''simple docstring'''






import warnings

from ...utils import logging
from .image_processing_mobilenet_va import MobileNetVaImageProcessor


__UpperCAmelCase    				=							logging.get_logger(__name__)




class 		a__				(							a__  ):





							'''simple docstring'''






							def __init__(		self					,    *lowerCamelCase_					,    **lowerCamelCase_   )       ->							None:
														warnings.warn(
														    '''The class MobileNetV2FeatureExtractor is deprecated and will be removed in version 5 of Transformers.'''
														    ''' Please use MobileNetV2ImageProcessor instead.'''					,    lowerCamelCase_					,    )
														super().__init__(*lowerCamelCase_					,    **lowerCamelCase_   )