



"""simple docstring"""







import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader






class       UpperCamelCase__			(   _lowerCAmelCase      ):

			"""simple docstring"""


			def __init__(      self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__ = None						,	SCREAMING_SNAKE_CASE__ = None						,	SCREAMING_SNAKE_CASE__ = None						,	SCREAMING_SNAKE_CASE__ = False						,	SCREAMING_SNAKE_CASE__ = False						,	SCREAMING_SNAKE_CASE__ = None						,	SCREAMING_SNAKE_CASE__ = None						,	**SCREAMING_SNAKE_CASE__						,	)      ->	Optional[Any]:
				super().__init__(
				    SCREAMING_SNAKE_CASE__						,	split=SCREAMING_SNAKE_CASE__						,	features=SCREAMING_SNAKE_CASE__						,	cache_dir=SCREAMING_SNAKE_CASE__						,	keep_in_memory=SCREAMING_SNAKE_CASE__						,	streaming=SCREAMING_SNAKE_CASE__						,	num_proc=SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__						,	)
				A__											=					field
				A__											=					path_or_paths if isinstance(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__		) else {self.split: path_or_paths}
				A__											=					Json(
				    cache_dir=SCREAMING_SNAKE_CASE__						,	data_files=SCREAMING_SNAKE_CASE__						,	features=SCREAMING_SNAKE_CASE__						,	field=SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__						,	)







			def   snake_case__	(      self		)      ->	Any:
				# Build iterable dataset
				if self.streaming:
					A__											=					self.builder.as_streaming_dataset(split=self.split		)
				# Build regular (map-style) dataset
				else:
					A__											=					None
					A__											=					None
					A__											=					None
					A__											=					None

					self.builder.download_and_prepare(
					    download_config=SCREAMING_SNAKE_CASE__						,	download_mode=SCREAMING_SNAKE_CASE__						,	verification_mode=SCREAMING_SNAKE_CASE__						,	base_path=SCREAMING_SNAKE_CASE__						,	num_proc=self.num_proc						,	)
					A__											=					self.builder.as_dataset(
					    split=self.split						,	verification_mode=SCREAMING_SNAKE_CASE__						,	in_memory=self.keep_in_memory		)
				return dataset






class       UpperCamelCase__			:

			"""simple docstring"""


			def __init__(      self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__ = None						,	SCREAMING_SNAKE_CASE__ = None						,	**SCREAMING_SNAKE_CASE__						,	)      ->	str:
				if num_proc is not None and num_proc <= 0:
					raise ValueError(f"""num_proc {num_proc} must be an integer > 0."""		)

				A__											=					dataset
				A__											=					path_or_buf
				A__											=					batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
				A__											=					num_proc
				A__											=					"utf-8"
				A__											=					to_json_kwargs


			def   snake_case__	(      self		)      ->	int:
				A__											=					self.to_json_kwargs.pop("path_or_buf"						,	SCREAMING_SNAKE_CASE__		)
				A__											=					self.to_json_kwargs.pop("orient"						,	"records"		)
				A__											=					self.to_json_kwargs.pop("lines"						,	True if orient == "records" else False		)
				A__											=					self.to_json_kwargs.pop("index"						,	False if orient in ["split", "table"] else True		)
				A__											=					self.to_json_kwargs.pop("compression"						,	SCREAMING_SNAKE_CASE__		)

				if compression not in [None, "infer", "gzip", "bz2", "xz"]:
					raise NotImplementedError(f"""`datasets` currently does not support {compression} compression"""		)

				if isinstance(self.path_or_buf						,	(str, bytes, os.PathLike)		):
					with fsspec.open(self.path_or_buf						,	"wb"						,	compression=SCREAMING_SNAKE_CASE__		) as buffer:
						A__											=					self._write(file_obj=SCREAMING_SNAKE_CASE__						,	orient=SCREAMING_SNAKE_CASE__						,	lines=SCREAMING_SNAKE_CASE__						,	index=SCREAMING_SNAKE_CASE__						,	**self.to_json_kwargs		)
				else:
					if compression:
						raise NotImplementedError(
						    f"""The compression parameter is not supported when writing to a buffer, but compression={compression}"""
						    " was passed. Please provide a local path instead."		)
					A__											=					self._write(
					    file_obj=self.path_or_buf						,	orient=SCREAMING_SNAKE_CASE__						,	lines=SCREAMING_SNAKE_CASE__						,	index=SCREAMING_SNAKE_CASE__						,	**self.to_json_kwargs		)
				return written


			def   snake_case__	(      self						,	SCREAMING_SNAKE_CASE__		)      ->	List[Any]:
				A__							,							A__							,							A__							,							A__							,							A__											=					args

				A__											=					query_table(
				    table=self.dataset.data						,	key=slice(SCREAMING_SNAKE_CASE__						,	offset + self.batch_size		)						,	indices=self.dataset._indices						,	)
				A__											=					batch.to_pandas().to_json(
				    path_or_buf=SCREAMING_SNAKE_CASE__						,	orient=SCREAMING_SNAKE_CASE__						,	lines=SCREAMING_SNAKE_CASE__						,	index=SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__		)
				if not json_str.endswith("\n"		):
					json_str += "\n"
				return json_str.encode(self.encoding		)







			def   snake_case__	(      self						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__						,	)      ->	int:
				A__											=					0

				if self.num_proc is None or self.num_proc == 1:
					for offset in logging.tqdm(
					    range(0						,	len(self.dataset		)						,	self.batch_size		)						,	unit="ba"						,	disable=not logging.is_progress_bar_enabled()						,	desc="Creating json from Arrow format"						,	):
						A__											=					self._batch_json((offset, orient, lines, index, to_json_kwargs)		)
						written += file_obj.write(SCREAMING_SNAKE_CASE__		)
				else:
					A__							,							A__											=					len(self.dataset		), self.batch_size
					with multiprocessing.Pool(self.num_proc		) as pool:
						for json_str in logging.tqdm(
						    pool.imap(
						        self._batch_json						,	[(offset, orient, lines, index, to_json_kwargs) for offset in range(0						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__		)]						,	)						,	total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size						,	unit="ba"						,	disable=not logging.is_progress_bar_enabled()						,	desc="Creating json from Arrow format"						,	):
							written += file_obj.write(SCREAMING_SNAKE_CASE__		)

				return written








"""simple docstring"""






import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding






class      A(  lowerCamelCase__		):

		"""simple docstring"""





		A    =							["image_processor", "tokenizer"]
		A    =							"ViTImageProcessor"
		A    =							("CLIPTokenizer", "CLIPTokenizerFast")
		def __init__( self					,				SCREAMING_SNAKE_CASE__=None					,				SCREAMING_SNAKE_CASE__=None					,				**SCREAMING_SNAKE_CASE__   )      ->	List[str]:

					"""simple docstring"""



					_UpperCamelCase					:List[str]					     =	None
					if "feature_extractor" in kwargs:
								warnings.warn(
								    '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
								    ''' instead.'''					,				SCREAMING_SNAKE_CASE__					,				)
								_UpperCamelCase					:List[str]					     =	kwargs.pop('''feature_extractor'''   )

					_UpperCamelCase					:Optional[Any]					     =	image_processor if image_processor is not None else feature_extractor
					if image_processor is None:
								raise ValueError('''You need to specify an `image_processor`.'''   )
					if tokenizer is None:
								raise ValueError('''You need to specify a `tokenizer`.'''   )

					super().__init__(SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__   )
		def __call__( self					,				SCREAMING_SNAKE_CASE__=None					,				SCREAMING_SNAKE_CASE__=None					,				SCREAMING_SNAKE_CASE__=None					,				SCREAMING_SNAKE_CASE__=None					,				**SCREAMING_SNAKE_CASE__   )      ->	Tuple:

					"""simple docstring"""



					if text is None and visual_prompt is None and images is None:
								raise ValueError('''You have to specify either text, visual prompt or images.'''   )

					if text is not None and visual_prompt is not None:
								raise ValueError('''You have to specify exactly one type of prompt. Either text or visual prompt.'''   )

					if text is not None:
								_UpperCamelCase					:Dict					     =	self.tokenizer(SCREAMING_SNAKE_CASE__					,				return_tensors=SCREAMING_SNAKE_CASE__					,				**SCREAMING_SNAKE_CASE__   )

					if visual_prompt is not None:
								_UpperCamelCase					:Optional[Any]					     =	self.image_processor(SCREAMING_SNAKE_CASE__					,				return_tensors=SCREAMING_SNAKE_CASE__					,				**SCREAMING_SNAKE_CASE__   )

					if images is not None:
								_UpperCamelCase					:Union[str, Any]					     =	self.image_processor(SCREAMING_SNAKE_CASE__					,				return_tensors=SCREAMING_SNAKE_CASE__					,				**SCREAMING_SNAKE_CASE__   )

					if visual_prompt is not None and images is not None:
								_UpperCamelCase					:List[str]					     =	{
								    '''pixel_values''': image_features.pixel_values,
								    '''conditional_pixel_values''': prompt_features.pixel_values,
								}
								return encoding
					elif text is not None and images is not None:
								_UpperCamelCase					:List[Any]					     =	image_features.pixel_values
								return encoding
					elif text is not None:
								return encoding
					elif visual_prompt is not None:
								_UpperCamelCase					:Optional[Any]					     =	{
								    '''conditional_pixel_values''': prompt_features.pixel_values,
								}
								return encoding
					else:
								return BatchEncoding(data=dict(**SCREAMING_SNAKE_CASE__   )					,				tensor_type=SCREAMING_SNAKE_CASE__   )
		def   _UpperCamelCase( self					,				*SCREAMING_SNAKE_CASE__					,				**SCREAMING_SNAKE_CASE__   )      ->	Any:

					"""simple docstring"""



					return self.tokenizer.batch_decode(*SCREAMING_SNAKE_CASE__					,				**SCREAMING_SNAKE_CASE__   )
		def   _UpperCamelCase( self					,				*SCREAMING_SNAKE_CASE__					,				**SCREAMING_SNAKE_CASE__   )      ->	Tuple:

					"""simple docstring"""



					return self.tokenizer.decode(*SCREAMING_SNAKE_CASE__					,				**SCREAMING_SNAKE_CASE__   )
		@property
		def   _UpperCamelCase( self   )      ->	Tuple:

					"""simple docstring"""



					warnings.warn(
					    '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'''					,				SCREAMING_SNAKE_CASE__					,				)
					return self.image_processor_class





		@property
		def   _UpperCamelCase( self   )      ->	Union[str, Any]:

					"""simple docstring"""



					warnings.warn(
					    '''`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'''					,				SCREAMING_SNAKE_CASE__					,				)
					return self.image_processor

