





import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


SCREAMING_SNAKE_CASE_   :      Union[str, Any]			=   datasets.utils.logging.get_logger(__name__)

@dataclass
class 		snake_case_   (							datasets.BuilderConfig						):







	'''simple docstring'''
	__UpperCamelCase   =		None
	__UpperCamelCase   =		"utf-8"
	__UpperCamelCase   =		None
	__UpperCamelCase   =		None
	__UpperCamelCase   =		True  # deprecated
	__UpperCamelCase   =		None  # deprecated
	__UpperCamelCase   =		10 << 20  # 10MB
	__UpperCamelCase   =		None





class 		snake_case_   (							datasets.ArrowBasedBuilder						):







	'''simple docstring'''
	__UpperCamelCase   =		JsonConfig




	def    UpperCAmelCase			(						self							:     Any   )       -> Optional[int]:

			'''simple docstring'''




			if self.config.block_size is not None:
					logger.warning('The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'   )
					__lowercase											=     self.config.block_size
			if self.config.use_threads is not True:
					logger.warning(
					    'The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'   )
			if self.config.newlines_in_values is not None:
					raise ValueError('The JSON loader parameter `newlines_in_values` is no longer supported'   )
			return datasets.DatasetInfo(features=self.config.features   )




	def    UpperCAmelCase			(						self							:     Optional[Any]					,      __lowerCamelCase							:     Optional[int]   )       -> Tuple:

			'''simple docstring'''




			if not self.config.data_files:
					raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}"   )
			__lowercase											=     dl_manager.download_and_extract(self.config.data_files   )
			if isinstance(__lowerCamelCase					,      (str, list, tuple)   ):
					__lowercase											=     data_files
					if isinstance(__lowerCamelCase					,      __lowerCamelCase   ):
							__lowercase											=     [files]
					__lowercase											=     [dl_manager.iter_files(__lowerCamelCase   ) for file in files]
					return [datasets.SplitGenerator(name=datasets.Split.TRAIN					,      gen_kwargs={'files': files}   )]
			__lowercase											=     []
			for split_name, files in data_files.items():
					if isinstance(__lowerCamelCase					,      __lowerCamelCase   ):
							__lowercase											=     [files]
					__lowercase											=     [dl_manager.iter_files(__lowerCamelCase   ) for file in files]
					splits.append(datasets.SplitGenerator(name=__lowerCamelCase					,      gen_kwargs={'files': files}   )   )
			return splits




	def    UpperCAmelCase			(						self							:     int					,      __lowerCamelCase							:     pa.Table   )       -> pa.Table:

			'''simple docstring'''




			if self.config.features is not None:
					# adding missing columns
					for column_name in set(self.config.features   ) - set(pa_table.column_names   ):
							__lowercase											=     self.config.features.arrow_schema.field(__lowerCamelCase   ).type
							__lowercase											=     pa_table.append_column(__lowerCamelCase					,      pa.array([None] * len(__lowerCamelCase   )					,      type=__lowerCamelCase   )   )
					# more expensive cast to support nested structures with keys in a different order
					# allows str <-> int/float or str to Audio for example
					__lowercase											=     table_cast(__lowerCamelCase					,      self.config.features.arrow_schema   )
			return pa_table






	def    UpperCAmelCase			(						self							:     Dict					,      __lowerCamelCase							:     int   )       -> Optional[Any]:

			'''simple docstring'''




			for file_idx, file in enumerate(itertools.chain.from_iterable(__lowerCamelCase   )   ):
					# If the file is one json object and if we need to look at the list of items in one specific field
					if self.config.field is not None:
							with open(__lowerCamelCase					,      encoding=self.config.encoding					,      errors=self.config.encoding_errors   ) as f:
									__lowercase											=     json.load(__lowerCamelCase   )

							# We keep only the field we are interested in
							__lowercase											=     dataset[self.config.field]

							# We accept two format: a list of dicts or a dict of lists
							if isinstance(__lowerCamelCase					,      (list, tuple)   ):
									__lowercase											=     set().union(*[row.keys() for row in dataset]   )
									__lowercase											=     {col: [row.get(__lowerCamelCase   ) for row in dataset] for col in keys}
							else:
									__lowercase											=     dataset
							__lowercase											=     pa.Table.from_pydict(__lowerCamelCase   )
							yield file_idx, self._cast_table(__lowerCamelCase   )

					# If the file has one json object per line
					else:
							with open(__lowerCamelCase					,      'rb'   ) as f:
									__lowercase											=     0
									# Use block_size equal to the chunk size divided by 32 to leverage multithreading
									# Set a default minimum value of 16kB if the chunk size is really small
									__lowercase											=     max(self.config.chunksize // 32					,      16 << 10   )
									__lowercase											=     (
									    self.config.encoding_errors if self.config.encoding_errors is not None else 'strict'
									)
									while True:
											__lowercase											=     f.read(self.config.chunksize   )
											if not batch:
													break
											# Finish current line
											try:
													batch += f.readline()
											except (AttributeError, io.UnsupportedOperation):
													batch += readline(__lowerCamelCase   )
											# PyArrow only accepts utf-8 encoded bytes
											if self.config.encoding != "utf-8":
													__lowercase											=     batch.decode(self.config.encoding					,      errors=__lowerCamelCase   ).encode('utf-8'   )
											try:
													while True:
															try:
																	__lowercase											=     paj.read_json(
																	    io.BytesIO(__lowerCamelCase   )					,      read_options=paj.ReadOptions(block_size=__lowerCamelCase   )   )
																	break
															except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																	if (
																	    isinstance(__lowerCamelCase					,      pa.ArrowInvalid   )
																	    and "straddling" not in str(__lowerCamelCase   )
																	    or block_size > len(__lowerCamelCase   )
																	):
																			raise
																	else:
																			# Increase the block size in case it was too small.
																			# The block size will be reset for the next file.
																			logger.debug(
																			    F"Batch of {len(__lowerCamelCase   )} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."   )
																			block_size *= 2
											except pa.ArrowInvalid as e:
													try:
															with open(
															    __lowerCamelCase					,      encoding=self.config.encoding					,      errors=self.config.encoding_errors   ) as f:
																	__lowercase											=     json.load(__lowerCamelCase   )
													except json.JSONDecodeError:
															logger.error(F"Failed to read file '{file}' with error {type(__lowerCamelCase   )}: {e}"   )
															raise e
													# If possible, parse the file as a list of json objects and exit the loop
													if isinstance(__lowerCamelCase					,      __lowerCamelCase   ):  # list is the only sequence type supported in JSON
															try:
																	__lowercase											=     set().union(*[row.keys() for row in dataset]   )
																	__lowercase											=     {col: [row.get(__lowerCamelCase   ) for row in dataset] for col in keys}
																	__lowercase											=     pa.Table.from_pydict(__lowerCamelCase   )
															except (pa.ArrowInvalid, AttributeError) as e:
																	logger.error(F"Failed to read file '{file}' with error {type(__lowerCamelCase   )}: {e}"   )
																	raise ValueError(F"Not able to read records in the JSON file at {file}."   ) from None
															yield file_idx, self._cast_table(__lowerCamelCase   )
															break
													else:
															logger.error(F"Failed to read file '{file}' with error {type(__lowerCamelCase   )}: {e}"   )
															raise ValueError(
															    F"Not able to read records in the JSON file at {file}. "
															    F"You should probably indicate the field of the JSON file containing your records. "
															    F"This JSON file contain the following fields: {str(list(dataset.keys()   )   )}. "
															    F"Select the correct one and provide it as `field='XXX'` to the dataset loading method. "   ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
											yield (file_idx, batch_idx), self._cast_table(__lowerCamelCase   )
											batch_idx += 1











from typing import Union

from ..utils import add_end_docstrings, is_torch_available, is_vision_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_vision_available():
		from PIL import Image

		from ..image_utils import load_image

if is_torch_available():
		from ..models.auto.modeling_auto import MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING

SCREAMING_SNAKE_CASE_   :      Union[str, Any]			=   logging.get_logger(__name__)

@add_end_docstrings(UpperCAmelCase_						)
class 		snake_case_   (							UpperCAmelCase_						):







	'''simple docstring'''




	def __init__(						self							:     Optional[int]					,      *__lowerCamelCase							:     Optional[int]					,      **__lowerCamelCase							:     List[Any]   )       -> Optional[Any]:

			'''simple docstring'''




			super().__init__(*__lowerCamelCase					,      **__lowerCamelCase   )
			self.check_model_type(__lowerCamelCase   )




	def    UpperCAmelCase			(						self							:     Optional[int]					,      __lowerCamelCase							:     int=None					,      __lowerCamelCase							:     List[Any]=None					,      __lowerCamelCase							:     Any=None					,      **__lowerCamelCase							:     Optional[int]   )       -> Any:

			'''simple docstring'''




			__lowercase						,      __lowercase											=     {}, {}
			if padding is not None:
					__lowercase											=     padding
			if truncation is not None:
					__lowercase											=     truncation
			if top_k is not None:
					__lowercase											=     top_k
			return preprocess_params, {}, postprocess_params




	def __call__(						self							:     str					,      __lowerCamelCase							:     Union["Image.Image", str]					,      __lowerCamelCase							:     str = None					,      **__lowerCamelCase							:     Dict   )       -> Tuple:

			'''simple docstring'''




			if isinstance(__lowerCamelCase					,      (Image.Image, str)   ) and isinstance(__lowerCamelCase					,      __lowerCamelCase   ):
					__lowercase											=     {'image': image, 'question': question}
			else:
					__lowercase											=     image
			__lowercase											=     super().__call__(__lowerCamelCase					,      **__lowerCamelCase   )
			return results




	def    UpperCAmelCase			(						self							:     List[Any]					,      __lowerCamelCase							:     List[str]					,      __lowerCamelCase							:     Optional[Any]=False					,      __lowerCamelCase							:     Optional[Any]=False   )       -> Optional[int]:

			'''simple docstring'''




			__lowercase											=     load_image(inputs['image']   )
			__lowercase											=     self.tokenizer(
			    inputs['question']					,      return_tensors=self.framework					,      padding=__lowerCamelCase					,      truncation=__lowerCamelCase   )
			__lowercase											=     self.image_processor(images=__lowerCamelCase					,      return_tensors=self.framework   )
			model_inputs.update(__lowerCamelCase   )
			return model_inputs




	def    UpperCAmelCase			(						self							:     str					,      __lowerCamelCase							:     Tuple   )       -> Any:

			'''simple docstring'''




			__lowercase											=     self.model(**__lowerCamelCase   )
			return model_outputs






	def    UpperCAmelCase			(						self							:     int					,      __lowerCamelCase							:     Any					,      __lowerCamelCase							:     int=5   )       -> Tuple:

			'''simple docstring'''




			if top_k > self.model.config.num_labels:
					__lowercase											=     self.model.config.num_labels

			if self.framework == "pt":
					__lowercase											=     model_outputs.logits.sigmoid()[0]
					__lowercase						,      __lowercase											=     probs.topk(__lowerCamelCase   )
			else:
					raise ValueError(F"Unsupported framework: {self.framework}"   )

			__lowercase											=     scores.tolist()
			__lowercase											=     ids.tolist()
			return [{"score": score, "answer": self.model.config.idalabel[_id]} for score, _id in zip(__lowerCamelCase					,      __lowerCamelCase   )]




