





"""simple docstring"""





import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


lowerCAmelCase__     			=			datasets.utils.logging.get_logger(__name__)





@dataclass
class        _lowerCamelCase      (       datasets.BuilderConfig ):
						UpperCAmelCase_ =			None
						UpperCAmelCase_ =			"utf-8"
						UpperCAmelCase_ =			None
						UpperCAmelCase_ =			None
						UpperCAmelCase_ =			True  # deprecated
						UpperCAmelCase_ =			None  # deprecated
						UpperCAmelCase_ =			10 << 20  # 10MB
						UpperCAmelCase_ =			None





class        _lowerCamelCase      (       datasets.ArrowBasedBuilder ):
						UpperCAmelCase_ =			JsonConfig


						def     snake_case_   (self		) ->   Any:
								if self.config.block_size is not None:
										logger.warning("The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"		)
										UpperCamelCase   =	self.config.block_size
								if self.config.use_threads is not True:
										logger.warning(
										    "The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."		)
								if self.config.newlines_in_values is not None:
										raise ValueError("The JSON loader parameter `newlines_in_values` is no longer supported"		)
								return datasets.DatasetInfo(features=self.config.features		)


						def     snake_case_   (self   ,							__a		) ->   int:
								if not self.config.data_files:
										raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}"		)
								UpperCamelCase   =	dl_manager.download_and_extract(self.config.data_files		)
								if isinstance(__a   ,							(str, list, tuple)		):
										UpperCamelCase   =	data_files
										if isinstance(__a   ,							__a		):
												UpperCamelCase   =	[files]
										UpperCamelCase   =	[dl_manager.iter_files(__a		) for file in files]
										return [datasets.SplitGenerator(name=datasets.Split.TRAIN   ,							gen_kwargs={"files": files}		)]
								UpperCamelCase   =	[]
								for split_name, files in data_files.items():
										if isinstance(__a   ,							__a		):
												UpperCamelCase   =	[files]
										UpperCamelCase   =	[dl_manager.iter_files(__a		) for file in files]
										splits.append(datasets.SplitGenerator(name=__a   ,							gen_kwargs={"files": files}		)		)
								return splits


						def     snake_case_   (self   ,							__a		) ->   pa.Table:
								if self.config.features is not None:
										# adding missing columns
										for column_name in set(self.config.features		) - set(pa_table.column_names		):
												UpperCamelCase   =	self.config.features.arrow_schema.field(__a		).type
												UpperCamelCase   =	pa_table.append_column(__a   ,							pa.array([None] * len(__a		)   ,							type=__a		)		)
										# more expensive cast to support nested structures with keys in a different order
										# allows str <-> int/float or str to Audio for example
										UpperCamelCase   =	table_cast(__a   ,							self.config.features.arrow_schema		)
								return pa_table




						def     snake_case_   (self   ,							__a		) ->   Optional[int]:
								for file_idx, file in enumerate(itertools.chain.from_iterable(__a		)		):
										# If the file is one json object and if we need to look at the list of items in one specific field
										if self.config.field is not None:
												with open(__a   ,							encoding=self.config.encoding   ,							errors=self.config.encoding_errors		) as f:
														UpperCamelCase   =	json.load(__a		)

												# We keep only the field we are interested in
												UpperCamelCase   =	dataset[self.config.field]

												# We accept two format: a list of dicts or a dict of lists
												if isinstance(__a   ,							(list, tuple)		):
														UpperCamelCase   =	set().union(*[row.keys() for row in dataset]		)
														UpperCamelCase   =	{col: [row.get(__a		) for row in dataset] for col in keys}
												else:
														UpperCamelCase   =	dataset
												UpperCamelCase   =	pa.Table.from_pydict(__a		)
												yield file_idx, self._cast_table(__a		)

										# If the file has one json object per line
										else:
												with open(__a   ,							"rb"		) as f:
														UpperCamelCase   =	0
														# Use block_size equal to the chunk size divided by 32 to leverage multithreading
														# Set a default minimum value of 16kB if the chunk size is really small
														UpperCamelCase   =	max(self.config.chunksize // 32   ,							16 << 10		)
														UpperCamelCase   =	(
														    self.config.encoding_errors if self.config.encoding_errors is not None else "strict"
														)
														while True:
																UpperCamelCase   =	f.read(self.config.chunksize		)
																if not batch:
																		break
																# Finish current line
																try:
																		batch += f.readline()
																except (AttributeError, io.UnsupportedOperation):
																		batch += readline(__a		)
																# PyArrow only accepts utf-8 encoded bytes
																if self.config.encoding != "utf-8":
																		UpperCamelCase   =	batch.decode(self.config.encoding   ,							errors=__a		).encode("utf-8"		)
																try:
																		while True:
																				try:
																						UpperCamelCase   =	paj.read_json(
																						    io.BytesIO(__a		)   ,							read_options=paj.ReadOptions(block_size=__a		)		)
																						break
																				except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																						if (
																						    isinstance(__a   ,							pa.ArrowInvalid		)
																						    and "straddling" not in str(__a		)
																						    or block_size > len(__a		)
																						):
																								raise
																						else:
																								# Increase the block size in case it was too small.
																								# The block size will be reset for the next file.
																								logger.debug(
																								    F"Batch of {len(__a		)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."		)
																								block_size *= 2
																except pa.ArrowInvalid as e:
																		try:
																				with open(
																				    __a   ,							encoding=self.config.encoding   ,							errors=self.config.encoding_errors		) as f:
																						UpperCamelCase   =	json.load(__a		)
																		except json.JSONDecodeError:
																				logger.error(F"Failed to read file '{file}' with error {type(__a		)}: {e}"		)
																				raise e
																		# If possible, parse the file as a list of json objects and exit the loop
																		if isinstance(__a   ,							__a		):  # list is the only sequence type supported in JSON
																				try:
																						UpperCamelCase   =	set().union(*[row.keys() for row in dataset]		)
																						UpperCamelCase   =	{col: [row.get(__a		) for row in dataset] for col in keys}
																						UpperCamelCase   =	pa.Table.from_pydict(__a		)
																				except (pa.ArrowInvalid, AttributeError) as e:
																						logger.error(F"Failed to read file '{file}' with error {type(__a		)}: {e}"		)
																						raise ValueError(F"Not able to read records in the JSON file at {file}."		) from None
																				yield file_idx, self._cast_table(__a		)
																				break
																		else:
																				logger.error(F"Failed to read file '{file}' with error {type(__a		)}: {e}"		)
																				raise ValueError(
																				    F"Not able to read records in the JSON file at {file}. "
																				    F"You should probably indicate the field of the JSON file containing your records. "
																				    F"This JSON file contain the following fields: {str(list(dataset.keys()		)		)}. "
																				    F"Select the correct one and provide it as `field='XXX'` to the dataset loading method. "		) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																yield (file_idx, batch_idx), self._cast_table(__a		)
																batch_idx += 1













"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase__     			=			logging.get_logger(__name__)

lowerCAmelCase__     			=			{
    '''microsoft/swinv2-tiny-patch4-window8-256''': (
        '''https://huggingface.co/microsoft/swinv2-tiny-patch4-window8-256/resolve/main/config.json'''
    ),
}





class        _lowerCamelCase      (       _lowercase ):
						UpperCAmelCase_ =			"swinv2"

						UpperCAmelCase_ =			{
						    "num_attention_heads": "num_heads",
						    "num_hidden_layers": "num_layers",
						}


						def __init__(self   ,							__a=2_24   ,							__a=4   ,							__a=3   ,							__a=96   ,							__a=[2, 2, 6, 2]   ,							__a=[3, 6, 12, 24]   ,							__a=7   ,							__a=4.0   ,							__a=True   ,							__a=0.0   ,							__a=0.0   ,							__a=0.1   ,							__a="gelu"   ,							__a=False   ,							__a=0.02   ,							__a=1e-5   ,							__a=32   ,							**__a   ,							) ->   Optional[Any]:
								super().__init__(**__a		)

								UpperCamelCase   =	image_size
								UpperCamelCase   =	patch_size
								UpperCamelCase   =	num_channels
								UpperCamelCase   =	embed_dim
								UpperCamelCase   =	depths
								UpperCamelCase   =	len(__a		)
								UpperCamelCase   =	num_heads
								UpperCamelCase   =	window_size
								UpperCamelCase   =	mlp_ratio
								UpperCamelCase   =	qkv_bias
								UpperCamelCase   =	hidden_dropout_prob
								UpperCamelCase   =	attention_probs_dropout_prob
								UpperCamelCase   =	drop_path_rate
								UpperCamelCase   =	hidden_act
								UpperCamelCase   =	use_absolute_embeddings
								UpperCamelCase   =	layer_norm_eps
								UpperCamelCase   =	initializer_range
								UpperCamelCase   =	encoder_stride
								# we set the hidden_size attribute in order to make Swinv2 work with VisionEncoderDecoderModel
								# this indicates the channel dimension after the last stage of the model
								UpperCamelCase   =	int(embed_dim * 2 ** (len(__a		) - 1)		)
								UpperCamelCase   =	(0, 0, 0, 0)






