






import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


A        =			datasets.utils.logging.get_logger(__name__)


@dataclass
class 							__a							(  datasets.BuilderConfig						):





		'''simple docstring'''



		UpperCAmelCase__      :				Optional[datasets.Features]				    =						None
		UpperCAmelCase__      :				str				    =						"utf-8"
		UpperCAmelCase__      :				Optional[str]				    =						None
		UpperCAmelCase__      :				Optional[str]				    =						None
		UpperCAmelCase__      :				bool				    =						True  # deprecated
		UpperCAmelCase__      :				Optional[int]				    =						None  # deprecated
		UpperCAmelCase__      :				int				    =						10 << 20  # 10MB
		UpperCAmelCase__      :				Optional[bool]				    =						None






class 							__a							(  datasets.ArrowBasedBuilder						):





		'''simple docstring'''



		UpperCAmelCase__      :				List[Any]				    =						JsonConfig


		def   __snake_case			(						self       ):
						if self.config.block_size is not None:
										logger.warning('The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'       )
										SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   self.config.block_size
						if self.config.use_threads is not True:
										logger.warning(
										    'The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'       )
						if self.config.newlines_in_values is not None:
										raise ValueError('The JSON loader parameter `newlines_in_values` is no longer supported'       )
						return datasets.DatasetInfo(features=self.config.features       )


		def   __snake_case			(						self ,    UpperCamelCase__       ):
						if not self.config.data_files:
										raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''       )
						SCREAMING_SNAKE_CASE_   :	str													=   dl_manager.download_and_extract(self.config.data_files       )
						if isinstance(UpperCamelCase__ ,    (str, list, tuple)       ):
										SCREAMING_SNAKE_CASE_   :	List[str]													=   data_files
										if isinstance(UpperCamelCase__ ,    UpperCamelCase__       ):
														SCREAMING_SNAKE_CASE_   :	Tuple													=   [files]
										SCREAMING_SNAKE_CASE_   :	int													=   [dl_manager.iter_files(UpperCamelCase__       ) for file in files]
										return [datasets.SplitGenerator(name=datasets.Split.TRAIN ,    gen_kwargs={'files': files}       )]
						SCREAMING_SNAKE_CASE_   :	str													=   []
						for split_name, files in data_files.items():
										if isinstance(UpperCamelCase__ ,    UpperCamelCase__       ):
														SCREAMING_SNAKE_CASE_   :	Any													=   [files]
										SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   [dl_manager.iter_files(UpperCamelCase__       ) for file in files]
										splits.append(datasets.SplitGenerator(name=UpperCamelCase__ ,    gen_kwargs={'files': files}       )       )
						return splits


		def   __snake_case			(						self ,    UpperCamelCase__       ):
						if self.config.features is not None:
										# adding missing columns
										for column_name in set(self.config.features       ) - set(pa_table.column_names       ):
														SCREAMING_SNAKE_CASE_   :	str													=   self.config.features.arrow_schema.field(UpperCamelCase__       ).type
														SCREAMING_SNAKE_CASE_   :	Any													=   pa_table.append_column(UpperCamelCase__ ,    pa.array([None] * len(UpperCamelCase__       ) ,    type=UpperCamelCase__       )       )
										# more expensive cast to support nested structures with keys in a different order
										# allows str <-> int/float or str to Audio for example
										SCREAMING_SNAKE_CASE_   :	Dict													=   table_cast(UpperCamelCase__ ,    self.config.features.arrow_schema       )
						return pa_table






		def   __snake_case			(						self ,    UpperCamelCase__       ):
						for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCamelCase__       )       ):
										# If the file is one json object and if we need to look at the list of items in one specific field
										if self.config.field is not None:
														with open(UpperCamelCase__ ,    encoding=self.config.encoding ,    errors=self.config.encoding_errors       ) as f:
																		SCREAMING_SNAKE_CASE_   :	List[str]													=   json.load(UpperCamelCase__       )

														# We keep only the field we are interested in
														SCREAMING_SNAKE_CASE_   :	Tuple													=   dataset[self.config.field]

														# We accept two format: a list of dicts or a dict of lists
														if isinstance(UpperCamelCase__ ,    (list, tuple)       ):
																		SCREAMING_SNAKE_CASE_   :	Tuple													=   set().union(*[row.keys() for row in dataset]       )
																		SCREAMING_SNAKE_CASE_   :	List[Any]													=   {col: [row.get(UpperCamelCase__       ) for row in dataset] for col in keys}
														else:
																		SCREAMING_SNAKE_CASE_   :	str													=   dataset
														SCREAMING_SNAKE_CASE_   :	Optional[int]													=   pa.Table.from_pydict(UpperCamelCase__       )
														yield file_idx, self._cast_table(UpperCamelCase__       )

										# If the file has one json object per line
										else:
														with open(UpperCamelCase__ ,    'rb'       ) as f:
																		SCREAMING_SNAKE_CASE_   :	List[str]													=   0
																		# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																		# Set a default minimum value of 16kB if the chunk size is really small
																		SCREAMING_SNAKE_CASE_   :	Any													=   max(self.config.chunksize // 32 ,    16 << 10       )
																		SCREAMING_SNAKE_CASE_   :	Optional[int]													=   (
																		    self.config.encoding_errors if self.config.encoding_errors is not None else 'strict'
																		)
																		while True:
																						SCREAMING_SNAKE_CASE_   :	List[str]													=   f.read(self.config.chunksize       )
																						if not batch:
																										break
																						# Finish current line
																						try:
																										batch += f.readline()
																						except (AttributeError, io.UnsupportedOperation):
																										batch += readline(UpperCamelCase__       )
																						# PyArrow only accepts utf-8 encoded bytes
																						if self.config.encoding != "utf-8":
																										SCREAMING_SNAKE_CASE_   :	List[Any]													=   batch.decode(self.config.encoding ,    errors=UpperCamelCase__       ).encode('utf-8'       )
																						try:
																										while True:
																														try:
																																		SCREAMING_SNAKE_CASE_   :	Dict													=   paj.read_json(
																																		    io.BytesIO(UpperCamelCase__       ) ,    read_options=paj.ReadOptions(block_size=UpperCamelCase__       )       )
																																		break
																														except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																		if (
																																		    isinstance(UpperCamelCase__ ,    pa.ArrowInvalid       )
																																		    and "straddling" not in str(UpperCamelCase__       )
																																		    or block_size > len(UpperCamelCase__       )
																																		):
																																						raise
																																		else:
																																						# Increase the block size in case it was too small.
																																						# The block size will be reset for the next file.
																																						logger.debug(
																																						    F'''Batch of {len(UpperCamelCase__       )} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'''       )
																																						block_size *= 2
																						except pa.ArrowInvalid as e:
																										try:
																														with open(
																														    UpperCamelCase__ ,    encoding=self.config.encoding ,    errors=self.config.encoding_errors       ) as f:
																																		SCREAMING_SNAKE_CASE_   :	Optional[int]													=   json.load(UpperCamelCase__       )
																										except json.JSONDecodeError:
																														logger.error(F'''Failed to read file \'{file}\' with error {type(UpperCamelCase__       )}: {e}'''       )
																														raise e
																										# If possible, parse the file as a list of json objects and exit the loop
																										if isinstance(UpperCamelCase__ ,    UpperCamelCase__       ):  # list is the only sequence type supported in JSON
																														try:
																																		SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   set().union(*[row.keys() for row in dataset]       )
																																		SCREAMING_SNAKE_CASE_   :	Dict													=   {col: [row.get(UpperCamelCase__       ) for row in dataset] for col in keys}
																																		SCREAMING_SNAKE_CASE_   :	int													=   pa.Table.from_pydict(UpperCamelCase__       )
																														except (pa.ArrowInvalid, AttributeError) as e:
																																		logger.error(F'''Failed to read file \'{file}\' with error {type(UpperCamelCase__       )}: {e}'''       )
																																		raise ValueError(F'''Not able to read records in the JSON file at {file}.'''       ) from None
																														yield file_idx, self._cast_table(UpperCamelCase__       )
																														break
																										else:
																														logger.error(F'''Failed to read file \'{file}\' with error {type(UpperCamelCase__       )}: {e}'''       )
																														raise ValueError(
																														    F'''Not able to read records in the JSON file at {file}. '''
																														    F'''You should probably indicate the field of the JSON file containing your records. '''
																														    F'''This JSON file contain the following fields: {str(list(dataset.keys()       )       )}. '''
																														    F'''Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '''       ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																						yield (file_idx, batch_idx), self._cast_table(UpperCamelCase__       )
																						batch_idx += 1







import importlib
import inspect
import os
import re


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
A        =			'src/transformers'


# This is to make sure the transformers module imported is the one in the repo.
A        =			importlib.util.spec_from_file_location(
    'transformers',
    os.path.join(PATH_TO_TRANSFORMERS, '__init__.py'),
    submodule_search_locations=[PATH_TO_TRANSFORMERS],
)
A        =			spec.loader.load_module()

A        =			transformers.models.auto.configuration_auto.CONFIG_MAPPING

# Regex pattern used to find the checkpoint mentioned in the docstring of `config_class`.
# For example, `[bert-base-uncased](https://huggingface.co/bert-base-uncased)`
A        =			re.compile('\[(.+?)\]\((https://huggingface\.co/.+?)\)')


A        =			{
    'CLIPConfigMixin',
    'DecisionTransformerConfigMixin',
    'EncoderDecoderConfigMixin',
    'RagConfigMixin',
    'SpeechEncoderDecoderConfigMixin',
    'VisionEncoderDecoderConfigMixin',
    'VisionTextDualEncoderConfigMixin',
}



def 					_lowerCamelCase(			):





				'''simple docstring'''







				SCREAMING_SNAKE_CASE_   :	Optional[int]													=   []

				for config_class in list(CONFIG_MAPPING.values()      ):
								SCREAMING_SNAKE_CASE_   :	int													=   False

								# source code of `config_class`
								SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   inspect.getsource(lowerCAmelCase__      )
								SCREAMING_SNAKE_CASE_   :	List[Any]													=   _re_checkpoint.findall(lowerCAmelCase__      )

								for checkpoint in checkpoints:
												# Each `checkpoint` is a tuple of a checkpoint name and a checkpoint link.
												# For example, `('bert-base-uncased', 'https://huggingface.co/bert-base-uncased')`
												SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_   :	Tuple													=   checkpoint

												# verify the checkpoint name corresponds to the checkpoint link
												SCREAMING_SNAKE_CASE_   :	List[str]													=   F'''https://huggingface.co/{ckpt_name}'''
												if ckpt_link == ckpt_link_from_name:
																SCREAMING_SNAKE_CASE_   :	Optional[int]													=   True
																break

								SCREAMING_SNAKE_CASE_   :	Tuple													=   config_class.__name__
								if not checkpoint_found and name not in CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK:
												configs_without_checkpoint.append(lowerCAmelCase__      )

				if len(lowerCAmelCase__      ) > 0:
								SCREAMING_SNAKE_CASE_   :	str													=   '\n'.join(sorted(lowerCAmelCase__      )      )
								raise ValueError(F'''The following configurations don\'t contain any valid checkpoint:\n{message}'''      )


if __name__ == "__main__":
				check_config_docstrings_have_checkpoints()