



'''simple docstring'''





import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


UpperCAmelCase    					=   datasets.utils.logging.get_logger(__name__)

@dataclass
class 						a   (				datasets.BuilderConfig    ):
				_snake_case		    = None
				_snake_case		    = "utf-8"
				_snake_case		    = None
				_snake_case		    = None
				_snake_case		    = True  # deprecated
				_snake_case		    = None  # deprecated
				_snake_case		    = 10 << 20  # 10MB
				_snake_case		    = None




class 						a   (				datasets.ArrowBasedBuilder    ):
				_snake_case		    = JsonConfig




				def 	__snake_case							(							self	:				Optional[int]      ):
										if self.config.block_size is not None:
																logger.warning('''The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'''      )
																snake_case       :     Union[str, Any] =    self.config.block_size
										if self.config.use_threads is not True:
																logger.warning(
																    '''The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'''      )
										if self.config.newlines_in_values is not None:
																raise ValueError('''The JSON loader parameter `newlines_in_values` is no longer supported'''      )
										return datasets.DatasetInfo(features=self.config.features      )




				def 	__snake_case							(							self	:				List[Any],    SCREAMING_SNAKE_CASE_	:				List[Any]      ):
										if not self.config.data_files:
																raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""      )
										snake_case       :     str =    dl_manager.download_and_extract(self.config.data_files      )
										if isinstance(UpperCamelCase__,    (str, list, tuple)      ):
																snake_case       :     List[str] =    data_files
																if isinstance(UpperCamelCase__,    UpperCamelCase__      ):
																						snake_case       :     Tuple =    [files]
																snake_case       :     int =    [dl_manager.iter_files(UpperCamelCase__      ) for file in files]
																return [datasets.SplitGenerator(name=datasets.Split.TRAIN,    gen_kwargs={'''files''': files}      )]
										snake_case       :     str =    []
										for split_name, files in data_files.items():
																if isinstance(UpperCamelCase__,    UpperCamelCase__      ):
																						snake_case       :     Any =    [files]
																snake_case       :     Union[str, Any] =    [dl_manager.iter_files(UpperCamelCase__      ) for file in files]
																splits.append(datasets.SplitGenerator(name=UpperCamelCase__,    gen_kwargs={'''files''': files}      )      )
										return splits




				def 	__snake_case							(							self	:				List[str],    SCREAMING_SNAKE_CASE_	:				Any      ):
										if self.config.features is not None:
																# adding missing columns
																for column_name in set(self.config.features      ) - set(pa_table.column_names      ):
																						snake_case       :     str =    self.config.features.arrow_schema.field(UpperCamelCase__      ).type
																						snake_case       :     Any =    pa_table.append_column(UpperCamelCase__,    pa.array([None] * len(UpperCamelCase__      ),    type=UpperCamelCase__      )      )
																# more expensive cast to support nested structures with keys in a different order
																# allows str <-> int/float or str to Audio for example
																snake_case       :     Dict =    table_cast(UpperCamelCase__,    self.config.features.arrow_schema      )
										return pa_table




				def 	__snake_case							(							self	:				Tuple,    SCREAMING_SNAKE_CASE_	:				int      ):
										for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCamelCase__      )      ):
																# If the file is one json object and if we need to look at the list of items in one specific field
																if self.config.field is not None:
																						with open(UpperCamelCase__,    encoding=self.config.encoding,    errors=self.config.encoding_errors      ) as f:
																												snake_case       :     List[str] =    json.load(UpperCamelCase__      )

																						# We keep only the field we are interested in
																						snake_case       :     Tuple =    dataset[self.config.field]

																						# We accept two format: a list of dicts or a dict of lists
																						if isinstance(UpperCamelCase__,    (list, tuple)      ):
																												snake_case       :     Tuple =    set().union(*[row.keys() for row in dataset]      )
																												snake_case       :     List[Any] =    {col: [row.get(UpperCamelCase__      ) for row in dataset] for col in keys}
																						else:
																												snake_case       :     str =    dataset
																						snake_case       :     Optional[int] =    pa.Table.from_pydict(UpperCamelCase__      )
																						yield file_idx, self._cast_table(UpperCamelCase__      )

																# If the file has one json object per line
																else:
																						with open(UpperCamelCase__,    '''rb'''      ) as f:
																												snake_case       :     List[str] =    0
																												# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																												# Set a default minimum value of 16kB if the chunk size is really small
																												snake_case       :     Any =    max(self.config.chunksize // 32,    16 << 10      )
																												snake_case       :     Optional[int] =    (
																												    self.config.encoding_errors if self.config.encoding_errors is not None else 'strict'
																												)
																												while True:
																																		snake_case       :     List[str] =    f.read(self.config.chunksize      )
																																		if not batch:
																																								break
																																		# Finish current line
																																		try:
																																								batch += f.readline()
																																		except (AttributeError, io.UnsupportedOperation):
																																								batch += readline(UpperCamelCase__      )
																																		# PyArrow only accepts utf-8 encoded bytes
																																		if self.config.encoding != "utf-8":
																																								snake_case       :     List[Any] =    batch.decode(self.config.encoding,    errors=UpperCamelCase__      ).encode('''utf-8'''      )
																																		try:
																																								while True:
																																														try:
																																																				snake_case       :     Dict =    paj.read_json(
																																																				    io.BytesIO(UpperCamelCase__      ),    read_options=paj.ReadOptions(block_size=UpperCamelCase__      )      )
																																																				break
																																														except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																																				if (
																																																				    isinstance(UpperCamelCase__,    pa.ArrowInvalid      )
																																																				    and "straddling" not in str(UpperCamelCase__      )
																																																				    or block_size > len(UpperCamelCase__      )
																																																				):
																																																										raise
																																																				else:
																																																										# Increase the block size in case it was too small.
																																																										# The block size will be reset for the next file.
																																																										logger.debug(
																																																										    F"""Batch of {len(UpperCamelCase__      )} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."""      )
																																																										block_size *= 2
																																		except pa.ArrowInvalid as e:
																																								try:
																																														with open(
																																														    UpperCamelCase__,    encoding=self.config.encoding,    errors=self.config.encoding_errors      ) as f:
																																																				snake_case       :     Optional[int] =    json.load(UpperCamelCase__      )
																																								except json.JSONDecodeError:
																																														logger.error(F"""Failed to read file \'{file}\' with error {type(UpperCamelCase__      )}: {e}"""      )
																																														raise e
																																								# If possible, parse the file as a list of json objects and exit the loop
																																								if isinstance(UpperCamelCase__,    UpperCamelCase__      ):  # list is the only sequence type supported in JSON
																																														try:
																																																				snake_case       :     Union[str, Any] =    set().union(*[row.keys() for row in dataset]      )
																																																				snake_case       :     Dict =    {col: [row.get(UpperCamelCase__      ) for row in dataset] for col in keys}
																																																				snake_case       :     int =    pa.Table.from_pydict(UpperCamelCase__      )
																																														except (pa.ArrowInvalid, AttributeError) as e:
																																																				logger.error(F"""Failed to read file \'{file}\' with error {type(UpperCamelCase__      )}: {e}"""      )
																																																				raise ValueError(F"""Not able to read records in the JSON file at {file}."""      ) from None
																																														yield file_idx, self._cast_table(UpperCamelCase__      )
																																														break
																																								else:
																																														logger.error(F"""Failed to read file \'{file}\' with error {type(UpperCamelCase__      )}: {e}"""      )
																																														raise ValueError(
																																														    F"""Not able to read records in the JSON file at {file}. """
																																														    F"""You should probably indicate the field of the JSON file containing your records. """
																																														    F"""This JSON file contain the following fields: {str(list(dataset.keys()      )      )}. """
																																														    F"""Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. """      ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																		yield (file_idx, batch_idx), self._cast_table(UpperCamelCase__      )
																																		batch_idx += 1



'''simple docstring'''





import inspect
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
UpperCAmelCase    					=   "src/transformers"


# This is to make sure the transformers module imported is the one in the repo.
UpperCAmelCase    					=   direct_transformers_import(PATH_TO_TRANSFORMERS)

UpperCAmelCase    					=   transformers.models.auto.configuration_auto.CONFIG_MAPPING

# Regex pattern used to find the checkpoint mentioned in the docstring of `config_class`.
# For example, `[bert-base-uncased](https://huggingface.co/bert-base-uncased)`
UpperCAmelCase    					=   re.compile(r"\[(.+?)\]\((https://huggingface\.co/.+?)\)")


UpperCAmelCase    					=   {
    "DecisionTransformerConfig",
    "EncoderDecoderConfig",
    "MusicgenConfig",
    "RagConfig",
    "SpeechEncoderDecoderConfig",
    "TimmBackboneConfig",
    "VisionEncoderDecoderConfig",
    "VisionTextDualEncoderConfig",
    "LlamaConfig",
}

def      A (       A_							:					Union[str, Any]       ):
						snake_case       :     Tuple =    None

						# source code of `config_class`
						snake_case       :     Union[str, Any] =    inspect.getsource(A_       )
						snake_case       :     List[Any] =    _re_checkpoint.findall(A_       )

						# Each `checkpoint` is a tuple of a checkpoint name and a checkpoint link.
						# For example, `('bert-base-uncased', 'https://huggingface.co/bert-base-uncased')`
						for ckpt_name, ckpt_link in checkpoints:
												# allow the link to end with `/`
												if ckpt_link.endswith('''/'''       ):
																		snake_case       :     Optional[int] =    ckpt_link[:-1]

												# verify the checkpoint name corresponds to the checkpoint link
												snake_case       :     Dict =    F"""https://huggingface.co/{ckpt_name}"""
												if ckpt_link == ckpt_link_from_name:
																		snake_case       :     Dict =    ckpt_name
																		break

						return checkpoint

def      A (       ):
						snake_case       :     List[str] =    []

						for config_class in list(CONFIG_MAPPING.values()       ):
												# Skip deprecated models
												if "models.deprecated" in config_class.__module__:
																		continue
												snake_case       :     Tuple =    get_checkpoint_from_config_class(A_       )

												snake_case       :     Optional[Any] =    config_class.__name__
												if checkpoint is None and name not in CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK:
																		configs_without_checkpoint.append(A_       )

						if len(A_       ) > 0:
												snake_case       :     Optional[int] =    '''\n'''.join(sorted(A_       )       )
												raise ValueError(F"""The following configurations don't contain any valid checkpoint:\n{message}"""       )


if __name__ == "__main__":
						check_config_docstrings_have_checkpoints()

