



'''simple docstring'''






import importlib
import inspect
import os
import re


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
lowerCAmelCase:   Any					=		'src/transformers'


# This is to make sure the transformers module imported is the one in the repo.
lowerCAmelCase:   Optional[int]					=		importlib.util.spec_from_file_location(
    'transformers',
    os.path.join(PATH_TO_TRANSFORMERS, '__init__.py'),
    submodule_search_locations=[PATH_TO_TRANSFORMERS],
)
lowerCAmelCase:   Optional[int]					=		spec.loader.load_module()

lowerCAmelCase:   int					=		transformers.models.auto.configuration_auto.CONFIG_MAPPING

# Regex pattern used to find the checkpoint mentioned in the docstring of `config_class`.
# For example, `[bert-base-uncased](https://huggingface.co/bert-base-uncased)`
lowerCAmelCase:   List[str]					=		re.compile('\[(.+?)\]\((https://huggingface\.co/.+?)\)')


lowerCAmelCase:   List[str]					=		{
    'CLIPConfigMixin',
    'DecisionTransformerConfigMixin',
    'EncoderDecoderConfigMixin',
    'RagConfigMixin',
    'SpeechEncoderDecoderConfigMixin',
    'VisionEncoderDecoderConfigMixin',
    'VisionTextDualEncoderConfigMixin',
}
def  lowerCamelCase__			(		):
  a		:   Any	  =	[]

  for config_class in list(CONFIG_MAPPING.values()							):
    a		:   List[Any]	  =	False

    # source code of `config_class`
    a		:   Any	  =	inspect.getsource(_A							)
    a		:   Tuple	  =	_re_checkpoint.findall(_A							)

    for checkpoint in checkpoints:
      # Each `checkpoint` is a tuple of a checkpoint name and a checkpoint link.
      # For example, `('bert-base-uncased', 'https://huggingface.co/bert-base-uncased')`
      a  ,      a		:   Optional[int]	  =	checkpoint

      # verify the checkpoint name corresponds to the checkpoint link
      a		:   Dict	  =	f"""https://huggingface.co/{ckpt_name}"""
      if ckpt_link == ckpt_link_from_name:
        a		:   str	  =	True
        break

    a		:   str	  =	config_class.__name__
    if not checkpoint_found and name not in CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK:
      configs_without_checkpoint.append(_A							)

  if len(_A							) > 0:
    a		:   int	  =	'\n'.join(sorted(_A							)							)
    raise ValueError(f"""The following configurations don't contain any valid checkpoint:\n{message}"""							)


if __name__ == "__main__":
     check_config_docstrings_have_checkpoints()




'''simple docstring'''






from typing import Callable, Optional

from .. import Features
from ..packaged_modules.generator.generator import Generator
from .abc import AbstractDatasetInputStream

class 				a__(   lowerCamelCase__      ):







    def __init__( self					:    int		,     __snake_case					:    Callable		,     __snake_case					:    Optional[Features] = None		,     __snake_case					:    str = None		,     __snake_case					:    bool = False		,     __snake_case					:    bool = False		,     __snake_case					:    Optional[dict] = None		,     __snake_case					:    Optional[int] = None		,     **__snake_case					:    Optional[int]		,     ):
      super().__init__(
          features=__snake_case		,     cache_dir=__snake_case		,     keep_in_memory=__snake_case		,     streaming=__snake_case		,     num_proc=__snake_case		,     **__snake_case		,     )
      a		:   List[Any]	  =	Generator(
          cache_dir=__snake_case		,     features=__snake_case		,     generator=__snake_case		,     gen_kwargs=__snake_case		,     **__snake_case		,     )







    def      lowercase_			( self					:    Any      ):
      # Build iterable dataset
      if self.streaming:
        a		:   Optional[int]	  =	self.builder.as_streaming_dataset(split='train'      )
      # Build regular (map-style) dataset
      else:
        a		:   List[Any]	  =	None
        a		:   Any	  =	None
        a		:   Optional[int]	  =	None
        a		:   List[str]	  =	None

        self.builder.download_and_prepare(
            download_config=__snake_case		,     download_mode=__snake_case		,     verification_mode=__snake_case		,     base_path=__snake_case		,     num_proc=self.num_proc		,     )
        a		:   Dict	  =	self.builder.as_dataset(
            split='train'		,     verification_mode=__snake_case		,     in_memory=self.keep_in_memory      )
      return dataset