





import argparse
import collections
import os
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_table.py
UpperCamelCase__  			="""src/transformers"""
UpperCamelCase__  			="""docs/source/en"""
UpperCamelCase__  			="""."""






def      lowerCamelCase__      (__lowerCamelCase,							__lowerCamelCase,							__lowerCamelCase						):


     with open(_A,							"r",							encoding="utf-8",							newline="\n"						) as f:
          _SCREAMING_SNAKE_CASE							:						str         =				f.readlines()
     # Find the start prompt.
     _SCREAMING_SNAKE_CASE							:						Union[str, Any]         =				0
     while not lines[start_index].startswith(_A						):
          start_index += 1
     start_index += 1

     _SCREAMING_SNAKE_CASE							:						Union[str, Any]         =				start_index
     while not lines[end_index].startswith(_A						):
          end_index += 1
     end_index -= 1

     while len(lines[start_index]						) <= 1:
          start_index += 1
     while len(lines[end_index]						) <= 1:
          end_index -= 1
     end_index += 1
     return "".join(lines[start_index:end_index]						), start_index, end_index, lines


# Add here suffixes that are used to identify models, separated by |
UpperCamelCase__  			="""Model|Encoder|Decoder|ForConditionalGeneration"""
# Regexes that match TF/Flax/PT model names.
UpperCamelCase__  			=re.compile(R'TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
UpperCamelCase__  			=re.compile(R'Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
UpperCamelCase__  			=re.compile(R'(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')


# This is to make sure the transformers module imported is the one in the repo.
UpperCamelCase__  			=direct_transformers_import(TRANSFORMERS_PATH)






def      lowerCamelCase__      (__lowerCamelCase						):
     _SCREAMING_SNAKE_CASE							:						Optional[Any]         =				re.finditer(".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)",							_A						)
     return [m.group(0						) for m in matches]






def      lowerCamelCase__      (__lowerCamelCase,							__lowerCamelCase						):
     _SCREAMING_SNAKE_CASE							:						int         =				2 if text == "✅" or text == "❌" else len(_A						)
     _SCREAMING_SNAKE_CASE							:						Union[str, Any]         =				(width - text_length) // 2
     _SCREAMING_SNAKE_CASE							:						Optional[Any]         =				width - text_length - left_indent
     return " " * left_indent + text + " " * right_indent






def      lowerCamelCase__      ():
     _SCREAMING_SNAKE_CASE							:						Union[str, Any]         =				transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
     _SCREAMING_SNAKE_CASE							:						Tuple         =				{
         name: config_maping_names[code]
         for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
         if code in config_maping_names
     }
     _SCREAMING_SNAKE_CASE							:						Dict         =				{name: config.replace("Config",							""						) for name, config in model_name_to_config.items()}

     # Dictionaries flagging if each model prefix has a slow/fast tokenizer, backend in PT/TF/Flax.
     _SCREAMING_SNAKE_CASE							:						str         =				collections.defaultdict(_A						)
     _SCREAMING_SNAKE_CASE							:						List[Any]         =				collections.defaultdict(_A						)
     _SCREAMING_SNAKE_CASE							:						str         =				collections.defaultdict(_A						)
     _SCREAMING_SNAKE_CASE							:						List[Any]         =				collections.defaultdict(_A						)
     _SCREAMING_SNAKE_CASE							:						Tuple         =				collections.defaultdict(_A						)

     # Let's lookup through all transformers object (once).
     for attr_name in dir(_A						):
          _SCREAMING_SNAKE_CASE							:						Union[str, Any]         =				None
          if attr_name.endswith("Tokenizer"						):
               _SCREAMING_SNAKE_CASE							:						Dict         =				slow_tokenizers
               _SCREAMING_SNAKE_CASE							:						Tuple         =				attr_name[:-9]
          elif attr_name.endswith("TokenizerFast"						):
               _SCREAMING_SNAKE_CASE							:						str         =				fast_tokenizers
               _SCREAMING_SNAKE_CASE							:						Any         =				attr_name[:-13]
          elif _re_tf_models.match(_A						) is not None:
               _SCREAMING_SNAKE_CASE							:						str         =				tf_models
               _SCREAMING_SNAKE_CASE							:						Optional[Any]         =				_re_tf_models.match(_A						).groups()[0]
          elif _re_flax_models.match(_A						) is not None:
               _SCREAMING_SNAKE_CASE							:						int         =				flax_models
               _SCREAMING_SNAKE_CASE							:						Optional[int]         =				_re_flax_models.match(_A						).groups()[0]
          elif _re_pt_models.match(_A						) is not None:
               _SCREAMING_SNAKE_CASE							:						List[Any]         =				pt_models
               _SCREAMING_SNAKE_CASE							:						List[Any]         =				_re_pt_models.match(_A						).groups()[0]

          if lookup_dict is not None:
               while len(_A						) > 0:
                    if attr_name in model_name_to_prefix.values():
                         _SCREAMING_SNAKE_CASE							:						str         =				True
                         break
                    # Try again after removing the last word in the name
                    _SCREAMING_SNAKE_CASE							:						List[str]         =				"".join(camel_case_split(_A						)[:-1]						)

                    # Let's build that table!
     _SCREAMING_SNAKE_CASE							:						Optional[int]         =				list(model_name_to_config.keys()						)
     model_names.sort(key=str.lower						)
     _SCREAMING_SNAKE_CASE							:						Union[str, Any]         =				["Model", "Tokenizer slow", "Tokenizer fast", "PyTorch support", "TensorFlow support", "Flax Support"]
     # We'll need widths to properly display everything in the center (+2 is to leave one extra space on each side).
     _SCREAMING_SNAKE_CASE							:						Optional[Any]         =				[len(_A						) + 2 for c in columns]
     _SCREAMING_SNAKE_CASE							:						Optional[Any]         =				max([len(_A						) for name in model_names]						) + 2

     # Build the table per se
     _SCREAMING_SNAKE_CASE							:						List[Any]         =				"|" + "|".join([_center_text(_A,							_A						) for c, w in zip(_A,							_A						)]						) + "|\n"
     # Use ":-----:" format to center-aligned table cell texts
     table += "|" + "|".join([":" + "-" * (w - 2) + ":" for w in widths]						) + "|\n"

     _SCREAMING_SNAKE_CASE							:						Tuple         =				{True: "✅", False: "❌"}
     for name in model_names:
          _SCREAMING_SNAKE_CASE							:						List[str]         =				model_name_to_prefix[name]
          _SCREAMING_SNAKE_CASE							:						Tuple         =				[
              name,
              check[slow_tokenizers[prefix]],
              check[fast_tokenizers[prefix]],
              check[pt_models[prefix]],
              check[tf_models[prefix]],
              check[flax_models[prefix]],
          ]
          table += "|" + "|".join([_center_text(_A,							_A						) for l, w in zip(_A,							_A						)]						) + "|\n"
     return table






def      lowerCamelCase__      (__lowerCamelCase=False						):
     _SCREAMING_SNAKE_CASE     ,      _SCREAMING_SNAKE_CASE     ,      _SCREAMING_SNAKE_CASE     ,      _SCREAMING_SNAKE_CASE							:						str         =				_find_text_in_file(
         filename=os.path.join(_A,							"index.md"						),							start_prompt="<!--This table is updated automatically from the auto modules",							end_prompt="<!-- End table-->",							)
     _SCREAMING_SNAKE_CASE							:						Optional[int]         =				get_model_table_from_auto_modules()

     if current_table != new_table:
          if overwrite:
               with open(os.path.join(_A,							"index.md"						),							"w",							encoding="utf-8",							newline="\n"						) as f:
                    f.writelines(lines[:start_index] + [new_table] + lines[end_index:]						)
          else:
               raise ValueError(
                   "The model table in the `index.md` has not been updated. Run `make fix-copies` to fix this."						)


if __name__ == "__main__":
       UpperCamelCase__  			=argparse.ArgumentParser()
       parser.add_argument('--fix_and_overwrite', action='store_true', help='Whether to fix inconsistencies.')
       UpperCamelCase__  			=parser.parse_args()

       check_model_table(args.fix_and_overwrite)
from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE			:						int				  =		logging.get_logger(__name__)

SCREAMING_SNAKE_CASE			:						Dict				  =		{
    """google/canine-s""": """https://huggingface.co/google/canine-s/resolve/main/config.json""",
    # See all CANINE models at https://huggingface.co/models?filter=canine
}
class  A_			(							a_			):
  _SCREAMING_SNAKE_CASE     					=      """canine"""

  def __init__(		self			:   int  ,				__SCREAMING_SNAKE_CASE			:   List[str]=7_68  ,				__SCREAMING_SNAKE_CASE			:   Any=12  ,				__SCREAMING_SNAKE_CASE			:   str=12  ,				__SCREAMING_SNAKE_CASE			:   int=30_72  ,				__SCREAMING_SNAKE_CASE			:   int="gelu"  ,				__SCREAMING_SNAKE_CASE			:   Optional[int]=0.1  ,				__SCREAMING_SNAKE_CASE			:   List[str]=0.1  ,				__SCREAMING_SNAKE_CASE			:   List[Any]=1_63_84  ,				__SCREAMING_SNAKE_CASE			:   List[Any]=16  ,				__SCREAMING_SNAKE_CASE			:   int=0.02  ,				__SCREAMING_SNAKE_CASE			:   Optional[Any]=1E-12  ,				__SCREAMING_SNAKE_CASE			:   Union[str, Any]=0  ,				__SCREAMING_SNAKE_CASE			:   List[str]=0xE000  ,				__SCREAMING_SNAKE_CASE			:   int=0xE001  ,				__SCREAMING_SNAKE_CASE			:   Optional[int]=4  ,				__SCREAMING_SNAKE_CASE			:   Optional[Any]=4  ,				__SCREAMING_SNAKE_CASE			:   List[Any]=8  ,				__SCREAMING_SNAKE_CASE			:   List[str]=1_63_84  ,				__SCREAMING_SNAKE_CASE			:   Optional[Any]=1_28  ,				**__SCREAMING_SNAKE_CASE			:   List[str]  ,				):
   super().__init__(pad_token_id=__SCREAMING_SNAKE_CASE  ,				bos_token_id=__SCREAMING_SNAKE_CASE  ,				eos_token_id=__SCREAMING_SNAKE_CASE  ,				**__SCREAMING_SNAKE_CASE							)

   __a							=	max_position_embeddings
   __a							=	hidden_size
   __a							=	num_hidden_layers
   __a							=	num_attention_heads
   __a							=	intermediate_size
   __a							=	hidden_act
   __a							=	hidden_dropout_prob
   __a							=	attention_probs_dropout_prob
   __a							=	initializer_range
   __a							=	type_vocab_size
   __a							=	layer_norm_eps

   # Character config:
   __a							=	downsampling_rate
   __a							=	upsampling_kernel_size
   __a							=	num_hash_functions
   __a							=	num_hash_buckets
   __a							=	local_transformer_stride


