






import argparse
import collections
import os
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_table.py
lowerCamelCase__							=     """src/transformers"""
lowerCamelCase__							=     """docs/source/en"""
lowerCamelCase__							=     """."""






def   UpperCamelCase  (       snake_case__     :   Any	,snake_case__     :   Optional[Any]	,snake_case__     :   Tuple  ):




     '''simple docstring'''






     with open(snake_case__	,"""r"""	,encoding="""utf-8"""	,newline="""\n"""  ) as f:
          __snake_case			:Union[str, Any]          =						f.readlines()
     # Find the start prompt.
     __snake_case			:Optional[int]          =						0
     while not lines[start_index].startswith(snake_case__  ):
          start_index += 1
     start_index += 1

     __snake_case			:Tuple          =						start_index
     while not lines[end_index].startswith(snake_case__  ):
          end_index += 1
     end_index -= 1

     while len(lines[start_index]  ) <= 1:
          start_index += 1
     while len(lines[end_index]  ) <= 1:
          end_index -= 1
     end_index += 1
     return "".join(lines[start_index:end_index]  ), start_index, end_index, lines


# Add here suffixes that are used to identify models, separated by |
lowerCamelCase__							=     """Model|Encoder|Decoder|ForConditionalGeneration"""
# Regexes that match TF/Flax/PT model names.
lowerCamelCase__							=     re.compile(R"""TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
lowerCamelCase__							=     re.compile(R"""Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
lowerCamelCase__							=     re.compile(R"""(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")


# This is to make sure the transformers module imported is the one in the repo.
lowerCamelCase__							=     direct_transformers_import(TRANSFORMERS_PATH)






def   UpperCamelCase  (       snake_case__     :   Tuple  ):




     '''simple docstring'''






     __snake_case			:Optional[Any]          =						re.finditer(""".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)"""	,snake_case__  )
     return [m.group(0  ) for m in matches]






def   UpperCamelCase  (       snake_case__     :   Dict	,snake_case__     :   str  ):




     '''simple docstring'''






     __snake_case			:str          =						2 if text == """✅""" or text == """❌""" else len(snake_case__  )
     __snake_case			:Dict          =						(width - text_length) // 2
     __snake_case			:Tuple          =						width - text_length - left_indent
     return " " * left_indent + text + " " * right_indent






def   UpperCamelCase  (       ):




     '''simple docstring'''






     __snake_case			:Dict          =						transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
     __snake_case			:Union[str, Any]          =						{
         name: config_maping_names[code]
         for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
         if code in config_maping_names
     }
     __snake_case			:str          =						{name: config.replace("""Config"""	,""""""  ) for name, config in model_name_to_config.items()}

     # Dictionaries flagging if each model prefix has a slow/fast tokenizer, backend in PT/TF/Flax.
     __snake_case			:Optional[Any]          =						collections.defaultdict(snake_case__  )
     __snake_case			:Dict          =						collections.defaultdict(snake_case__  )
     __snake_case			:Dict          =						collections.defaultdict(snake_case__  )
     __snake_case			:Any          =						collections.defaultdict(snake_case__  )
     __snake_case			:str          =						collections.defaultdict(snake_case__  )

     # Let's lookup through all transformers object (once).
     for attr_name in dir(snake_case__  ):
          __snake_case			:int          =						None
          if attr_name.endswith("""Tokenizer"""  ):
               __snake_case			:Optional[Any]          =						slow_tokenizers
               __snake_case			:List[str]          =						attr_name[:-9]
          elif attr_name.endswith("""TokenizerFast"""  ):
               __snake_case			:Tuple          =						fast_tokenizers
               __snake_case			:Any          =						attr_name[:-13]
          elif _re_tf_models.match(snake_case__  ) is not None:
               __snake_case			:str          =						tf_models
               __snake_case			:List[str]          =						_re_tf_models.match(snake_case__  ).groups()[0]
          elif _re_flax_models.match(snake_case__  ) is not None:
               __snake_case			:List[Any]          =						flax_models
               __snake_case			:Dict          =						_re_flax_models.match(snake_case__  ).groups()[0]
          elif _re_pt_models.match(snake_case__  ) is not None:
               __snake_case			:Dict          =						pt_models
               __snake_case			:Any          =						_re_pt_models.match(snake_case__  ).groups()[0]

          if lookup_dict is not None:
               while len(snake_case__  ) > 0:
                    if attr_name in model_name_to_prefix.values():
                         __snake_case			:Tuple          =						True
                         break
                    # Try again after removing the last word in the name
                    __snake_case			:Union[str, Any]          =						"""""".join(camel_case_split(snake_case__  )[:-1]  )

    # Let's build that table!
     __snake_case			:int          =						list(model_name_to_config.keys()  )
     model_names.sort(key=str.lower  )
     __snake_case			:List[Any]          =						["""Model""", """Tokenizer slow""", """Tokenizer fast""", """PyTorch support""", """TensorFlow support""", """Flax Support"""]
     # We'll need widths to properly display everything in the center (+2 is to leave one extra space on each side).
     __snake_case			:List[str]          =						[len(snake_case__  ) + 2 for c in columns]
     __snake_case			:Optional[Any]          =						max([len(snake_case__  ) for name in model_names]  ) + 2

     # Build the table per se
     __snake_case			:Optional[int]          =						"""|""" + """|""".join([_center_text(snake_case__	,snake_case__  ) for c, w in zip(snake_case__	,snake_case__  )]  ) + """|\n"""
     # Use ":-----:" format to center-aligned table cell texts
     table += "|" + "|".join([""":""" + """-""" * (w - 2) + """:""" for w in widths]  ) + "|\n"

     __snake_case			:int          =						{True: """✅""", False: """❌"""}
     for name in model_names:
          __snake_case			:str          =						model_name_to_prefix[name]
          __snake_case			:List[str]          =						[
              name,
              check[slow_tokenizers[prefix]],
              check[fast_tokenizers[prefix]],
              check[pt_models[prefix]],
              check[tf_models[prefix]],
              check[flax_models[prefix]],
          ]
          table += "|" + "|".join([_center_text(snake_case__	,snake_case__  ) for l, w in zip(snake_case__	,snake_case__  )]  ) + "|\n"
     return table






def   UpperCamelCase  (       snake_case__     :   Any=False  ):




     '''simple docstring'''






     __snake_case       , __snake_case       , __snake_case       , __snake_case			:Tuple          =						_find_text_in_file(
         filename=os.path.join(snake_case__	,"""index.md"""  )	,start_prompt="""<!--This table is updated automatically from the auto modules"""	,end_prompt="""<!-- End table-->"""	,)
     __snake_case			:Union[str, Any]          =						get_model_table_from_auto_modules()

     if current_table != new_table:
          if overwrite:
               with open(os.path.join(snake_case__	,"""index.md"""  )	,"""w"""	,encoding="""utf-8"""	,newline="""\n"""  ) as f:
                    f.writelines(lines[:start_index] + [new_table] + lines[end_index:]  )
          else:
               raise ValueError(
                   """The model table in the `index.md` has not been updated. Run `make fix-copies` to fix this."""  )


if __name__ == "__main__":
     lowerCamelCase__							=     argparse.ArgumentParser()
     parser.add_argument("""--fix_and_overwrite""", action="""store_true""", help="""Whether to fix inconsistencies.""")
     lowerCamelCase__							=     parser.parse_args()

     check_model_table(args.fix_and_overwrite)










import importlib
import json
import os
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
import transformers.models.auto
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.bert.configuration_bert import BertConfig
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402


lowerCamelCase__							=     get_tests_dir("""fixtures/dummy-config.json""")




class 			snake_case__   (							unittest.TestCase):

       '''simple docstring'''



       def 					__lowercase						(							self    ) ->      Union[str, Any]:






            '''simple docstring'''





            __snake_case			:Optional[int]          =						0



       def 					__lowercase						(							self    ) ->      int:






            '''simple docstring'''





            self.assertIsNotNone(transformers.models.auto.__spec__    )
            self.assertIsNotNone(importlib.util.find_spec("""transformers.models.auto"""    )    )



       def 					__lowercase						(							self    ) ->      List[str]:






            '''simple docstring'''





            __snake_case			:Optional[Any]          =						AutoConfig.from_pretrained("""bert-base-uncased"""    )
            self.assertIsInstance(a__			,						a__    )



       def 					__lowercase						(							self    ) ->      Optional[int]:






            '''simple docstring'''





            __snake_case			:List[str]          =						AutoConfig.from_pretrained(a__    )
            self.assertIsInstance(a__			,						a__    )



       def 					__lowercase						(							self    ) ->      List[Any]:






            '''simple docstring'''





            __snake_case			:Tuple          =						AutoConfig.from_pretrained(a__    )
            self.assertIsInstance(a__			,						a__    )



       def 					__lowercase						(							self    ) ->      str:






            '''simple docstring'''





            __snake_case			:Optional[int]          =						AutoConfig.for_model("""roberta"""    )
            self.assertIsInstance(a__			,						a__    )



       def 					__lowercase						(							self    ) ->      List[Any]:






            '''simple docstring'''





            with tempfile.TemporaryDirectory() as tmp_dir:
                 # This model name contains bert and roberta, but roberta ends up being picked.
                 __snake_case			:List[Any]          =						os.path.join(a__			,						"""fake-roberta"""    )
                 os.makedirs(a__			,						exist_ok=a__    )
                 with open(os.path.join(a__			,						"""config.json"""    )			,						"""w"""    ) as f:
                      f.write(json.dumps({}    )    )
                 __snake_case			:Optional[Any]          =						AutoConfig.from_pretrained(a__    )
                 self.assertEqual(type(a__    )			,						a__    )



       def 					__lowercase						(							self    ) ->      Dict:






            '''simple docstring'''





            try:
                 AutoConfig.register("""custom"""			,						a__    )
                 # Wrong model type will raise an error
                 with self.assertRaises(a__    ):
                      AutoConfig.register("""model"""			,						a__    )
                 # Trying to register something existing in the Transformers library will raise an error
                 with self.assertRaises(a__    ):
                      AutoConfig.register("""bert"""			,						a__    )

                 # Now that the config is registered, it can be used as any other config with the auto-API
                 __snake_case			:Optional[int]          =						CustomConfig()
                 with tempfile.TemporaryDirectory() as tmp_dir:
                      config.save_pretrained(a__    )
                      __snake_case			:Tuple          =						AutoConfig.from_pretrained(a__    )
                      self.assertIsInstance(a__			,						a__    )

            finally:
                 if "custom" in CONFIG_MAPPING._extra_content:
                      del CONFIG_MAPPING._extra_content["custom"]



       def 					__lowercase						(							self    ) ->      int:






            '''simple docstring'''





            with self.assertRaisesRegex(
                a__			,						"""bert-base is not a local folder and is not a valid model identifier"""    ):
                 __snake_case			:str          =						AutoConfig.from_pretrained("""bert-base"""    )



       def 					__lowercase						(							self    ) ->      str:






            '''simple docstring'''





            with self.assertRaisesRegex(
                a__			,						R"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""    ):
                 __snake_case			:Dict          =						AutoConfig.from_pretrained(a__			,						revision="""aaaaaa"""    )



       def 					__lowercase						(							self    ) ->      Optional[Any]:






            '''simple docstring'''





            with self.assertRaisesRegex(
                a__			,						"""hf-internal-testing/no-config-test-repo does not appear to have a file named config.json."""			,						):
                 __snake_case			:Optional[int]          =						AutoConfig.from_pretrained("""hf-internal-testing/no-config-test-repo"""    )



       def 					__lowercase						(							self    ) ->      int:






            '''simple docstring'''





            with self.assertRaises(a__    ):
                 __snake_case			:Dict          =						AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""    )
            # If remote code is disabled, we can't load this config.
            with self.assertRaises(a__    ):
                 __snake_case			:int          =						AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""			,						trust_remote_code=a__    )

            __snake_case			:Tuple          =						AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""			,						trust_remote_code=a__    )
            self.assertEqual(config.__class__.__name__			,						"""NewModelConfig"""    )

            # Test config can be reloaded.
            with tempfile.TemporaryDirectory() as tmp_dir:
                 config.save_pretrained(a__    )
                 __snake_case			:Union[str, Any]          =						AutoConfig.from_pretrained(a__			,						trust_remote_code=a__    )
            self.assertEqual(reloaded_config.__class__.__name__			,						"""NewModelConfig"""    )



       def 					__lowercase						(							self    ) ->      Dict:






            '''simple docstring'''





            class 			snake_case__   (							lowercase_):

                   '''simple docstring'''
                   lowerCamelCase							:				Optional[Any] 				= "new-model"

            try:
                 AutoConfig.register("""new-model"""			,						a__    )
                 # If remote code is not set, the default is to use local
                 __snake_case			:List[Any]          =						AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""    )
                 self.assertEqual(config.__class__.__name__			,						"""NewModelConfigLocal"""    )

                 # If remote code is disabled, we load the local one.
                 __snake_case			:Tuple          =						AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""			,						trust_remote_code=a__    )
                 self.assertEqual(config.__class__.__name__			,						"""NewModelConfigLocal"""    )

                 # If remote is enabled, we load from the Hub
                 __snake_case			:List[Any]          =						AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""			,						trust_remote_code=a__    )
                 self.assertEqual(config.__class__.__name__			,						"""NewModelConfig"""    )

            finally:
                 if "new-model" in CONFIG_MAPPING._extra_content:
                      del CONFIG_MAPPING._extra_content["new-model"]


