






import collections
import importlib.util
import os
import re
from pathlib import Path


SCREAMING_SNAKE_CASE 							=						'src/transformers'


# Matches is_xxx_available()
SCREAMING_SNAKE_CASE 							=						re.compile(R'is\_([a-z_]*)_available()')
# Catches a one-line _import_struct = {xxx}
SCREAMING_SNAKE_CASE 							=						re.compile(R'^_import_structure\s+=\s+\{([^\}]+)\}')
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
SCREAMING_SNAKE_CASE 							=						re.compile(R'\s+"\S*":\s+\[([^\]]*)\]')
# Catches a line if not is_foo_available
SCREAMING_SNAKE_CASE 							=						re.compile(R'^\s*if\s+not\s+is\_[a-z_]*\_available\(\)')
# Catches a line _import_struct["bla"].append("foo")
SCREAMING_SNAKE_CASE 							=						re.compile(R'^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)')
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
SCREAMING_SNAKE_CASE 							=						re.compile(R'^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]')
# Catches a line with an object between quotes and a comma:     "MyModel",
SCREAMING_SNAKE_CASE 							=						re.compile('^\s+"([^"]+)",')
# Catches a line with objects between brackets only:    ["foo", "bar"],
SCREAMING_SNAKE_CASE 							=						re.compile('^\s+\[([^\]]+)\]')
# Catches a line with from foo import bar, bla, boo
SCREAMING_SNAKE_CASE 							=						re.compile(R'\s+from\s+\S*\s+import\s+([^\(\s].*)\n')
# Catches a line with try:
SCREAMING_SNAKE_CASE 							=						re.compile(R'^\s*try:')
# Catches a line with else:
SCREAMING_SNAKE_CASE 							=						re.compile(R'^\s*else:')

def      _lowerCamelCase	(						__A    :							Union[str, Any] )      ->					List[Any]:
    if _re_test_backend.search(__A ) is None:
        return None
    _UpperCAmelCase				:      int			  =				[b[0] for b in _re_backend.findall(__A )]
    backends.sort()
    return "_and_".join(__A )

def      _lowerCamelCase	(						__A    :							List[str] )      ->					Optional[Any]:
    with open(__A		,		'''r'''		,		encoding='''utf-8'''		,		newline='''\n''' ) as f:
        _UpperCAmelCase				:      Optional[int]			  =				f.readlines()

    _UpperCAmelCase				:      Union[str, Any]			  =				0
    while line_index < len(__A ) and not lines[line_index].startswith('''_import_structure = {''' ):
        line_index += 1

    # If this is a traditional init, just return.
    if line_index >= len(__A ):
        return None

    # First grab the objects without a specific backend in _import_structure
    _UpperCAmelCase				:      str			  =				[]
    while not lines[line_index].startswith('''if TYPE_CHECKING''' ) and find_backend(lines[line_index] ) is None:
        _UpperCAmelCase				:      Dict			  =				lines[line_index]
        # If we have everything on a single line, let's deal with it.
        if _re_one_line_import_struct.search(__A ):
            _UpperCAmelCase				:      Dict			  =				_re_one_line_import_struct.search(__A ).groups()[0]
            _UpperCAmelCase				:      List[Any]			  =				re.findall('''\[([^\]]+)\]'''		,		__A )
            for imp in imports:
                objects.extend([obj[1:-1] for obj in imp.split(''', ''' )] )
            line_index += 1
            continue
        _UpperCAmelCase				:      Optional[Any]			  =				_re_import_struct_key_value.search(__A )
        if single_line_import_search is not None:
            _UpperCAmelCase				:      int			  =				[obj[1:-1] for obj in single_line_import_search.groups()[0].split(''', ''' ) if len(__A ) > 0]
            objects.extend(__A )
        elif line.startswith(''' ''' * 8 + '''"''' ):
            objects.append(line[9:-3] )
        line_index += 1

    _UpperCAmelCase				:      str			  =				{'''none''': objects}
    # Let's continue with backend-specific objects in _import_structure
    while not lines[line_index].startswith('''if TYPE_CHECKING''' ):
        # If the line is an if not is_backend_available, we grab all objects associated.
        _UpperCAmelCase				:      Optional[Any]			  =				find_backend(lines[line_index] )
        # Check if the backend declaration is inside a try block:
        if _re_try.search(lines[line_index - 1] ) is None:
            _UpperCAmelCase				:      List[Any]			  =				None

        if backend is not None:
            line_index += 1

            # Scroll until we hit the else block of try-except-else
            while _re_else.search(lines[line_index] ) is None:
                line_index += 1

            line_index += 1

            _UpperCAmelCase				:      List[str]			  =				[]
            # Until we unindent, add backend objects to the list
            while len(lines[line_index] ) <= 1 or lines[line_index].startswith(''' ''' * 4 ):
                _UpperCAmelCase				:      List[str]			  =				lines[line_index]
                if _re_import_struct_add_one.search(__A ) is not None:
                    objects.append(_re_import_struct_add_one.search(__A ).groups()[0] )
                elif _re_import_struct_add_many.search(__A ) is not None:
                    _UpperCAmelCase				:      Dict			  =				_re_import_struct_add_many.search(__A ).groups()[0].split(''', ''' )
                    _UpperCAmelCase				:      Any			  =				[obj[1:-1] for obj in imports if len(__A ) > 0]
                    objects.extend(__A )
                elif _re_between_brackets.search(__A ) is not None:
                    _UpperCAmelCase				:      Optional[int]			  =				_re_between_brackets.search(__A ).groups()[0].split(''', ''' )
                    _UpperCAmelCase				:      Optional[int]			  =				[obj[1:-1] for obj in imports if len(__A ) > 0]
                    objects.extend(__A )
                elif _re_quote_object.search(__A ) is not None:
                    objects.append(_re_quote_object.search(__A ).groups()[0] )
                elif line.startswith(''' ''' * 8 + '''"''' ):
                    objects.append(line[9:-3] )
                elif line.startswith(''' ''' * 12 + '''"''' ):
                    objects.append(line[13:-3] )
                line_index += 1

            _UpperCAmelCase				:      Optional[Any]			  =				objects
        else:
            line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
    _UpperCAmelCase				:      Optional[Any]			  =				[]
    while (
        line_index < len(__A )
        and find_backend(lines[line_index] ) is None
        and not lines[line_index].startswith('''else''' )
    ):
        _UpperCAmelCase				:      int			  =				lines[line_index]
        _UpperCAmelCase				:      Tuple			  =				_re_import.search(__A )
        if single_line_import_search is not None:
            objects.extend(single_line_import_search.groups()[0].split(''', ''' ) )
        elif line.startswith(''' ''' * 8 ):
            objects.append(line[8:-2] )
        line_index += 1

    _UpperCAmelCase				:      Tuple			  =				{'''none''': objects}
    # Let's continue with backend-specific objects
    while line_index < len(__A ):
        # If the line is an if is_backend_available, we grab all objects associated.
        _UpperCAmelCase				:      Union[str, Any]			  =				find_backend(lines[line_index] )
        # Check if the backend declaration is inside a try block:
        if _re_try.search(lines[line_index - 1] ) is None:
            _UpperCAmelCase				:      List[str]			  =				None

        if backend is not None:
            line_index += 1

            # Scroll until we hit the else block of try-except-else
            while _re_else.search(lines[line_index] ) is None:
                line_index += 1

            line_index += 1

            _UpperCAmelCase				:      Union[str, Any]			  =				[]
            # Until we unindent, add backend objects to the list
            while len(lines[line_index] ) <= 1 or lines[line_index].startswith(''' ''' * 8 ):
                _UpperCAmelCase				:      List[str]			  =				lines[line_index]
                _UpperCAmelCase				:      Dict			  =				_re_import.search(__A )
                if single_line_import_search is not None:
                    objects.extend(single_line_import_search.groups()[0].split(''', ''' ) )
                elif line.startswith(''' ''' * 12 ):
                    objects.append(line[12:-2] )
                line_index += 1

            _UpperCAmelCase				:      List[str]			  =				objects
        else:
            line_index += 1

    return import_dict_objects, type_hint_objects

def      _lowerCamelCase	(						__A    :							int		,		__A    :							Tuple )      ->					List[Any]:

    def find_duplicates(__A    :							Optional[Any] ):
        return [k for k, v in collections.Counter(__A ).items() if v > 1]

    if list(import_dict_objects.keys() ) != list(type_hint_objects.keys() ):
        return ["Both sides of the init do not have the same backends!"]

    _UpperCAmelCase				:      List[str]			  =				[]
    for key in import_dict_objects.keys():
        _UpperCAmelCase				:      Optional[Any]			  =				find_duplicates(import_dict_objects[key] )
        if duplicate_imports:
            errors.append(f'''Duplicate _import_structure definitions for: {duplicate_imports}''' )
        _UpperCAmelCase				:      Optional[Any]			  =				find_duplicates(type_hint_objects[key] )
        if duplicate_type_hints:
            errors.append(f'''Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}''' )

        if sorted(set(import_dict_objects[key] ) ) != sorted(set(type_hint_objects[key] ) ):
            _UpperCAmelCase				:      int			  =				'''base imports''' if key == '''none''' else f'''{key} backend'''
            errors.append(f'''Differences for {name}:''' )
            for a in type_hint_objects[key]:
                if a not in import_dict_objects[key]:
                    errors.append(f'''  {a} in TYPE_HINT but not in _import_structure.''' )
            for a in import_dict_objects[key]:
                if a not in type_hint_objects[key]:
                    errors.append(f'''  {a} in _import_structure but not in TYPE_HINT.''' )
    return errors

def      _lowerCamelCase	(						)      ->					Optional[Any]:
    _UpperCAmelCase				:      Union[str, Any]			  =				[]
    for root, _, files in os.walk(__A ):
        if "__init__.py" in files:
            _UpperCAmelCase				:      List[Any]			  =				os.path.join(__A		,		'''__init__.py''' )
            _UpperCAmelCase				:      List[Any]			  =				parse_init(__A )
            if objects is not None:
                _UpperCAmelCase				:      Tuple			  =				analyze_results(*__A )
                if len(__A ) > 0:
                    _UpperCAmelCase				:      Optional[int]			  =				f'''Problem in {fname}, both halves do not define the same objects.\n{errors[0]}'''
                    failures.append('''\n'''.join(__A ) )
    if len(__A ) > 0:
        raise ValueError('''\n\n'''.join(__A ) )

def      _lowerCamelCase	(						)      ->					Dict:
    _UpperCAmelCase				:      List[Any]			  =				[]
    for path, directories, files in os.walk(__A ):
        for folder in directories:
            # Ignore private modules
            if folder.startswith('''_''' ):
                directories.remove(__A )
                continue
            # Ignore leftovers from branches (empty folders apart from pycache)
            if len(list((Path(__A ) / folder).glob('''*.py''' ) ) ) == 0:
                continue
            _UpperCAmelCase				:      Any			  =				str((Path(__A ) / folder).relative_to(__A ) )
            _UpperCAmelCase				:      Optional[int]			  =				short_path.replace(os.path.sep		,		'''.''' )
            submodules.append(__A )
        for fname in files:
            if fname == "__init__.py":
                continue
            _UpperCAmelCase				:      int			  =				str((Path(__A ) / fname).relative_to(__A ) )
            _UpperCAmelCase				:      Union[str, Any]			  =				short_path.replace('''.py'''		,		'''''' ).replace(os.path.sep		,		'''.''' )
            if len(submodule.split('''.''' ) ) == 1:
                submodules.append(__A )
    return submodules


SCREAMING_SNAKE_CASE 							=						[
    'convert_pytorch_checkpoint_to_tf2',
    'modeling_flax_pytorch_utils',
]

def      _lowerCamelCase	(						)      ->					Tuple:
    # This is to make sure the transformers module imported is the one in the repo.
    _UpperCAmelCase				:      Union[str, Any]			  =				importlib.util.spec_from_file_location(
        '''transformers'''		,		os.path.join(__A		,		'''__init__.py''' )		,		submodule_search_locations=[PATH_TO_TRANSFORMERS]		,		)
    _UpperCAmelCase				:      Optional[Any]			  =				spec.loader.load_module()

    _UpperCAmelCase				:      List[Any]			  =				[
        module
        for module in get_transformers_submodules()
        if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys()
    ]
    if len(__A ) > 0:
        _UpperCAmelCase				:      Tuple			  =				'''\n'''.join(f'''- {module}''' for module in module_not_registered )
        raise ValueError(
            '''The following submodules are not properly registered in the main init of Transformers:\n'''
            f'''{list_of_modules}\n'''
            '''Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value.''' )


if __name__ == "__main__":
  check_all_inits()
  check_submodules()







import importlib
import json
import os
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
import transformers.models.auto
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.bert.configuration_bert import BertConfig
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402


SCREAMING_SNAKE_CASE 							=						get_tests_dir('fixtures/dummy-config.json')
class 							A_    (						unittest.TestCase			):






 '''simple docstring'''




 def  snake_case__				(	self) ->		Optional[int]:

     """simple docstring"""



     _UpperCAmelCase				:      Union[str, Any]			  =				0


 def  snake_case__				(	self) ->		Union[str, Any]:

     """simple docstring"""



     self.assertIsNotNone(transformers.models.auto.__spec__)
     self.assertIsNotNone(importlib.util.find_spec('''transformers.models.auto'''))


 def  snake_case__				(	self) ->		Optional[Any]:

     """simple docstring"""



     _UpperCAmelCase				:      List[str]			  =				AutoConfig.from_pretrained('''bert-base-uncased''')
     self.assertIsInstance(_A							,  _A)


 def  snake_case__				(	self) ->		Dict:

     """simple docstring"""



     _UpperCAmelCase				:      Tuple			  =				AutoConfig.from_pretrained(_A)
     self.assertIsInstance(_A							,  _A)


 def  snake_case__				(	self) ->		Optional[int]:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[Any]			  =				AutoConfig.from_pretrained(_A)
     self.assertIsInstance(_A							,  _A)


 def  snake_case__				(	self) ->		Any:

     """simple docstring"""



     _UpperCAmelCase				:      str			  =				AutoConfig.for_model('''roberta''')
     self.assertIsInstance(_A							,  _A)


 def  snake_case__				(	self) ->		int:

     """simple docstring"""



     with tempfile.TemporaryDirectory() as tmp_dir:
         # This model name contains bert and roberta, but roberta ends up being picked.
         _UpperCAmelCase				:      str			  =				os.path.join(_A							,  '''fake-roberta''')
         os.makedirs(_A							,  exist_ok=_A)
         with open(os.path.join(_A							,  '''config.json''')							,  '''w''') as f:
             f.write(json.dumps({}))
         _UpperCAmelCase				:      Any			  =				AutoConfig.from_pretrained(_A)
         self.assertEqual(type(_A)							,  _A)


 def  snake_case__				(	self) ->		List[Any]:

     """simple docstring"""



     try:
         AutoConfig.register('''custom'''							,  _A)
         # Wrong model type will raise an error
         with self.assertRaises(_A):
             AutoConfig.register('''model'''							,  _A)
         # Trying to register something existing in the Transformers library will raise an error
         with self.assertRaises(_A):
             AutoConfig.register('''bert'''							,  _A)

         # Now that the config is registered, it can be used as any other config with the auto-API
         _UpperCAmelCase				:      Tuple			  =				CustomConfig()
         with tempfile.TemporaryDirectory() as tmp_dir:
             config.save_pretrained(_A)
             _UpperCAmelCase				:      Tuple			  =				AutoConfig.from_pretrained(_A)
             self.assertIsInstance(_A							,  _A)

     finally:
         if "custom" in CONFIG_MAPPING._extra_content:
             del CONFIG_MAPPING._extra_content["custom"]


 def  snake_case__				(	self) ->		Union[str, Any]:

     """simple docstring"""



     with self.assertRaisesRegex(
         _A							,  '''bert-base is not a local folder and is not a valid model identifier'''):
         _UpperCAmelCase				:      Tuple			  =				AutoConfig.from_pretrained('''bert-base''')


 def  snake_case__				(	self) ->		int:

     """simple docstring"""



     with self.assertRaisesRegex(
         _A							,  R'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''):
         _UpperCAmelCase				:      List[Any]			  =				AutoConfig.from_pretrained(_A							,  revision='''aaaaaa''')


 def  snake_case__				(	self) ->		Optional[Any]:

     """simple docstring"""



     with self.assertRaisesRegex(
         _A							,  '''hf-internal-testing/no-config-test-repo does not appear to have a file named config.json.'''							,  ):
         _UpperCAmelCase				:      List[str]			  =				AutoConfig.from_pretrained('''hf-internal-testing/no-config-test-repo''')


 def  snake_case__				(	self) ->		List[str]:

     """simple docstring"""



     with self.assertRaises(_A):
         _UpperCAmelCase				:      Dict			  =				AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model''')
     # If remote code is disabled, we can't load this config.
     with self.assertRaises(_A):
         _UpperCAmelCase				:      Optional[int]			  =				AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model'''							,  trust_remote_code=_A)

     _UpperCAmelCase				:      int			  =				AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model'''							,  trust_remote_code=_A)
     self.assertEqual(config.__class__.__name__							,  '''NewModelConfig''')

     # Test config can be reloaded.
     with tempfile.TemporaryDirectory() as tmp_dir:
         config.save_pretrained(_A)
         _UpperCAmelCase				:      Any			  =				AutoConfig.from_pretrained(_A							,  trust_remote_code=_A)
     self.assertEqual(reloaded_config.__class__.__name__							,  '''NewModelConfig''')


 def  snake_case__				(	self) ->		Tuple:

     """simple docstring"""



     class 							A_    (						__lowercase			):






      '''simple docstring'''




      _SCREAMING_SNAKE_CASE							:			Union[str, Any] =					"new-model"

     try:
         AutoConfig.register('''new-model'''							,  _A)
         # If remote code is not set, the default is to use local
         _UpperCAmelCase				:      List[str]			  =				AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model''')
         self.assertEqual(config.__class__.__name__							,  '''NewModelConfigLocal''')

         # If remote code is disabled, we load the local one.
         _UpperCAmelCase				:      List[Any]			  =				AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model'''							,  trust_remote_code=_A)
         self.assertEqual(config.__class__.__name__							,  '''NewModelConfigLocal''')

         # If remote is enabled, we load from the Hub
         _UpperCAmelCase				:      Dict			  =				AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model'''							,  trust_remote_code=_A)
         self.assertEqual(config.__class__.__name__							,  '''NewModelConfig''')

     finally:
         if "new-model" in CONFIG_MAPPING._extra_content:
             del CONFIG_MAPPING._extra_content["new-model"]




