





import collections
import importlib.util
import os
import re
from pathlib import Path


__lowercase  :   Optional[int]						      ="""src/transformers"""


# Matches is_xxx_available()
__lowercase  :   Union[str, Any]						      =re.compile(R"""is\_([a-z_]*)_available()""")
# Catches a one-line _import_struct = {xxx}
__lowercase  :   List[str]						      =re.compile(R"""^_import_structure\s+=\s+\{([^\}]+)\}""")
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
__lowercase  :   str						      =re.compile(R"""\s+\"\S*\":\s+\[([^\]]*)\]""")
# Catches a line if not is_foo_available
__lowercase  :   List[Any]						      =re.compile(R"""^\s*if\s+not\s+is\_[a-z_]*\_available\(\)""")
# Catches a line _import_struct["bla"].append("foo")
__lowercase  :   Optional[int]						      =re.compile(R"""^\s*_import_structure\[\"\S*\"\]\.append\(\"(\S*)\"\)""")
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
__lowercase  :   Optional[int]						      =re.compile(R"""^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]""")
# Catches a line with an object between quotes and a comma:     "MyModel",
__lowercase  :   Optional[int]						      =re.compile("""^\s+\"([^\"]+)\",""")
# Catches a line with objects between brackets only:    ["foo", "bar"],
__lowercase  :   Tuple						      =re.compile("""^\s+\[([^\]]+)\]""")
# Catches a line with from foo import bar, bla, boo
__lowercase  :   Optional[Any]						      =re.compile(R"""\s+from\s+\S*\s+import\s+([^\(\s].*)\n""")
# Catches a line with try:
__lowercase  :   str						      =re.compile(R"""^\s*try:""")
# Catches a line with else:
__lowercase  :   Dict						      =re.compile(R"""^\s*else:""")




def     a__ (					lowercase__  ):






   '''simple docstring'''






   if _re_test_backend.search(lowercase__  ) is None:
      return None
   UpperCAmelCase_											=[b[0] for b in _re_backend.findall(lowercase__  )]
   backends.sort()
   return "_and_".join(lowercase__  )




def     a__ (					lowercase__  ):






   '''simple docstring'''






   with open(lowercase__  ,    "r"  ,    encoding="utf-8"  ,    newline="\n"  ) as f:
      UpperCAmelCase_											=f.readlines()

   UpperCAmelCase_											=0
   while line_index < len(lowercase__  ) and not lines[line_index].startswith("_import_structure = {"  ):
      line_index += 1

   # If this is a traditional init, just return.
   if line_index >= len(lowercase__  ):
      return None

   # First grab the objects without a specific backend in _import_structure
   UpperCAmelCase_											=[]
   while not lines[line_index].startswith("if TYPE_CHECKING"  ) and find_backend(lines[line_index]  ) is None:
      UpperCAmelCase_											=lines[line_index]
      # If we have everything on a single line, let's deal with it.
      if _re_one_line_import_struct.search(lowercase__  ):
         UpperCAmelCase_											=_re_one_line_import_struct.search(lowercase__  ).groups()[0]
         UpperCAmelCase_											=re.findall("\[([^\]]+)\]"  ,    lowercase__  )
         for imp in imports:
            objects.extend([obj[1:-1] for obj in imp.split(", "  )]  )
         line_index += 1
         continue
      UpperCAmelCase_											=_re_import_struct_key_value.search(lowercase__  )
      if single_line_import_search is not None:
         UpperCAmelCase_											=[obj[1:-1] for obj in single_line_import_search.groups()[0].split(", "  ) if len(lowercase__  ) > 0]
         objects.extend(lowercase__  )
      elif line.startswith(" " * 8 + "\""  ):
         objects.append(line[9:-3]  )
      line_index += 1

   UpperCAmelCase_											={"none": objects}
   # Let's continue with backend-specific objects in _import_structure
   while not lines[line_index].startswith("if TYPE_CHECKING"  ):
      # If the line is an if not is_backend_available, we grab all objects associated.
      UpperCAmelCase_											=find_backend(lines[line_index]  )
      # Check if the backend declaration is inside a try block:
      if _re_try.search(lines[line_index - 1]  ) is None:
         UpperCAmelCase_											=None

      if backend is not None:
         line_index += 1

         # Scroll until we hit the else block of try-except-else
         while _re_else.search(lines[line_index]  ) is None:
            line_index += 1

         line_index += 1

         UpperCAmelCase_											=[]
         # Until we unindent, add backend objects to the list
         while len(lines[line_index]  ) <= 1 or lines[line_index].startswith(" " * 4  ):
            UpperCAmelCase_											=lines[line_index]
            if _re_import_struct_add_one.search(lowercase__  ) is not None:
               objects.append(_re_import_struct_add_one.search(lowercase__  ).groups()[0]  )
            elif _re_import_struct_add_many.search(lowercase__  ) is not None:
               UpperCAmelCase_											=_re_import_struct_add_many.search(lowercase__  ).groups()[0].split(", "  )
               UpperCAmelCase_											=[obj[1:-1] for obj in imports if len(lowercase__  ) > 0]
               objects.extend(lowercase__  )
            elif _re_between_brackets.search(lowercase__  ) is not None:
               UpperCAmelCase_											=_re_between_brackets.search(lowercase__  ).groups()[0].split(", "  )
               UpperCAmelCase_											=[obj[1:-1] for obj in imports if len(lowercase__  ) > 0]
               objects.extend(lowercase__  )
            elif _re_quote_object.search(lowercase__  ) is not None:
               objects.append(_re_quote_object.search(lowercase__  ).groups()[0]  )
            elif line.startswith(" " * 8 + "\""  ):
               objects.append(line[9:-3]  )
            elif line.startswith(" " * 1_2 + "\""  ):
               objects.append(line[1_3:-3]  )
            line_index += 1

         UpperCAmelCase_											=objects
      else:
         line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
   UpperCAmelCase_											=[]
   while (
       line_index < len(lowercase__  )
       and find_backend(lines[line_index]  ) is None
       and not lines[line_index].startswith("else"  )
   ):
      UpperCAmelCase_											=lines[line_index]
      UpperCAmelCase_											=_re_import.search(lowercase__  )
      if single_line_import_search is not None:
         objects.extend(single_line_import_search.groups()[0].split(", "  )  )
      elif line.startswith(" " * 8  ):
         objects.append(line[8:-2]  )
      line_index += 1

   UpperCAmelCase_											={"none": objects}
   # Let's continue with backend-specific objects
   while line_index < len(lowercase__  ):
      # If the line is an if is_backend_available, we grab all objects associated.
      UpperCAmelCase_											=find_backend(lines[line_index]  )
      # Check if the backend declaration is inside a try block:
      if _re_try.search(lines[line_index - 1]  ) is None:
         UpperCAmelCase_											=None

      if backend is not None:
         line_index += 1

         # Scroll until we hit the else block of try-except-else
         while _re_else.search(lines[line_index]  ) is None:
            line_index += 1

         line_index += 1

         UpperCAmelCase_											=[]
         # Until we unindent, add backend objects to the list
         while len(lines[line_index]  ) <= 1 or lines[line_index].startswith(" " * 8  ):
            UpperCAmelCase_											=lines[line_index]
            UpperCAmelCase_											=_re_import.search(lowercase__  )
            if single_line_import_search is not None:
               objects.extend(single_line_import_search.groups()[0].split(", "  )  )
            elif line.startswith(" " * 1_2  ):
               objects.append(line[1_2:-2]  )
            line_index += 1

         UpperCAmelCase_											=objects
      else:
         line_index += 1

   return import_dict_objects, type_hint_objects




def     a__ (					lowercase__  ,    lowercase__  ):






   '''simple docstring'''






   def find_duplicates(lowercase__  ):
      return [k for k, v in collections.Counter(lowercase__  ).items() if v > 1]

   if list(import_dict_objects.keys()  ) != list(type_hint_objects.keys()  ):
      return ["Both sides of the init do not have the same backends!"]

   UpperCAmelCase_											=[]
   for key in import_dict_objects.keys():
      UpperCAmelCase_											=find_duplicates(import_dict_objects[key]  )
      if duplicate_imports:
         errors.append(F'Duplicate _import_structure definitions for: {duplicate_imports}'  )
      UpperCAmelCase_											=find_duplicates(type_hint_objects[key]  )
      if duplicate_type_hints:
         errors.append(F'Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}'  )

      if sorted(set(import_dict_objects[key]  )  ) != sorted(set(type_hint_objects[key]  )  ):
         UpperCAmelCase_											="base imports" if key == "none" else F'{key} backend'
         errors.append(F'Differences for {name}:'  )
         for a in type_hint_objects[key]:
            if a not in import_dict_objects[key]:
               errors.append(F'  {a} in TYPE_HINT but not in _import_structure.'  )
         for a in import_dict_objects[key]:
            if a not in type_hint_objects[key]:
               errors.append(F'  {a} in _import_structure but not in TYPE_HINT.'  )
   return errors




def     a__ (					):






   '''simple docstring'''






   UpperCAmelCase_											=[]
   for root, _, files in os.walk(lowercase__  ):
      if "__init__.py" in files:
         UpperCAmelCase_											=os.path.join(lowercase__  ,    "__init__.py"  )
         UpperCAmelCase_											=parse_init(lowercase__  )
         if objects is not None:
            UpperCAmelCase_											=analyze_results(*lowercase__  )
            if len(lowercase__  ) > 0:
               UpperCAmelCase_											=F'Problem in {fname}, both halves do not define the same objects.\n{errors[0]}'
               failures.append("\n".join(lowercase__  )  )
   if len(lowercase__  ) > 0:
      raise ValueError("\n\n".join(lowercase__  )  )




def     a__ (					):






   '''simple docstring'''






   UpperCAmelCase_											=[]
   for path, directories, files in os.walk(lowercase__  ):
      for folder in directories:
         # Ignore private modules
         if folder.startswith("_"  ):
            directories.remove(lowercase__  )
            continue
         # Ignore leftovers from branches (empty folders apart from pycache)
         if len(list((Path(lowercase__  ) / folder).glob("*.py"  )  )  ) == 0:
            continue
         UpperCAmelCase_											=str((Path(lowercase__  ) / folder).relative_to(lowercase__  )  )
         UpperCAmelCase_											=short_path.replace(os.path.sep  ,    "."  )
         submodules.append(lowercase__  )
      for fname in files:
         if fname == "__init__.py":
            continue
         UpperCAmelCase_											=str((Path(lowercase__  ) / fname).relative_to(lowercase__  )  )
         UpperCAmelCase_											=short_path.replace(".py"  ,    ""  ).replace(os.path.sep  ,    "."  )
         if len(submodule.split("."  )  ) == 1:
            submodules.append(lowercase__  )
   return submodules


__lowercase  :   List[str]						      =[
    """convert_pytorch_checkpoint_to_tf2""",
    """modeling_flax_pytorch_utils""",
]




def     a__ (					):






   '''simple docstring'''






   UpperCAmelCase_											=importlib.util.spec_from_file_location(
       "transformers"  ,    os.path.join(lowercase__  ,    "__init__.py"  )  ,    submodule_search_locations=[PATH_TO_TRANSFORMERS]  ,    )
   UpperCAmelCase_											=spec.loader.load_module()

   UpperCAmelCase_											=[
       module
       for module in get_transformers_submodules()
       if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys()
   ]
   if len(lowercase__  ) > 0:
      UpperCAmelCase_											="\n".join(F'- {module}' for module in module_not_registered  )
      raise ValueError(
          "The following submodules are not properly registered in the main init of Transformers:\n"
          F'{list_of_modules}\n'
          "Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value."  )


if __name__ == "__main__":
     check_all_inits()
     check_submodules()











import sys

from .dependency_versions_table import deps
from .utils.versions import require_version, require_version_core


# define which module versions we always want to check at run time
# (usually the ones defined in `install_requires` in setup.py)
#
# order specific notes:
# - tqdm must be checked before tokenizers

__lowercase  :   Optional[int]						      ="""python tqdm regex requests packaging filelock numpy tokenizers""".split()
if sys.version_info < (3, 7):
     pkgs_to_check_at_runtime.append("""dataclasses""")
if sys.version_info < (3, 8):
     pkgs_to_check_at_runtime.append("""importlib_metadata""")

for pkg in pkgs_to_check_at_runtime:
     if pkg in deps:
          if pkg == "tokenizers":
               # must be loaded here, or else tqdm check may fail
               from .utils import is_tokenizers_available

               if not is_tokenizers_available():
                    continue  # not required, check version only if installed

          require_version_core(deps[pkg])
     else:
          raise ValueError(f"""can't find {pkg} in {deps.keys()}, check dependency_versions_table.py""")




def     a__ (					lowercase__  ,    lowercase__=None  ):






   '''simple docstring'''






   require_version(deps[pkg]  ,    lowercase__  )




