


'''simple docstring'''



import collections
import importlib.util
import os
import re
from pathlib import Path


__A		='''src/transformers'''


# Matches is_xxx_available()
__A		=re.compile(R'is\_([a-z_]*)_available()')
# Catches a one-line _import_struct = {xxx}
__A		=re.compile(R'^_import_structure\s+=\s+\{([^\}]+)\}')
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
__A		=re.compile(R'\s+"\S*":\s+\[([^\]]*)\]')
# Catches a line if not is_foo_available
__A		=re.compile(R'^\s*if\s+not\s+is\_[a-z_]*\_available\(\)')
# Catches a line _import_struct["bla"].append("foo")
__A		=re.compile(R'^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)')
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
__A		=re.compile(R'^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]')
# Catches a line with an object between quotes and a comma:     "MyModel",
__A		=re.compile('^\s+"([^"]+)",')
# Catches a line with objects between brackets only:    ["foo", "bar"],
__A		=re.compile('^\s+\[([^\]]+)\]')
# Catches a line with from foo import bar, bla, boo
__A		=re.compile(R'\s+from\s+\S*\s+import\s+([^\(\s].*)\n')
# Catches a line with try:
__A		=re.compile(R'^\s*try:')
# Catches a line with else:
__A		=re.compile(R'^\s*else:')







def       _UpperCamelCase (						UpperCamelCase__   ):
			if _re_test_backend.search(_UpperCAmelCase   ) is None:
						return None
			UpperCAmelCase__					:   Dict       				=       [b[0] for b in _re_backend.findall(_UpperCAmelCase   )]
			backends.sort()
			return "_and_".join(_UpperCAmelCase   )







def       _UpperCamelCase (						UpperCamelCase__   ):
			with open(_UpperCAmelCase	, """r"""	, encoding="""utf-8"""	, newline="""\n"""   ) as f:
						UpperCAmelCase__					:   Dict       				=       f.readlines()

			UpperCAmelCase__					:   Optional[int]       				=       0
			while line_index < len(_UpperCAmelCase   ) and not lines[line_index].startswith("""_import_structure = {"""   ):
						line_index += 1

			# If this is a traditional init, just return.
			if line_index >= len(_UpperCAmelCase   ):
						return None

			# First grab the objects without a specific backend in _import_structure
			UpperCAmelCase__					:   Dict       				=       []
			while not lines[line_index].startswith("""if TYPE_CHECKING"""   ) and find_backend(lines[line_index]   ) is None:
						UpperCAmelCase__					:   Tuple       				=       lines[line_index]
						# If we have everything on a single line, let's deal with it.
						if _re_one_line_import_struct.search(_UpperCAmelCase   ):
									UpperCAmelCase__					:   Tuple       				=       _re_one_line_import_struct.search(_UpperCAmelCase   ).groups()[0]
									UpperCAmelCase__					:   str       				=       re.findall("""\[([^\]]+)\]"""	, _UpperCAmelCase   )
									for imp in imports:
												objects.extend([obj[1:-1] for obj in imp.split(""", """   )]   )
									line_index += 1
									continue
						UpperCAmelCase__					:   int       				=       _re_import_struct_key_value.search(_UpperCAmelCase   )
						if single_line_import_search is not None:
									UpperCAmelCase__					:   Dict       				=       [obj[1:-1] for obj in single_line_import_search.groups()[0].split(""", """   ) if len(_UpperCAmelCase   ) > 0]
									objects.extend(_UpperCAmelCase   )
						elif line.startswith(""" """ * 8 + """\""""   ):
									objects.append(line[9:-3]   )
						line_index += 1

			UpperCAmelCase__					:   Union[str, Any]       				=       {'none': objects}
			# Let's continue with backend-specific objects in _import_structure
			while not lines[line_index].startswith("""if TYPE_CHECKING"""   ):
						# If the line is an if not is_backend_available, we grab all objects associated.
						UpperCAmelCase__					:   Tuple       				=       find_backend(lines[line_index]   )
						# Check if the backend declaration is inside a try block:
						if _re_try.search(lines[line_index - 1]   ) is None:
									UpperCAmelCase__					:   List[str]       				=       None

						if backend is not None:
									line_index += 1

									# Scroll until we hit the else block of try-except-else
									while _re_else.search(lines[line_index]   ) is None:
												line_index += 1

									line_index += 1

									UpperCAmelCase__					:   str       				=       []
									# Until we unindent, add backend objects to the list
									while len(lines[line_index]   ) <= 1 or lines[line_index].startswith(""" """ * 4   ):
												UpperCAmelCase__					:   List[str]       				=       lines[line_index]
												if _re_import_struct_add_one.search(_UpperCAmelCase   ) is not None:
															objects.append(_re_import_struct_add_one.search(_UpperCAmelCase   ).groups()[0]   )
												elif _re_import_struct_add_many.search(_UpperCAmelCase   ) is not None:
															UpperCAmelCase__					:   Any       				=       _re_import_struct_add_many.search(_UpperCAmelCase   ).groups()[0].split(""", """   )
															UpperCAmelCase__					:   Optional[Any]       				=       [obj[1:-1] for obj in imports if len(_UpperCAmelCase   ) > 0]
															objects.extend(_UpperCAmelCase   )
												elif _re_between_brackets.search(_UpperCAmelCase   ) is not None:
															UpperCAmelCase__					:   List[Any]       				=       _re_between_brackets.search(_UpperCAmelCase   ).groups()[0].split(""", """   )
															UpperCAmelCase__					:   List[Any]       				=       [obj[1:-1] for obj in imports if len(_UpperCAmelCase   ) > 0]
															objects.extend(_UpperCAmelCase   )
												elif _re_quote_object.search(_UpperCAmelCase   ) is not None:
															objects.append(_re_quote_object.search(_UpperCAmelCase   ).groups()[0]   )
												elif line.startswith(""" """ * 8 + """\""""   ):
															objects.append(line[9:-3]   )
												elif line.startswith(""" """ * 1_2 + """\""""   ):
															objects.append(line[1_3:-3]   )
												line_index += 1

									UpperCAmelCase__					:   Tuple       				=       objects
						else:
									line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
			UpperCAmelCase__					:   str       				=       []
			while (
			    line_index < len(_UpperCAmelCase   )
			    and find_backend(lines[line_index]   ) is None
			    and not lines[line_index].startswith("""else"""   )
			):
						UpperCAmelCase__					:   Any       				=       lines[line_index]
						UpperCAmelCase__					:   Tuple       				=       _re_import.search(_UpperCAmelCase   )
						if single_line_import_search is not None:
									objects.extend(single_line_import_search.groups()[0].split(""", """   )   )
						elif line.startswith(""" """ * 8   ):
									objects.append(line[8:-2]   )
						line_index += 1

			UpperCAmelCase__					:   List[str]       				=       {'none': objects}
			# Let's continue with backend-specific objects
			while line_index < len(_UpperCAmelCase   ):
						# If the line is an if is_backend_available, we grab all objects associated.
						UpperCAmelCase__					:   List[Any]       				=       find_backend(lines[line_index]   )
						# Check if the backend declaration is inside a try block:
						if _re_try.search(lines[line_index - 1]   ) is None:
									UpperCAmelCase__					:   Dict       				=       None

						if backend is not None:
									line_index += 1

									# Scroll until we hit the else block of try-except-else
									while _re_else.search(lines[line_index]   ) is None:
												line_index += 1

									line_index += 1

									UpperCAmelCase__					:   Optional[Any]       				=       []
									# Until we unindent, add backend objects to the list
									while len(lines[line_index]   ) <= 1 or lines[line_index].startswith(""" """ * 8   ):
												UpperCAmelCase__					:   Optional[int]       				=       lines[line_index]
												UpperCAmelCase__					:   Optional[int]       				=       _re_import.search(_UpperCAmelCase   )
												if single_line_import_search is not None:
															objects.extend(single_line_import_search.groups()[0].split(""", """   )   )
												elif line.startswith(""" """ * 1_2   ):
															objects.append(line[1_2:-2]   )
												line_index += 1

									UpperCAmelCase__					:   Optional[Any]       				=       objects
						else:
									line_index += 1

			return import_dict_objects, type_hint_objects







def       _UpperCamelCase (						UpperCamelCase__	, UpperCamelCase__   ):

			def find_duplicates(UpperCamelCase__   ):
						return [k for k, v in collections.Counter(_UpperCAmelCase   ).items() if v > 1]

			if list(import_dict_objects.keys()   ) != list(type_hint_objects.keys()   ):
						return ["Both sides of the init do not have the same backends!"]

			UpperCAmelCase__					:   Optional[Any]       				=       []
			for key in import_dict_objects.keys():
						UpperCAmelCase__					:   List[Any]       				=       find_duplicates(import_dict_objects[key]   )
						if duplicate_imports:
									errors.append(f'''Duplicate _import_structure definitions for: {duplicate_imports}'''   )
						UpperCAmelCase__					:   Optional[int]       				=       find_duplicates(type_hint_objects[key]   )
						if duplicate_type_hints:
									errors.append(f'''Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}'''   )

						if sorted(set(import_dict_objects[key]   )   ) != sorted(set(type_hint_objects[key]   )   ):
									UpperCAmelCase__					:   Tuple       				=       'base imports' if key == 'none' else f'''{key} backend'''
									errors.append(f'''Differences for {name}:'''   )
									for a in type_hint_objects[key]:
												if a not in import_dict_objects[key]:
															errors.append(f'''  {a} in TYPE_HINT but not in _import_structure.'''   )
									for a in import_dict_objects[key]:
												if a not in type_hint_objects[key]:
															errors.append(f'''  {a} in _import_structure but not in TYPE_HINT.'''   )
			return errors







def       _UpperCamelCase (						):
			UpperCAmelCase__					:   Tuple       				=       []
			for root, _, files in os.walk(_UpperCAmelCase   ):
						if "__init__.py" in files:
									UpperCAmelCase__					:   Any       				=       os.path.join(_UpperCAmelCase	, """__init__.py"""   )
									UpperCAmelCase__					:   Union[str, Any]       				=       parse_init(_UpperCAmelCase   )
									if objects is not None:
												UpperCAmelCase__					:   Tuple       				=       analyze_results(*_UpperCAmelCase   )
												if len(_UpperCAmelCase   ) > 0:
															UpperCAmelCase__					:   Any       				=       f'''Problem in {fname}, both halves do not define the same objects.\n{errors[0]}'''
															failures.append("""\n""".join(_UpperCAmelCase   )   )
			if len(_UpperCAmelCase   ) > 0:
						raise ValueError("""\n\n""".join(_UpperCAmelCase   )   )







def       _UpperCamelCase (						):
			UpperCAmelCase__					:   Optional[int]       				=       []
			for path, directories, files in os.walk(_UpperCAmelCase   ):
						for folder in directories:
									# Ignore private modules
									if folder.startswith("""_"""   ):
												directories.remove(_UpperCAmelCase   )
												continue
									# Ignore leftovers from branches (empty folders apart from pycache)
									if len(list((Path(_UpperCAmelCase   ) / folder).glob("""*.py"""   )   )   ) == 0:
												continue
									UpperCAmelCase__					:   Dict       				=       str((Path(_UpperCAmelCase   ) / folder).relative_to(_UpperCAmelCase   )   )
									UpperCAmelCase__					:   Any       				=       short_path.replace(os.path.sep	, """."""   )
									submodules.append(_UpperCAmelCase   )
						for fname in files:
									if fname == "__init__.py":
												continue
									UpperCAmelCase__					:   Any       				=       str((Path(_UpperCAmelCase   ) / fname).relative_to(_UpperCAmelCase   )   )
									UpperCAmelCase__					:   Any       				=       short_path.replace(""".py"""	, """"""   ).replace(os.path.sep	, """."""   )
									if len(submodule.split("""."""   )   ) == 1:
												submodules.append(_UpperCAmelCase   )
			return submodules


__A		=[
    '''convert_pytorch_checkpoint_to_tf2''',
    '''modeling_flax_pytorch_utils''',
]







def       _UpperCamelCase (						):
			# This is to make sure the transformers module imported is the one in the repo.
			UpperCAmelCase__					:   Any       				=       importlib.util.spec_from_file_location(
			    """transformers"""	, os.path.join(_UpperCAmelCase	, """__init__.py"""   )	, submodule_search_locations=[PATH_TO_TRANSFORMERS]	, )
			UpperCAmelCase__					:   Tuple       				=       spec.loader.load_module()

			UpperCAmelCase__					:   Optional[int]       				=       [
			    module
			    for module in get_transformers_submodules()
			    if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys()
			]
			if len(_UpperCAmelCase   ) > 0:
						UpperCAmelCase__					:   Any       				=       '\n'.join(f'''- {module}''' for module in module_not_registered   )
						raise ValueError(
						    """The following submodules are not properly registered in the main init of Transformers:\n"""
						    f'''{list_of_modules}\n'''
						    """Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value."""   )


if __name__ == "__main__":
							check_all_inits()
							check_submodules()



'''simple docstring'''



import os
from typing import Optional

import fsspec
from fsspec.archive import AbstractArchiveFileSystem
from fsspec.utils import DEFAULT_BLOCK_SIZE


class 						_snake_case   (       a__		):
					lowerCAmelCase :Optional[int]						=   ''''''
					lowerCAmelCase :str						=   (
					    None  # protocol passed in prefix to the url. ex: "gzip", for gzip://file.txt::http://foo.bar/file.txt.gz
					)
					lowerCAmelCase :str						=   None  # compression type in fsspec. ex: "gzip"
					lowerCAmelCase :str						=   None  # extension of the filename to strip. ex: "".gz" to get file.txt from file.txt.gz




					def __init__( self							,							_lowerCamelCase = ""							,							_lowerCamelCase = None							,							_lowerCamelCase = None							,							**_lowerCamelCase):
								super().__init__(self							,							**_lowerCamelCase)
								# always open as "rb" since fsspec can then use the TextIOWrapper to make it work for "r" mode
								UpperCAmelCase__					:   Optional[Any]       				=       fsspec.open(
								    _lowerCamelCase							,							mode="""rb"""							,							protocol=_lowerCamelCase							,							compression=self.compression							,							client_kwargs={
								        """requote_redirect_url""": False,  # see https://github.com/huggingface/datasets/pull/5459
								        """trust_env""": True,  # Enable reading proxy env variables.
								        **(target_options or {}).pop("""client_kwargs"""							,							{}),  # To avoid issues if it was already passed.
								    }							,							**(target_options or {})							,							)
								UpperCAmelCase__					:   List[Any]       				=       os.path.basename(self.file.path.split("""::""")[0])
								UpperCAmelCase__					:   Dict       				=       (
								    self.compressed_name[: self.compressed_name.rindex(""".""")]
								    if """.""" in self.compressed_name
								    else self.compressed_name
								)
								UpperCAmelCase__					:   Tuple       				=       None




					@classmethod
					def 			snake_case__  ( cls							,							_lowerCamelCase):
								# compressed file paths are always relative to the archive root
								return super()._strip_protocol(_lowerCamelCase).lstrip("""/""")




					def 			snake_case__  ( self):
								if self.dir_cache is None:
											UpperCAmelCase__					:   Optional[Any]       				=       {**self.file.fs.info(self.file.path), """name""": self.uncompressed_name}
											UpperCAmelCase__					:   Union[str, Any]       				=       {f["""name"""]: f}




					def 			snake_case__  ( self							,							_lowerCamelCase):
								return self.file.open().read()




					def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase = "rb"							,							_lowerCamelCase=None							,							_lowerCamelCase=True							,							_lowerCamelCase=None							,							**_lowerCamelCase							,							):
								UpperCAmelCase__					:   List[str]       				=       self._strip_protocol(_lowerCamelCase)
								if mode != "rb":
											raise ValueError(f'''Tried to read with mode {mode} on file {self.file.path} opened with mode \'rb\'''')
								return self.file.open()


class 						_snake_case   (       a__		):
					lowerCAmelCase :Dict						=   '''bz2'''
					lowerCAmelCase :List[str]						=   '''bz2'''
					lowerCAmelCase :Dict						=   '''.bz2'''


class 						_snake_case   (       a__		):
					lowerCAmelCase :int						=   '''gzip'''
					lowerCAmelCase :Tuple						=   '''gzip'''
					lowerCAmelCase :str						=   '''.gz'''


class 						_snake_case   (       a__		):
					lowerCAmelCase :List[str]						=   '''lz4'''
					lowerCAmelCase :Any						=   '''lz4'''
					lowerCAmelCase :int						=   '''.lz4'''


class 						_snake_case   (       a__		):
					lowerCAmelCase :Union[str, Any]						=   '''xz'''
					lowerCAmelCase :int						=   '''xz'''
					lowerCAmelCase :List[Any]						=   '''.xz'''


class 						_snake_case   (       a__		):
					lowerCAmelCase :Tuple						=   '''zstd'''
					lowerCAmelCase :List[str]						=   '''zstd'''
					lowerCAmelCase :Union[str, Any]						=   '''.zst'''




					def __init__( self							,							_lowerCamelCase							,							_lowerCamelCase = "rb"							,							_lowerCamelCase = None							,							_lowerCamelCase = None							,							_lowerCamelCase = DEFAULT_BLOCK_SIZE							,							**_lowerCamelCase							,							):
								super().__init__(
								    fo=_lowerCamelCase							,							mode=_lowerCamelCase							,							target_protocol=_lowerCamelCase							,							target_options=_lowerCamelCase							,							block_size=_lowerCamelCase							,							**_lowerCamelCase							,							)
								# We need to wrap the zstd decompressor to avoid this error in fsspec==2021.7.0 and zstandard==0.15.2:
								#
								# File "/Users/user/.virtualenvs/hf-datasets/lib/python3.7/site-packages/fsspec/core.py", line 145, in open
								#     out.close = close
								# AttributeError: 'zstd.ZstdDecompressionReader' object attribute 'close' is read-only
								#
								# see https://github.com/intake/filesystem_spec/issues/725
								UpperCAmelCase__					:   Dict       				=       self.file.__enter__


								class 						_snake_case   :




													def __init__( self							,							_lowerCamelCase):
																UpperCAmelCase__					:   Optional[int]       				=       file_




													def __enter__( self):
																self._file.__enter__()
																return self




													def __exit__( self							,							*_lowerCamelCase							,							**_lowerCamelCase):
																self._file.__exit__(*_lowerCamelCase							,							**_lowerCamelCase)




													def __iter__( self):
																return iter(self._file)




													def 			snake_case__  ( self):
																return next(self._file)




													def __getattr__( self							,							_lowerCamelCase):
																return getattr(self._file							,							_lowerCamelCase)

								def fixed_enter(*_lowerCamelCase							,							**_lowerCamelCase):
											return WrappedFile(_enter(*_lowerCamelCase							,							**_lowerCamelCase))

								UpperCAmelCase__					:   List[Any]       				=       fixed_enter