


'''simple docstring'''



import collections
import os
import re
from pathlib import Path


__A	='src/transformers'


# Matches is_xxx_available()
__A	=re.compile(R'is\_([a-z_]*)_available()')
# Catches a one-line _import_struct = {xxx}
__A	=re.compile(R'^_import_structure\s+=\s+\{([^\}]+)\}')
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
__A	=re.compile(R'\s+"\S*":\s+\[([^\]]*)\]')
# Catches a line if not is_foo_available
__A	=re.compile(R'^\s*if\s+not\s+is\_[a-z_]*\_available\(\)')
# Catches a line _import_struct["bla"].append("foo")
__A	=re.compile(R'^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)')
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
__A	=re.compile(R'^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]')
# Catches a line with an object between quotes and a comma:     "MyModel",
__A	=re.compile(R'^\s+"([^"]+)",')
# Catches a line with objects between brackets only:    ["foo", "bar"],
__A	=re.compile(R'^\s+\[([^\]]+)\]')
# Catches a line with from foo import bar, bla, boo
__A	=re.compile(R'\s+from\s+\S*\s+import\s+([^\(\s].*)\n')
# Catches a line with try:
__A	=re.compile(R'^\s*try:')
# Catches a line with else:
__A	=re.compile(R'^\s*else:')







def     _UpperCamelCase      (		UpperCamelCase__					):
			if _re_test_backend.search(UpperCamelCase__					) is None:
						return None
			UpperCAmelCase__  :					Tuple        =     [b[0] for b in _re_backend.findall(UpperCamelCase__					)]
			backends.sort()
			return "_and_".join(UpperCamelCase__					)







def     _UpperCamelCase      (		UpperCamelCase__					):
			with open(UpperCamelCase__ ,      """r""" ,      encoding="""utf-8""" ,      newline="""\n"""					) as f:
						UpperCAmelCase__  :					Union[str, Any]        =     f.readlines()

			UpperCAmelCase__  :					Union[str, Any]        =     0
			while line_index < len(UpperCamelCase__					) and not lines[line_index].startswith("""_import_structure = {"""					):
						line_index += 1

			# If this is a traditional init, just return.
			if line_index >= len(UpperCamelCase__					):
						return None

			# First grab the objects without a specific backend in _import_structure
			UpperCAmelCase__  :					int        =     []
			while not lines[line_index].startswith("""if TYPE_CHECKING"""					) and find_backend(lines[line_index]					) is None:
						UpperCAmelCase__  :					List[Any]        =     lines[line_index]
						# If we have everything on a single line, let's deal with it.
						if _re_one_line_import_struct.search(UpperCamelCase__					):
									UpperCAmelCase__  :					Any        =     _re_one_line_import_struct.search(UpperCamelCase__					).groups()[0]
									UpperCAmelCase__  :					Optional[Any]        =     re.findall(R"""\[([^\]]+)\]""" ,      UpperCamelCase__					)
									for imp in imports:
												objects.extend([obj[1:-1] for obj in imp.split(""", """					)]					)
									line_index += 1
									continue
						UpperCAmelCase__  :					List[Any]        =     _re_import_struct_key_value.search(UpperCamelCase__					)
						if single_line_import_search is not None:
									UpperCAmelCase__  :					Optional[Any]        =     [obj[1:-1] for obj in single_line_import_search.groups()[0].split(""", """					) if len(UpperCamelCase__					) > 0]
									objects.extend(UpperCamelCase__					)
						elif line.startswith(""" """ * 8 + """\""""					):
									objects.append(line[9:-3]					)
						line_index += 1

			UpperCAmelCase__  :					Union[str, Any]        =     {"""none""": objects}
			# Let's continue with backend-specific objects in _import_structure
			while not lines[line_index].startswith("""if TYPE_CHECKING"""					):
						# If the line is an if not is_backend_available, we grab all objects associated.
						UpperCAmelCase__  :					Optional[int]        =     find_backend(lines[line_index]					)
						# Check if the backend declaration is inside a try block:
						if _re_try.search(lines[line_index - 1]					) is None:
									UpperCAmelCase__  :					str        =     None

						if backend is not None:
									line_index += 1

									# Scroll until we hit the else block of try-except-else
									while _re_else.search(lines[line_index]					) is None:
												line_index += 1

									line_index += 1

									UpperCAmelCase__  :					Dict        =     []
									# Until we unindent, add backend objects to the list
									while len(lines[line_index]					) <= 1 or lines[line_index].startswith(""" """ * 4					):
												UpperCAmelCase__  :					Optional[Any]        =     lines[line_index]
												if _re_import_struct_add_one.search(UpperCamelCase__					) is not None:
															objects.append(_re_import_struct_add_one.search(UpperCamelCase__					).groups()[0]					)
												elif _re_import_struct_add_many.search(UpperCamelCase__					) is not None:
															UpperCAmelCase__  :					str        =     _re_import_struct_add_many.search(UpperCamelCase__					).groups()[0].split(""", """					)
															UpperCAmelCase__  :					int        =     [obj[1:-1] for obj in imports if len(UpperCamelCase__					) > 0]
															objects.extend(UpperCamelCase__					)
												elif _re_between_brackets.search(UpperCamelCase__					) is not None:
															UpperCAmelCase__  :					str        =     _re_between_brackets.search(UpperCamelCase__					).groups()[0].split(""", """					)
															UpperCAmelCase__  :					List[Any]        =     [obj[1:-1] for obj in imports if len(UpperCamelCase__					) > 0]
															objects.extend(UpperCamelCase__					)
												elif _re_quote_object.search(UpperCamelCase__					) is not None:
															objects.append(_re_quote_object.search(UpperCamelCase__					).groups()[0]					)
												elif line.startswith(""" """ * 8 + """\""""					):
															objects.append(line[9:-3]					)
												elif line.startswith(""" """ * 1_2 + """\""""					):
															objects.append(line[1_3:-3]					)
												line_index += 1

									UpperCAmelCase__  :					Any        =     objects
						else:
									line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
			UpperCAmelCase__  :					Tuple        =     []
			while (
			    line_index < len(UpperCamelCase__					)
			    and find_backend(lines[line_index]					) is None
			    and not lines[line_index].startswith("""else"""					)
			):
						UpperCAmelCase__  :					List[str]        =     lines[line_index]
						UpperCAmelCase__  :					Union[str, Any]        =     _re_import.search(UpperCamelCase__					)
						if single_line_import_search is not None:
									objects.extend(single_line_import_search.groups()[0].split(""", """					)					)
						elif line.startswith(""" """ * 8					):
									objects.append(line[8:-2]					)
						line_index += 1

			UpperCAmelCase__  :					List[str]        =     {"""none""": objects}
			# Let's continue with backend-specific objects
			while line_index < len(UpperCamelCase__					):
						# If the line is an if is_backend_available, we grab all objects associated.
						UpperCAmelCase__  :					Tuple        =     find_backend(lines[line_index]					)
						# Check if the backend declaration is inside a try block:
						if _re_try.search(lines[line_index - 1]					) is None:
									UpperCAmelCase__  :					Optional[int]        =     None

						if backend is not None:
									line_index += 1

									# Scroll until we hit the else block of try-except-else
									while _re_else.search(lines[line_index]					) is None:
												line_index += 1

									line_index += 1

									UpperCAmelCase__  :					Dict        =     []
									# Until we unindent, add backend objects to the list
									while len(lines[line_index]					) <= 1 or lines[line_index].startswith(""" """ * 8					):
												UpperCAmelCase__  :					int        =     lines[line_index]
												UpperCAmelCase__  :					Tuple        =     _re_import.search(UpperCamelCase__					)
												if single_line_import_search is not None:
															objects.extend(single_line_import_search.groups()[0].split(""", """					)					)
												elif line.startswith(""" """ * 1_2					):
															objects.append(line[1_2:-2]					)
												line_index += 1

									UpperCAmelCase__  :					Tuple        =     objects
						else:
									line_index += 1

			return import_dict_objects, type_hint_objects







def     _UpperCamelCase      (		UpperCamelCase__ ,      UpperCamelCase__					):

			def find_duplicates(UpperCamelCase__					):
						return [k for k, v in collections.Counter(UpperCamelCase__					).items() if v > 1]

			if list(import_dict_objects.keys()					) != list(type_hint_objects.keys()					):
						return ["Both sides of the init do not have the same backends!"]

			UpperCAmelCase__  :					List[Any]        =     []
			for key in import_dict_objects.keys():
						UpperCAmelCase__  :					str        =     find_duplicates(import_dict_objects[key]					)
						if duplicate_imports:
									errors.append(f'''Duplicate _import_structure definitions for: {duplicate_imports}'''					)
						UpperCAmelCase__  :					List[Any]        =     find_duplicates(type_hint_objects[key]					)
						if duplicate_type_hints:
									errors.append(f'''Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}'''					)

						if sorted(set(import_dict_objects[key]					)					) != sorted(set(type_hint_objects[key]					)					):
									UpperCAmelCase__  :					str        =     """base imports""" if key == """none""" else f'''{key} backend'''
									errors.append(f'''Differences for {name}:'''					)
									for a in type_hint_objects[key]:
												if a not in import_dict_objects[key]:
															errors.append(f'''  {a} in TYPE_HINT but not in _import_structure.'''					)
									for a in import_dict_objects[key]:
												if a not in type_hint_objects[key]:
															errors.append(f'''  {a} in _import_structure but not in TYPE_HINT.'''					)
			return errors







def     _UpperCamelCase      (		):
			UpperCAmelCase__  :					Any        =     []
			for root, _, files in os.walk(UpperCamelCase__					):
						if "__init__.py" in files:
									UpperCAmelCase__  :					Dict        =     os.path.join(UpperCamelCase__ ,      """__init__.py"""					)
									UpperCAmelCase__  :					List[Any]        =     parse_init(UpperCamelCase__					)
									if objects is not None:
												UpperCAmelCase__  :					Any        =     analyze_results(*UpperCamelCase__					)
												if len(UpperCamelCase__					) > 0:
															UpperCAmelCase__  :					str        =     f'''Problem in {fname}, both halves do not define the same objects.\n{errors[0]}'''
															failures.append("""\n""".join(UpperCamelCase__					)					)
			if len(UpperCamelCase__					) > 0:
						raise ValueError("""\n\n""".join(UpperCamelCase__					)					)







def     _UpperCamelCase      (		):
			UpperCAmelCase__  :					Optional[Any]        =     []
			for path, directories, files in os.walk(UpperCamelCase__					):
						for folder in directories:
									# Ignore private modules
									if folder.startswith("""_"""					):
												directories.remove(UpperCamelCase__					)
												continue
									# Ignore leftovers from branches (empty folders apart from pycache)
									if len(list((Path(UpperCamelCase__					) / folder).glob("""*.py"""					)					)					) == 0:
												continue
									UpperCAmelCase__  :					int        =     str((Path(UpperCamelCase__					) / folder).relative_to(UpperCamelCase__					)					)
									UpperCAmelCase__  :					Optional[int]        =     short_path.replace(os.path.sep ,      """."""					)
									submodules.append(UpperCamelCase__					)
						for fname in files:
									if fname == "__init__.py":
												continue
									UpperCAmelCase__  :					str        =     str((Path(UpperCamelCase__					) / fname).relative_to(UpperCamelCase__					)					)
									UpperCAmelCase__  :					Union[str, Any]        =     short_path.replace(""".py""" ,      """"""					).replace(os.path.sep ,      """."""					)
									if len(submodule.split("""."""					)					) == 1:
												submodules.append(UpperCamelCase__					)
			return submodules


__A	=[
    'convert_pytorch_checkpoint_to_tf2',
    'modeling_flax_pytorch_utils',
    'models.esm.openfold_utils',
]







def     _UpperCamelCase      (		):
			# This is to make sure the transformers module imported is the one in the repo.
			from transformers.utils import direct_transformers_import

			UpperCAmelCase__  :					str        =     direct_transformers_import(UpperCamelCase__					)

			UpperCAmelCase__  :					Any        =     set(transformers._import_structure.keys()					)
			# This contains all the base keys of the _import_structure object defined in the init, but if the user is missing
			# some optional dependencies, they may not have all of them. Thus we read the init to read all additions and
			# (potentiall re-) add them.
			with open(os.path.join(UpperCamelCase__ ,      """__init__.py"""					) ,      """r"""					) as f:
						UpperCAmelCase__  :					str        =     f.read()
			import_structure_keys.update(set(re.findall(R"""import_structure\[\"([^\"]*)\"\]""" ,      UpperCamelCase__					)					)					)

			UpperCAmelCase__  :					str        =     [
			    module
			    for module in get_transformers_submodules()
			    if module not in IGNORE_SUBMODULES and module not in import_structure_keys
			]

			if len(UpperCamelCase__					) > 0:
						UpperCAmelCase__  :					Any        =     """\n""".join(f'''- {module}''' for module in module_not_registered					)
						raise ValueError(
						    """The following submodules are not properly registed in the main init of Transformers:\n"""
						    f'''{list_of_modules}\n'''
						    """Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value."""					)


if __name__ == "__main__":
		check_all_inits()
		check_submodules()



'''simple docstring'''



import unittest

from transformers import AutoTokenizer, is_flax_available
from transformers.testing_utils import require_flax, require_sentencepiece, require_tokenizers, slow


if is_flax_available():
		import jax.numpy as jnp

		from transformers import FlaxXLMRobertaModel


@require_sentencepiece
@require_tokenizers
@require_flax
class 		_snake_case					(     unittest.TestCase	):




					@slow
					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Union[str, Any]        =     FlaxXLMRobertaModel.from_pretrained("""xlm-roberta-base""")
								UpperCAmelCase__  :					List[str]        =     AutoTokenizer.from_pretrained("""xlm-roberta-base""")
								UpperCAmelCase__  :					Dict        =     """The dog is cute and lives in the garden house"""
								UpperCAmelCase__  :					Dict        =     jnp.array([tokenizer.encode(_lowerCamelCase)])

								UpperCAmelCase__  :					int        =     (1, 12, 768)  # batch_size, sequence_length, embedding_vector_dim
								UpperCAmelCase__  :					Any        =     jnp.array(
								    [[-0.0101, 0.1218, -0.0803, 0.0801, 0.1327, 0.0776, -0.1215, 0.2383, 0.3338, 0.3106, 0.0300, 0.0252]])

								UpperCAmelCase__  :					Any        =     model(_lowerCamelCase)["""last_hidden_state"""]
								self.assertEqual(output.shape							,							_lowerCamelCase)
								# compare the actual values for a slice of last dim
								self.assertTrue(jnp.allclose(output[:, :, -1]							,							_lowerCamelCase							,							atol=1e-3))