import collections
import importlib.util
import os
import re
from pathlib import Path


snake_case      =    "src/transformers"


# Matches is_xxx_available()
snake_case      =    re.compile(r"is\_([a-z_]*)_available()")
# Catches a one-line _import_struct = {xxx}
snake_case      =    re.compile(r"^_import_structure\s+=\s+\{([^\}]+)\}")
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
snake_case      =    re.compile(r"\s+\"\S*\":\s+\[([^\]]*)\]")
# Catches a line if not is_foo_available
snake_case      =    re.compile(r"^\s*if\s+not\s+is\_[a-z_]*\_available\(\)")
# Catches a line _import_struct["bla"].append("foo")
snake_case      =    re.compile(r"^\s*_import_structure\[\"\S*\"\]\.append\(\"(\S*)\"\)")
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
snake_case      =    re.compile(r"^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]")
# Catches a line with an object between quotes and a comma:     "MyModel",
snake_case      =    re.compile("^\s+\"([^\"]+)\",")
# Catches a line with objects between brackets only:    ["foo", "bar"],
snake_case      =    re.compile("^\s+\[([^\]]+)\]")
# Catches a line with from foo import bar, bla, boo
snake_case      =    re.compile(r"\s+from\s+\S*\s+import\s+([^\(\s].*)\n")
# Catches a line with try:
snake_case      =    re.compile(r"^\s*try:")
# Catches a line with else:
snake_case      =    re.compile(r"^\s*else:")






def        UpperCamelCase_							(			lowerCAmelCase__  ):



					"""simple docstring"""

					if _re_test_backend.search(__a  ) is None:
										return None
					_lowerCAmelCase			:		Optional[Any]          = [b[0] for b in _re_backend.findall(__a  )]
					backends.sort()
					return "_and_".join(__a  )






def        UpperCamelCase_							(			lowerCAmelCase__  ):



					"""simple docstring"""

					with open(__a						,	"r"						,	encoding="utf-8"						,	newline="\n"  ) as f:
										_lowerCAmelCase			:		List[str]          = f.readlines()

					_lowerCAmelCase			:		Tuple          = 0
					while line_index < len(__a  ) and not lines[line_index].startswith("_import_structure = {"  ):
										line_index += 1

					# If this is a traditional init, just return.
					if line_index >= len(__a  ):
										return None

					# First grab the objects without a specific backend in _import_structure
					_lowerCAmelCase			:		Union[str, Any]          = []
					while not lines[line_index].startswith("if TYPE_CHECKING"  ) and find_backend(lines[line_index]  ) is None:
										_lowerCAmelCase			:		Union[str, Any]          = lines[line_index]
										# If we have everything on a single line, let's deal with it.
										if _re_one_line_import_struct.search(__a  ):
															_lowerCAmelCase			:		Optional[Any]          = _re_one_line_import_struct.search(__a  ).groups()[0]
															_lowerCAmelCase			:		Union[str, Any]          = re.findall("\[([^\]]+)\]"						,	__a  )
															for imp in imports:
																				objects.extend([obj[1:-1] for obj in imp.split(", "  )]  )
															line_index += 1
															continue
										_lowerCAmelCase			:		List[Any]          = _re_import_struct_key_value.search(__a  )
										if single_line_import_search is not None:
															_lowerCAmelCase			:		Union[str, Any]          = [obj[1:-1] for obj in single_line_import_search.groups()[0].split(", "  ) if len(__a  ) > 0]
															objects.extend(__a  )
										elif line.startswith(" " * 8 + "\""  ):
															objects.append(line[9:-3]  )
										line_index += 1

					_lowerCAmelCase			:		str          = {'''none''': objects}
					# Let's continue with backend-specific objects in _import_structure
					while not lines[line_index].startswith("if TYPE_CHECKING"  ):
										# If the line is an if not is_backend_available, we grab all objects associated.
										_lowerCAmelCase			:		Optional[Any]          = find_backend(lines[line_index]  )
										# Check if the backend declaration is inside a try block:
										if _re_try.search(lines[line_index - 1]  ) is None:
															_lowerCAmelCase			:		str          = None

										if backend is not None:
															line_index += 1

															# Scroll until we hit the else block of try-except-else
															while _re_else.search(lines[line_index]  ) is None:
																				line_index += 1

															line_index += 1

															_lowerCAmelCase			:		Any          = []
															# Until we unindent, add backend objects to the list
															while len(lines[line_index]  ) <= 1 or lines[line_index].startswith(" " * 4  ):
																				_lowerCAmelCase			:		List[Any]          = lines[line_index]
																				if _re_import_struct_add_one.search(__a  ) is not None:
																									objects.append(_re_import_struct_add_one.search(__a  ).groups()[0]  )
																				elif _re_import_struct_add_many.search(__a  ) is not None:
																									_lowerCAmelCase			:		List[str]          = _re_import_struct_add_many.search(__a  ).groups()[0].split(", "  )
																									_lowerCAmelCase			:		Optional[int]          = [obj[1:-1] for obj in imports if len(__a  ) > 0]
																									objects.extend(__a  )
																				elif _re_between_brackets.search(__a  ) is not None:
																									_lowerCAmelCase			:		Dict          = _re_between_brackets.search(__a  ).groups()[0].split(", "  )
																									_lowerCAmelCase			:		int          = [obj[1:-1] for obj in imports if len(__a  ) > 0]
																									objects.extend(__a  )
																				elif _re_quote_object.search(__a  ) is not None:
																									objects.append(_re_quote_object.search(__a  ).groups()[0]  )
																				elif line.startswith(" " * 8 + "\""  ):
																									objects.append(line[9:-3]  )
																				elif line.startswith(" " * 12 + "\""  ):
																									objects.append(line[13:-3]  )
																				line_index += 1

															_lowerCAmelCase			:		Optional[int]          = objects
										else:
															line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
					_lowerCAmelCase			:		Union[str, Any]          = []
					while (
					    line_index < len(__a  )
					    and find_backend(lines[line_index]  ) is None
					    and not lines[line_index].startswith("else"  )
					):
										_lowerCAmelCase			:		Union[str, Any]          = lines[line_index]
										_lowerCAmelCase			:		Optional[Any]          = _re_import.search(__a  )
										if single_line_import_search is not None:
															objects.extend(single_line_import_search.groups()[0].split(", "  )  )
										elif line.startswith(" " * 8  ):
															objects.append(line[8:-2]  )
										line_index += 1

					_lowerCAmelCase			:		int          = {'''none''': objects}
					# Let's continue with backend-specific objects
					while line_index < len(__a  ):
										# If the line is an if is_backend_available, we grab all objects associated.
										_lowerCAmelCase			:		Optional[int]          = find_backend(lines[line_index]  )
										# Check if the backend declaration is inside a try block:
										if _re_try.search(lines[line_index - 1]  ) is None:
															_lowerCAmelCase			:		Union[str, Any]          = None

										if backend is not None:
															line_index += 1

															# Scroll until we hit the else block of try-except-else
															while _re_else.search(lines[line_index]  ) is None:
																				line_index += 1

															line_index += 1

															_lowerCAmelCase			:		Any          = []
															# Until we unindent, add backend objects to the list
															while len(lines[line_index]  ) <= 1 or lines[line_index].startswith(" " * 8  ):
																				_lowerCAmelCase			:		List[Any]          = lines[line_index]
																				_lowerCAmelCase			:		Any          = _re_import.search(__a  )
																				if single_line_import_search is not None:
																									objects.extend(single_line_import_search.groups()[0].split(", "  )  )
																				elif line.startswith(" " * 12  ):
																									objects.append(line[12:-2]  )
																				line_index += 1

															_lowerCAmelCase			:		Optional[Any]          = objects
										else:
															line_index += 1

					return import_dict_objects, type_hint_objects






def        UpperCamelCase_							(			lowerCAmelCase__						,	lowerCAmelCase__  ):



					"""simple docstring"""

					def find_duplicates(lowerCAmelCase__  ):
										return [k for k, v in collections.Counter(__a  ).items() if v > 1]

					if list(import_dict_objects.keys()  ) != list(type_hint_objects.keys()  ):
										return ["Both sides of the init do not have the same backends!"]

					_lowerCAmelCase			:		Optional[int]          = []
					for key in import_dict_objects.keys():
										_lowerCAmelCase			:		int          = find_duplicates(import_dict_objects[key]  )
										if duplicate_imports:
															errors.append(f"""Duplicate _import_structure definitions for: {duplicate_imports}"""  )
										_lowerCAmelCase			:		Optional[Any]          = find_duplicates(type_hint_objects[key]  )
										if duplicate_type_hints:
															errors.append(f"""Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}"""  )

										if sorted(set(import_dict_objects[key]  )  ) != sorted(set(type_hint_objects[key]  )  ):
															_lowerCAmelCase			:		Tuple          = '''base imports''' if key == '''none''' else f"""{key} backend"""
															errors.append(f"""Differences for {name}:"""  )
															for a in type_hint_objects[key]:
																				if a not in import_dict_objects[key]:
																									errors.append(f"""  {a} in TYPE_HINT but not in _import_structure."""  )
															for a in import_dict_objects[key]:
																				if a not in type_hint_objects[key]:
																									errors.append(f"""  {a} in _import_structure but not in TYPE_HINT."""  )
					return errors






def        UpperCamelCase_							(			):



					"""simple docstring"""

					_lowerCAmelCase			:		Union[str, Any]          = []
					for root, _, files in os.walk(__a  ):
										if "__init__.py" in files:
															_lowerCAmelCase			:		List[Any]          = os.path.join(__a						,	"__init__.py"  )
															_lowerCAmelCase			:		Any          = parse_init(__a  )
															if objects is not None:
																				_lowerCAmelCase			:		str          = analyze_results(*__a  )
																				if len(__a  ) > 0:
																									_lowerCAmelCase			:		List[str]          = f"""Problem in {fname}, both halves do not define the same objects.\n{errors[0]}"""
																									failures.append("\n".join(__a  )  )
					if len(__a  ) > 0:
										raise ValueError("\n\n".join(__a  )  )






def        UpperCamelCase_							(			):



					"""simple docstring"""

					_lowerCAmelCase			:		int          = []
					for path, directories, files in os.walk(__a  ):
										for folder in directories:
															# Ignore private modules
															if folder.startswith("_"  ):
																				directories.remove(__a  )
																				continue
															# Ignore leftovers from branches (empty folders apart from pycache)
															if len(list((Path(__a  ) / folder).glob("*.py"  )  )  ) == 0:
																				continue
															_lowerCAmelCase			:		Union[str, Any]          = str((Path(__a  ) / folder).relative_to(__a  )  )
															_lowerCAmelCase			:		Any          = short_path.replace(os.path.sep						,	"."  )
															submodules.append(__a  )
										for fname in files:
															if fname == "__init__.py":
																				continue
															_lowerCAmelCase			:		List[str]          = str((Path(__a  ) / fname).relative_to(__a  )  )
															_lowerCAmelCase			:		Optional[int]          = short_path.replace(".py"						,	""  ).replace(os.path.sep						,	"."  )
															if len(submodule.split("."  )  ) == 1:
																				submodules.append(__a  )
					return submodules


snake_case      =    [
    "convert_pytorch_checkpoint_to_tf2",
    "modeling_flax_pytorch_utils",
]






def        UpperCamelCase_							(			):



					"""simple docstring"""

					_lowerCAmelCase			:		Any          = importlib.util.spec_from_file_location(
					    "transformers"						,	os.path.join(__a						,	"__init__.py"  )						,	submodule_search_locations=[PATH_TO_TRANSFORMERS]						,	)
					_lowerCAmelCase			:		List[Any]          = spec.loader.load_module()

					_lowerCAmelCase			:		Optional[int]          = [
					    module
					    for module in get_transformers_submodules()
					    if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys()
					]
					if len(__a  ) > 0:
										_lowerCAmelCase			:		List[str]          = '''\n'''.join(f"""- {module}""" for module in module_not_registered  )
										raise ValueError(
										    "The following submodules are not properly registered in the main init of Transformers:\n"
										    f"""{list_of_modules}\n"""
										    "Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value."  )


if __name__ == "__main__":
						check_all_inits()
						check_submodules()










'''simple docstring'''
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow


if is_torch_available():
		import torch

		from transformers import XLMRobertaModel
@require_sentencepiece
@require_tokenizers
@require_torch
class   lowercase      (				unittest.TestCase   ):

		"""simple docstring"""

		@slow
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Optional[Any]				   =					XLMRobertaModel.from_pretrained('''xlm-roberta-base'''				)
							UpperCamelCase__  :List[str]				   =					torch.tensor([[0, 581, 10269, 83, 99942, 136, 60742, 23, 70, 80583, 18276, 2]]				)
							# The dog is cute and lives in the garden house

							UpperCamelCase__  :Union[str, Any]				   =					torch.Size((1, 12, 768)				)  # batch_size, sequence_length, embedding_vector_dim
							UpperCamelCase__  :Union[str, Any]				   =					torch.tensor(
							    [[-0.0101, 0.1218, -0.0803, 0.0801, 0.1327, 0.0776, -0.1215, 0.2383, 0.3338, 0.3106, 0.0300, 0.0252]]				)
							#  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')
							#  xlmr.eval()
							#  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
							with torch.no_grad():
												UpperCamelCase__  :Any				   =					model(UpperCamelCase_				)['''last_hidden_state'''].detach()
							self.assertEqual(output.shape       ,  UpperCamelCase_				)
							# compare the actual values for a slice of last dim
							self.assertTrue(torch.allclose(output[:, :, -1]       ,  UpperCamelCase_       ,  atol=1e-3				)				)




		@slow
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :int				   =					XLMRobertaModel.from_pretrained('''xlm-roberta-large'''				)
							UpperCamelCase__  :List[str]				   =					torch.tensor([[0, 581, 10269, 83, 99942, 136, 60742, 23, 70, 80583, 18276, 2]]				)
							# The dog is cute and lives in the garden house

							UpperCamelCase__  :Any				   =					torch.Size((1, 12, 1024)				)  # batch_size, sequence_length, embedding_vector_dim
							UpperCamelCase__  :List[Any]				   =					torch.tensor(
							    [[-0.0699, -0.0318, 0.0705, -0.1241, 0.0999, -0.0520, 0.1004, -0.1838, -0.4704, 0.1437, 0.0821, 0.0126]]				)
							#  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.large')
							#  xlmr.eval()
							#  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
							with torch.no_grad():
												UpperCamelCase__  :Optional[int]				   =					model(UpperCamelCase_				)['''last_hidden_state'''].detach()
							self.assertEqual(output.shape       ,  UpperCamelCase_				)
							# compare the actual values for a slice of last dim
							self.assertTrue(torch.allclose(output[:, :, -1]       ,  UpperCamelCase_       ,  atol=1e-3				)				)