import collections
import importlib.util
import os
import re
from pathlib import Path


__a    :	Union[str, Any]         =    """src/transformers"""


# Matches is_xxx_available()
__a    :	Optional[int]         =    re.compile(r"""is\_([a-z_]*)_available()""")
# Catches a one-line _import_struct = {xxx}
__a    :	Optional[int]         =    re.compile(r"""^_import_structure\s+=\s+\{([^\}]+)\}""")
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
__a    :	Any         =    re.compile(r"""\s+\"\S*\":\s+\[([^\]]*)\]""")
# Catches a line if not is_foo_available
__a    :	int         =    re.compile(r"""^\s*if\s+not\s+is\_[a-z_]*\_available\(\)""")
# Catches a line _import_struct["bla"].append("foo")
__a    :	Union[str, Any]         =    re.compile(r"""^\s*_import_structure\[\"\S*\"\]\.append\(\"(\S*)\"\)""")
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
__a    :	Optional[int]         =    re.compile(r"""^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]""")
# Catches a line with an object between quotes and a comma:     "MyModel",
__a    :	List[str]         =    re.compile("""^\s+\"([^\"]+)\",""")
# Catches a line with objects between brackets only:    ["foo", "bar"],
__a    :	str         =    re.compile("""^\s+\[([^\]]+)\]""")
# Catches a line with from foo import bar, bla, boo
__a    :	Dict         =    re.compile(r"""\s+from\s+\S*\s+import\s+([^\(\s].*)\n""")
# Catches a line with try:
__a    :	Tuple         =    re.compile(r"""^\s*try:""")
# Catches a line with else:
__a    :	List[str]         =    re.compile(r"""^\s*else:""")





def 							UpperCAmelCase		(       lowercase				):







				"""simple docstring"""



				if _re_test_backend.search(lowercase				) is None:
								return None
				__lowercase					     =				[b[0] for b in _re_backend.findall(lowercase				)]
				backends.sort()
				return "_and_".join(lowercase				)





def 							UpperCAmelCase		(       lowercase				):







				"""simple docstring"""



				with open(lowercase				,     '''r'''				,     encoding='''utf-8'''				,     newline='''\n'''				) as f:
								__lowercase					     =				f.readlines()

				__lowercase					     =				0
				while line_index < len(lowercase				) and not lines[line_index].startswith('''_import_structure = {'''				):
								line_index += 1

				# If this is a traditional init, just return.
				if line_index >= len(lowercase				):
								return None

				# First grab the objects without a specific backend in _import_structure
				__lowercase					     =				[]
				while not lines[line_index].startswith('''if TYPE_CHECKING'''				) and find_backend(lines[line_index]				) is None:
								__lowercase					     =				lines[line_index]
								# If we have everything on a single line, let's deal with it.
								if _re_one_line_import_struct.search(lowercase				):
												__lowercase					     =				_re_one_line_import_struct.search(lowercase				).groups()[0]
												__lowercase					     =				re.findall('''\[([^\]]+)\]'''				,     lowercase				)
												for imp in imports:
																objects.extend([obj[1:-1] for obj in imp.split(''', '''				)]				)
												line_index += 1
												continue
								__lowercase					     =				_re_import_struct_key_value.search(lowercase				)
								if single_line_import_search is not None:
												__lowercase					     =				[obj[1:-1] for obj in single_line_import_search.groups()[0].split(''', '''				) if len(lowercase				) > 0]
												objects.extend(lowercase				)
								elif line.startswith(''' ''' * 8 + '''"'''				):
												objects.append(line[9:-3]				)
								line_index += 1

				__lowercase					     =				{'''none''': objects}
				# Let's continue with backend-specific objects in _import_structure
				while not lines[line_index].startswith('''if TYPE_CHECKING'''				):
								# If the line is an if not is_backend_available, we grab all objects associated.
								__lowercase					     =				find_backend(lines[line_index]				)
								# Check if the backend declaration is inside a try block:
								if _re_try.search(lines[line_index - 1]				) is None:
												__lowercase					     =				None

								if backend is not None:
												line_index += 1

												# Scroll until we hit the else block of try-except-else
												while _re_else.search(lines[line_index]				) is None:
																line_index += 1

												line_index += 1

												__lowercase					     =				[]
												# Until we unindent, add backend objects to the list
												while len(lines[line_index]				) <= 1 or lines[line_index].startswith(''' ''' * 4				):
																__lowercase					     =				lines[line_index]
																if _re_import_struct_add_one.search(lowercase				) is not None:
																				objects.append(_re_import_struct_add_one.search(lowercase				).groups()[0]				)
																elif _re_import_struct_add_many.search(lowercase				) is not None:
																				__lowercase					     =				_re_import_struct_add_many.search(lowercase				).groups()[0].split(''', '''				)
																				__lowercase					     =				[obj[1:-1] for obj in imports if len(lowercase				) > 0]
																				objects.extend(lowercase				)
																elif _re_between_brackets.search(lowercase				) is not None:
																				__lowercase					     =				_re_between_brackets.search(lowercase				).groups()[0].split(''', '''				)
																				__lowercase					     =				[obj[1:-1] for obj in imports if len(lowercase				) > 0]
																				objects.extend(lowercase				)
																elif _re_quote_object.search(lowercase				) is not None:
																				objects.append(_re_quote_object.search(lowercase				).groups()[0]				)
																elif line.startswith(''' ''' * 8 + '''"'''				):
																				objects.append(line[9:-3]				)
																elif line.startswith(''' ''' * 12 + '''"'''				):
																				objects.append(line[13:-3]				)
																line_index += 1

												__lowercase					     =				objects
								else:
												line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
				__lowercase					     =				[]
				while (
				    line_index < len(lowercase				)
				    and find_backend(lines[line_index]				) is None
				    and not lines[line_index].startswith('''else'''				)
				):
								__lowercase					     =				lines[line_index]
								__lowercase					     =				_re_import.search(lowercase				)
								if single_line_import_search is not None:
												objects.extend(single_line_import_search.groups()[0].split(''', '''				)				)
								elif line.startswith(''' ''' * 8				):
												objects.append(line[8:-2]				)
								line_index += 1

				__lowercase					     =				{'''none''': objects}
				# Let's continue with backend-specific objects
				while line_index < len(lowercase				):
								# If the line is an if is_backend_available, we grab all objects associated.
								__lowercase					     =				find_backend(lines[line_index]				)
								# Check if the backend declaration is inside a try block:
								if _re_try.search(lines[line_index - 1]				) is None:
												__lowercase					     =				None

								if backend is not None:
												line_index += 1

												# Scroll until we hit the else block of try-except-else
												while _re_else.search(lines[line_index]				) is None:
																line_index += 1

												line_index += 1

												__lowercase					     =				[]
												# Until we unindent, add backend objects to the list
												while len(lines[line_index]				) <= 1 or lines[line_index].startswith(''' ''' * 8				):
																__lowercase					     =				lines[line_index]
																__lowercase					     =				_re_import.search(lowercase				)
																if single_line_import_search is not None:
																				objects.extend(single_line_import_search.groups()[0].split(''', '''				)				)
																elif line.startswith(''' ''' * 12				):
																				objects.append(line[12:-2]				)
																line_index += 1

												__lowercase					     =				objects
								else:
												line_index += 1

				return import_dict_objects, type_hint_objects





def 							UpperCAmelCase		(       lowercase				,     lowercase				):







				"""simple docstring"""



				def find_duplicates(lowercase				):
								return [k for k, v in collections.Counter(lowercase				).items() if v > 1]

				if list(import_dict_objects.keys()				) != list(type_hint_objects.keys()				):
								return ["Both sides of the init do not have the same backends!"]

				__lowercase					     =				[]
				for key in import_dict_objects.keys():
								__lowercase					     =				find_duplicates(import_dict_objects[key]				)
								if duplicate_imports:
												errors.append(F"Duplicate _import_structure definitions for: {duplicate_imports}"				)
								__lowercase					     =				find_duplicates(type_hint_objects[key]				)
								if duplicate_type_hints:
												errors.append(F"Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}"				)

								if sorted(set(import_dict_objects[key]				)				) != sorted(set(type_hint_objects[key]				)				):
												__lowercase					     =				'''base imports''' if key == '''none''' else F"{key} backend"
												errors.append(F"Differences for {name}:"				)
												for a in type_hint_objects[key]:
																if a not in import_dict_objects[key]:
																				errors.append(F"  {a} in TYPE_HINT but not in _import_structure."				)
												for a in import_dict_objects[key]:
																if a not in type_hint_objects[key]:
																				errors.append(F"  {a} in _import_structure but not in TYPE_HINT."				)
				return errors





def 							UpperCAmelCase		(       ):







				"""simple docstring"""



				__lowercase					     =				[]
				for root, _, files in os.walk(lowercase				):
								if "__init__.py" in files:
												__lowercase					     =				os.path.join(lowercase				,     '''__init__.py'''				)
												__lowercase					     =				parse_init(lowercase				)
												if objects is not None:
																__lowercase					     =				analyze_results(*lowercase				)
																if len(lowercase				) > 0:
																				__lowercase					     =				F"Problem in {fname}, both halves do not define the same objects.\n{errors[0]}"
																				failures.append('''\n'''.join(lowercase				)				)
				if len(lowercase				) > 0:
								raise ValueError('''\n\n'''.join(lowercase				)				)





def 							UpperCAmelCase		(       ):







				"""simple docstring"""



				__lowercase					     =				[]
				for path, directories, files in os.walk(lowercase				):
								for folder in directories:
												# Ignore private modules
												if folder.startswith('''_'''				):
																directories.remove(lowercase				)
																continue
												# Ignore leftovers from branches (empty folders apart from pycache)
												if len(list((Path(lowercase				) / folder).glob('''*.py'''				)				)				) == 0:
																continue
												__lowercase					     =				str((Path(lowercase				) / folder).relative_to(lowercase				)				)
												__lowercase					     =				short_path.replace(os.path.sep				,     '''.'''				)
												submodules.append(lowercase				)
								for fname in files:
												if fname == "__init__.py":
																continue
												__lowercase					     =				str((Path(lowercase				) / fname).relative_to(lowercase				)				)
												__lowercase					     =				short_path.replace('''.py'''				,     ''''''				).replace(os.path.sep				,     '''.'''				)
												if len(submodule.split('''.'''				)				) == 1:
																submodules.append(lowercase				)
				return submodules


__a    :	Union[str, Any]         =    [
    """convert_pytorch_checkpoint_to_tf2""",
    """modeling_flax_pytorch_utils""",
]





def 							UpperCAmelCase		(       ):







				"""simple docstring"""



				__lowercase					     =				importlib.util.spec_from_file_location(
				    '''transformers'''				,     os.path.join(lowercase				,     '''__init__.py'''				)				,     submodule_search_locations=[PATH_TO_TRANSFORMERS]				,     )
				__lowercase					     =				spec.loader.load_module()

				__lowercase					     =				[
				    module
				    for module in get_transformers_submodules()
				    if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys()
				]
				if len(lowercase				) > 0:
								__lowercase					     =				'''\n'''.join(F"- {module}" for module in module_not_registered				)
								raise ValueError(
								    '''The following submodules are not properly registered in the main init of Transformers:\n'''
								    F"{list_of_modules}\n"
								    '''Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value.'''				)


if __name__ == "__main__":
		check_all_inits()
		check_submodules()
import argparse
import json

import numpy
import torch

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()





def 							UpperCAmelCase		(       lowercase				,     lowercase				):







				"""simple docstring"""



				__lowercase					     =				torch.load(lowercase				,     map_location='''cpu'''				)

				__lowercase					     =				chkpt['''model''']

				# We have the base model one level deeper than the original XLM repository
				__lowercase					     =				{}
				for k, v in state_dict.items():
								if "pred_layer" in k:
												__lowercase					     =				v
								else:
												__lowercase					     =				v

				__lowercase					     =				chkpt['''params''']
				__lowercase					     =				{n: v for n, v in config.items() if not isinstance(lowercase				,     (torch.FloatTensor, numpy.ndarray)				)}

				__lowercase					     =				chkpt['''dico_word2id''']
				__lowercase					     =				{s + '''</w>''' if s.find('''@@'''				) == -1 and i > 13 else s.replace('''@@'''				,     ''''''				): i for s, i in vocab.items()}

				# Save pytorch-model
				__lowercase					     =				pytorch_dump_folder_path + '''/''' + WEIGHTS_NAME
				__lowercase					     =				pytorch_dump_folder_path + '''/''' + CONFIG_NAME
				__lowercase					     =				pytorch_dump_folder_path + '''/''' + VOCAB_FILES_NAMES['''vocab_file''']

				print(F"Save PyTorch model to {pytorch_weights_dump_path}"				)
				torch.save(lowercase				,     lowercase				)

				print(F"Save configuration file to {pytorch_config_dump_path}"				)
				with open(lowercase				,     '''w'''				,     encoding='''utf-8'''				) as f:
								f.write(json.dumps(lowercase				,     indent=2				) + '''\n'''				)

				print(F"Save vocab file to {pytorch_config_dump_path}"				)
				with open(lowercase				,     '''w'''				,     encoding='''utf-8'''				) as f:
								f.write(json.dumps(lowercase				,     indent=2				) + '''\n'''				)


if __name__ == "__main__":
		__a    :	Optional[int]         =    argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    """--xlm_checkpoint_path""", default=None, type=str, required=True, help="""Path the official PyTorch dump."""
		)
		parser.add_argument(
		    """--pytorch_dump_folder_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
		)
		__a    :	Any         =    parser.parse_args()
		convert_xlm_checkpoint_to_pytorch(args.xlm_checkpoint_path, args.pytorch_dump_folder_path)