

import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
	from transformers import LlamaTokenizerFast
except ImportError as e:
	warnings.warn(e)
	warnings.warn(
	    'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
	)
	A_			:					Union[str, Any]    			=     None
A_			:					Dict    			=     {
    '7B': 1_1008,
    '13B': 1_3824,
    '30B': 1_7920,
    '65B': 2_2016,
    '70B': 2_8672,
}
A_			:					str    			=     {
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}

def 		UpperCamelCase   (lowercase_:					str       , lowercase_:					int=1       , lowercase_:					List[Any]=256   )	-> str:
	return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3   )   ) + multiple_of - 1) // multiple_of)

def 		UpperCamelCase   (lowercase_:					List[str]   )	-> List[str]:
	with open(lowercase_       , """r"""   ) as f:
		return json.load(lowercase_   )

def 		UpperCamelCase   (lowercase_:					Union[str, Any]       , lowercase_:					Tuple   )	-> List[Any]:
	with open(lowercase_       , """w"""   ) as f:
		json.dump(lowercase_       , lowercase_   )

def 		UpperCamelCase   (lowercase_:					int       , lowercase_:					int       , lowercase_:					Union[str, Any]       , lowercase_:					str=True   )	-> str:
	os.makedirs(lowercase_       , exist_ok=lowercase_   )
	A__				:       str	  =						os.path.join(lowercase_       , """tmp"""   )
	os.makedirs(lowercase_       , exist_ok=lowercase_   )

	A__				:       Any	  =						read_json(os.path.join(lowercase_       , """params.json"""   )   )
	A__				:       Optional[Any]	  =						NUM_SHARDS[model_size]
	A__				:       int	  =						params["""n_layers"""]
	A__				:       List[Any]	  =						params["""n_heads"""]
	A__				:       Tuple	  =						n_heads // num_shards
	A__				:       Union[str, Any]	  =						params["""dim"""]
	A__				:       int	  =						dim // n_heads
	A__				:       int	  =						1_0000.0
	A__				:       List[str]	  =						1.0 / (base ** (torch.arange(0       , lowercase_       , 2   ).float() / dims_per_head))

	if "n_kv_heads" in params:
		A__				:       List[str]	  =						params["""n_kv_heads"""]  # for GQA / MQA
		A__				:       Optional[Any]	  =						n_heads_per_shard // num_key_value_heads
		A__				:       List[Any]	  =						dim // num_key_value_heads
	else:  # compatibility with other checkpoints
		A__				:       Union[str, Any]	  =						n_heads
		A__				:       Optional[Any]	  =						n_heads_per_shard
		A__				:       Optional[int]	  =						dim

	# permute for sliced rotary
	def permute(lowercase_:					Optional[Any]       , lowercase_:					Tuple=n_heads       , lowercase_:					List[str]=dim       , lowercase_:					Tuple=dim   ):
		return w.view(lowercase_       , dima // n_heads // 2       , 2       , lowercase_   ).transpose(1       , 2   ).reshape(lowercase_       , lowercase_   )

	print(f"""Fetching all parameters from the checkpoint at {input_base_path}."""   )
	# Load weights
	if model_size == "7B":
		# Not sharded
		# (The sharded implementation would also work, but this is simpler.)
		A__				:       List[str]	  =						torch.load(os.path.join(lowercase_       , """consolidated.00.pth"""   )       , map_location="""cpu"""   )
	else:
		# Sharded
		A__				:       int	  =						[
		    torch.load(os.path.join(lowercase_       , f"""consolidated.{i:02d}.pth"""   )       , map_location="""cpu"""   )
		    for i in range(lowercase_   )
		]
	A__				:       Optional[Any]	  =						0
	A__				:       List[str]	  =						{"""weight_map""": {}}
	for layer_i in range(lowercase_   ):
		A__				:       Dict	  =						f"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
		if model_size == "7B":
			# Unsharded
			A__				:       Optional[int]	  =						{
			    f"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
			        loaded[f"""layers.{layer_i}.attention.wq.weight"""]   ),
			    f"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
			        loaded[f"""layers.{layer_i}.attention.wk.weight"""]   ),
			    f"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[f"""layers.{layer_i}.attention.wv.weight"""],
			    f"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[f"""layers.{layer_i}.attention.wo.weight"""],
			    f"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w1.weight"""],
			    f"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w2.weight"""],
			    f"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w3.weight"""],
			    f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[f"""layers.{layer_i}.attention_norm.weight"""],
			    f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[f"""layers.{layer_i}.ffn_norm.weight"""],
			}
		else:
			# Sharded
			# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
			# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
			# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

			A__				:       Tuple	  =						{
			    f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
			        f"""layers.{layer_i}.attention_norm.weight"""
			    ].clone(),
			    f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
			        f"""layers.{layer_i}.ffn_norm.weight"""
			    ].clone(),
			}
			A__				:       Tuple	  =						permute(
			    torch.cat(
			        [
			            loaded[i][f"""layers.{layer_i}.attention.wq.weight"""].view(lowercase_       , lowercase_       , lowercase_   )
			            for i in range(lowercase_   )
			        ]       , dim=0       , ).reshape(lowercase_       , lowercase_   )   )
			A__				:       Dict	  =						permute(
			    torch.cat(
			        [
			            loaded[i][f"""layers.{layer_i}.attention.wk.weight"""].view(
			                lowercase_       , lowercase_       , lowercase_   )
			            for i in range(lowercase_   )
			        ]       , dim=0       , ).reshape(lowercase_       , lowercase_   )       , lowercase_       , lowercase_       , lowercase_       , )
			A__				:       Optional[Any]	  =						torch.cat(
			    [
			        loaded[i][f"""layers.{layer_i}.attention.wv.weight"""].view(
			            lowercase_       , lowercase_       , lowercase_   )
			        for i in range(lowercase_   )
			    ]       , dim=0       , ).reshape(lowercase_       , lowercase_   )

			A__				:       Optional[Any]	  =						torch.cat(
			    [loaded[i][f"""layers.{layer_i}.attention.wo.weight"""] for i in range(lowercase_   )]       , dim=1   )
			A__				:       str	  =						torch.cat(
			    [loaded[i][f"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(lowercase_   )]       , dim=0   )
			A__				:       Optional[int]	  =						torch.cat(
			    [loaded[i][f"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(lowercase_   )]       , dim=1   )
			A__				:       Optional[int]	  =						torch.cat(
			    [loaded[i][f"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(lowercase_   )]       , dim=0   )

		A__				:       Dict	  =						inv_freq
		for k, v in state_dict.items():
			A__				:       Any	  =						filename
			param_count += v.numel()
		torch.save(lowercase_       , os.path.join(lowercase_       , lowercase_   )   )

	A__				:       Tuple	  =						f"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
	if model_size == "7B":
		# Unsharded
		A__				:       Dict	  =						{
		    """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
		    """model.norm.weight""": loaded["""norm.weight"""],
		    """lm_head.weight""": loaded["""output.weight"""],
		}
	else:
		A__				:       Tuple	  =						{
		    """model.norm.weight""": loaded[0]["""norm.weight"""],
		    """model.embed_tokens.weight""": torch.cat(
		        [loaded[i]["""tok_embeddings.weight"""] for i in range(lowercase_   )]       , dim=1   ),
		    """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(lowercase_   )]       , dim=0   ),
		}

	for k, v in state_dict.items():
		A__				:       Any	  =						filename
		param_count += v.numel()
	torch.save(lowercase_       , os.path.join(lowercase_       , lowercase_   )   )

	# Write configs
	A__				:       Optional[int]	  =						{"""total_size""": param_count * 2}
	write_json(lowercase_       , os.path.join(lowercase_       , """pytorch_model.bin.index.json"""   )   )
	A__				:       Any	  =						params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
	A__				:       Tuple	  =						params["""multiple_of"""] if """multiple_of""" in params else 256
	A__				:       int	  =						LlamaConfig(
	    hidden_size=lowercase_       , intermediate_size=compute_intermediate_size(lowercase_       , lowercase_       , lowercase_   )       , num_attention_heads=params["""n_heads"""]       , num_hidden_layers=params["""n_layers"""]       , rms_norm_eps=params["""norm_eps"""]       , num_key_value_heads=lowercase_       , )
	config.save_pretrained(lowercase_   )

	# Make space so we can load the model properly now.
	del state_dict
	del loaded
	gc.collect()

	print("""Loading the checkpoint in a Llama model."""   )
	A__				:       Any	  =						LlamaForCausalLM.from_pretrained(lowercase_       , torch_dtype=torch.floataa       , low_cpu_mem_usage=lowercase_   )
	# Avoid saving this as part of the config.
	del model.config._name_or_path

	print("""Saving in the Transformers format."""   )
	model.save_pretrained(lowercase_       , safe_serialization=lowercase_   )
	shutil.rmtree(lowercase_   )

def 		UpperCamelCase   (lowercase_:					str       , lowercase_:					List[str]   )	-> List[Any]:
	# Initialize the tokenizer based on the `spm` model
	A__				:       Optional[Any]	  =						LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
	print(f"""Saving a {tokenizer_class.__name__} to {tokenizer_path}."""   )
	A__				:       List[str]	  =						tokenizer_class(lowercase_   )
	tokenizer.save_pretrained(lowercase_   )

def 		UpperCamelCase   ()	-> Optional[int]:
	A__				:       Any	  =						argparse.ArgumentParser()
	parser.add_argument(
	    """--input_dir"""       , help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""       , )
	parser.add_argument(
	    """--model_size"""       , choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]       , )
	parser.add_argument(
	    """--output_dir"""       , help="""Location to write HF model and tokenizer"""       , )
	parser.add_argument("""--safe_serialization"""       , type=lowercase_       , help="""Whether or not to save using `safetensors`."""   )
	A__				:       List[str]	  =						parser.parse_args()
	if args.model_size != "tokenizer_only":
		write_model(
		    model_path=args.output_dir       , input_base_path=os.path.join(args.input_dir       , args.model_size   )       , model_size=args.model_size       , safe_serialization=args.safe_serialization       , )
	A__				:       Optional[Any]	  =						os.path.join(args.input_dir       , """tokenizer.model"""   )
	write_tokenizer(args.output_dir       , lowercase_   )


if __name__ == "__main__":
	main()






import collections
import os
import re
from pathlib import Path


A_			:					List[str]    			=     'src/transformers'


# Matches is_xxx_available()
A_			:					Any    			=     re.compile(r'is\_([a-z_]*)_available()')
# Catches a one-line _import_struct = {xxx}
A_			:					Optional[int]    			=     re.compile(r'^_import_structure\s+=\s+\{([^\}]+)\}')
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
A_			:					Dict    			=     re.compile(r'\s+"\S*":\s+\[([^\]]*)\]')
# Catches a line if not is_foo_available
A_			:					Dict    			=     re.compile(r'^\s*if\s+not\s+is\_[a-z_]*\_available\(\)')
# Catches a line _import_struct["bla"].append("foo")
A_			:					Tuple    			=     re.compile(r'^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)')
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
A_			:					List[Any]    			=     re.compile(r'^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]')
# Catches a line with an object between quotes and a comma:     "MyModel",
A_			:					Dict    			=     re.compile(r'^\s+"([^"]+)",')
# Catches a line with objects between brackets only:    ["foo", "bar"],
A_			:					Tuple    			=     re.compile(r'^\s+\[([^\]]+)\]')
# Catches a line with from foo import bar, bla, boo
A_			:					Union[str, Any]    			=     re.compile(r'\s+from\s+\S*\s+import\s+([^\(\s].*)\n')
# Catches a line with try:
A_			:					Any    			=     re.compile(r'^\s*try:')
# Catches a line with else:
A_			:					Optional[Any]    			=     re.compile(r'^\s*else:')

def 		UpperCamelCase   (lowercase_:					Optional[Any]   )	-> Any:
	if _re_test_backend.search(lowercase_   ) is None:
		return None
	A__				:       Optional[int]	  =						[b[0] for b in _re_backend.findall(lowercase_   )]
	backends.sort()
	return "_and_".join(lowercase_   )

def 		UpperCamelCase   (lowercase_:					Any   )	-> Dict:
	with open(lowercase_       , """r"""       , encoding="""utf-8"""       , newline="""\n"""   ) as f:
		A__				:       Optional[Any]	  =						f.readlines()

	A__				:       Optional[Any]	  =						0
	while line_index < len(lowercase_   ) and not lines[line_index].startswith("""_import_structure = {"""   ):
		line_index += 1

	# If this is a traditional init, just return.
	if line_index >= len(lowercase_   ):
		return None

	# First grab the objects without a specific backend in _import_structure
	A__				:       List[Any]	  =						[]
	while not lines[line_index].startswith("""if TYPE_CHECKING"""   ) and find_backend(lines[line_index]   ) is None:
		A__				:       Tuple	  =						lines[line_index]
		# If we have everything on a single line, let's deal with it.
		if _re_one_line_import_struct.search(lowercase_   ):
			A__				:       str	  =						_re_one_line_import_struct.search(lowercase_   ).groups()[0]
			A__				:       Union[str, Any]	  =						re.findall(r"""\[([^\]]+)\]"""       , lowercase_   )
			for imp in imports:
				objects.extend([obj[1:-1] for obj in imp.split(""", """   )]   )
			line_index += 1
			continue
		A__				:       int	  =						_re_import_struct_key_value.search(lowercase_   )
		if single_line_import_search is not None:
			A__				:       int	  =						[obj[1:-1] for obj in single_line_import_search.groups()[0].split(""", """   ) if len(lowercase_   ) > 0]
			objects.extend(lowercase_   )
		elif line.startswith(""" """ * 8 + """\""""   ):
			objects.append(line[9:-3]   )
		line_index += 1

	A__				:       str	  =						{"""none""": objects}
	# Let's continue with backend-specific objects in _import_structure
	while not lines[line_index].startswith("""if TYPE_CHECKING"""   ):
		# If the line is an if not is_backend_available, we grab all objects associated.
		A__				:       Optional[int]	  =						find_backend(lines[line_index]   )
		# Check if the backend declaration is inside a try block:
		if _re_try.search(lines[line_index - 1]   ) is None:
			A__				:       List[str]	  =						None

		if backend is not None:
			line_index += 1

			# Scroll until we hit the else block of try-except-else
			while _re_else.search(lines[line_index]   ) is None:
				line_index += 1

			line_index += 1

			A__				:       Tuple	  =						[]
			# Until we unindent, add backend objects to the list
			while len(lines[line_index]   ) <= 1 or lines[line_index].startswith(""" """ * 4   ):
				A__				:       Any	  =						lines[line_index]
				if _re_import_struct_add_one.search(lowercase_   ) is not None:
					objects.append(_re_import_struct_add_one.search(lowercase_   ).groups()[0]   )
				elif _re_import_struct_add_many.search(lowercase_   ) is not None:
					A__				:       Any	  =						_re_import_struct_add_many.search(lowercase_   ).groups()[0].split(""", """   )
					A__				:       int	  =						[obj[1:-1] for obj in imports if len(lowercase_   ) > 0]
					objects.extend(lowercase_   )
				elif _re_between_brackets.search(lowercase_   ) is not None:
					A__				:       Any	  =						_re_between_brackets.search(lowercase_   ).groups()[0].split(""", """   )
					A__				:       Any	  =						[obj[1:-1] for obj in imports if len(lowercase_   ) > 0]
					objects.extend(lowercase_   )
				elif _re_quote_object.search(lowercase_   ) is not None:
					objects.append(_re_quote_object.search(lowercase_   ).groups()[0]   )
				elif line.startswith(""" """ * 8 + """\""""   ):
					objects.append(line[9:-3]   )
				elif line.startswith(""" """ * 12 + """\""""   ):
					objects.append(line[13:-3]   )
				line_index += 1

			A__				:       List[str]	  =						objects
		else:
			line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
	A__				:       Any	  =						[]
	while (
	    line_index < len(lowercase_   )
	    and find_backend(lines[line_index]   ) is None
	    and not lines[line_index].startswith("""else"""   )
	):
		A__				:       Dict	  =						lines[line_index]
		A__				:       Any	  =						_re_import.search(lowercase_   )
		if single_line_import_search is not None:
			objects.extend(single_line_import_search.groups()[0].split(""", """   )   )
		elif line.startswith(""" """ * 8   ):
			objects.append(line[8:-2]   )
		line_index += 1

	A__				:       List[str]	  =						{"""none""": objects}
	# Let's continue with backend-specific objects
	while line_index < len(lowercase_   ):
		# If the line is an if is_backend_available, we grab all objects associated.
		A__				:       Optional[Any]	  =						find_backend(lines[line_index]   )
		# Check if the backend declaration is inside a try block:
		if _re_try.search(lines[line_index - 1]   ) is None:
			A__				:       Any	  =						None

		if backend is not None:
			line_index += 1

			# Scroll until we hit the else block of try-except-else
			while _re_else.search(lines[line_index]   ) is None:
				line_index += 1

			line_index += 1

			A__				:       List[str]	  =						[]
			# Until we unindent, add backend objects to the list
			while len(lines[line_index]   ) <= 1 or lines[line_index].startswith(""" """ * 8   ):
				A__				:       Union[str, Any]	  =						lines[line_index]
				A__				:       List[str]	  =						_re_import.search(lowercase_   )
				if single_line_import_search is not None:
					objects.extend(single_line_import_search.groups()[0].split(""", """   )   )
				elif line.startswith(""" """ * 12   ):
					objects.append(line[12:-2]   )
				line_index += 1

			A__				:       Tuple	  =						objects
		else:
			line_index += 1

	return import_dict_objects, type_hint_objects

def 		UpperCamelCase   (lowercase_:					Optional[Any]       , lowercase_:					Union[str, Any]   )	-> List[Any]:

	def find_duplicates(lowercase_:					Tuple   ):
		return [k for k, v in collections.Counter(lowercase_   ).items() if v > 1]

	if list(import_dict_objects.keys()   ) != list(type_hint_objects.keys()   ):
		return ["Both sides of the init do not have the same backends!"]

	A__				:       str	  =						[]
	for key in import_dict_objects.keys():
		A__				:       Optional[int]	  =						find_duplicates(import_dict_objects[key]   )
		if duplicate_imports:
			errors.append(f"""Duplicate _import_structure definitions for: {duplicate_imports}"""   )
		A__				:       Tuple	  =						find_duplicates(type_hint_objects[key]   )
		if duplicate_type_hints:
			errors.append(f"""Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}"""   )

		if sorted(set(import_dict_objects[key]   )   ) != sorted(set(type_hint_objects[key]   )   ):
			A__				:       Tuple	  =						"""base imports""" if key == """none""" else f"""{key} backend"""
			errors.append(f"""Differences for {name}:"""   )
			for a in type_hint_objects[key]:
				if a not in import_dict_objects[key]:
					errors.append(f"""  {a} in TYPE_HINT but not in _import_structure."""   )
			for a in import_dict_objects[key]:
				if a not in type_hint_objects[key]:
					errors.append(f"""  {a} in _import_structure but not in TYPE_HINT."""   )
	return errors

def 		UpperCamelCase   ()	-> str:
	A__				:       str	  =						[]
	for root, _, files in os.walk(lowercase_   ):
		if "__init__.py" in files:
			A__				:       Tuple	  =						os.path.join(lowercase_       , """__init__.py"""   )
			A__				:       Union[str, Any]	  =						parse_init(lowercase_   )
			if objects is not None:
				A__				:       List[Any]	  =						analyze_results(*lowercase_   )
				if len(lowercase_   ) > 0:
					A__				:       int	  =						f"""Problem in {fname}, both halves do not define the same objects.\n{errors[0]}"""
					failures.append("""\n""".join(lowercase_   )   )
	if len(lowercase_   ) > 0:
		raise ValueError("""\n\n""".join(lowercase_   )   )

def 		UpperCamelCase   ()	-> Dict:
	A__				:       int	  =						[]
	for path, directories, files in os.walk(lowercase_   ):
		for folder in directories:
			# Ignore private modules
			if folder.startswith("""_"""   ):
				directories.remove(lowercase_   )
				continue
			# Ignore leftovers from branches (empty folders apart from pycache)
			if len(list((Path(lowercase_   ) / folder).glob("""*.py"""   )   )   ) == 0:
				continue
			A__				:       Union[str, Any]	  =						str((Path(lowercase_   ) / folder).relative_to(lowercase_   )   )
			A__				:       List[str]	  =						short_path.replace(os.path.sep       , """."""   )
			submodules.append(lowercase_   )
		for fname in files:
			if fname == "__init__.py":
				continue
			A__				:       Any	  =						str((Path(lowercase_   ) / fname).relative_to(lowercase_   )   )
			A__				:       Union[str, Any]	  =						short_path.replace(""".py"""       , """"""   ).replace(os.path.sep       , """."""   )
			if len(submodule.split("""."""   )   ) == 1:
				submodules.append(lowercase_   )
	return submodules


A_			:					str    			=     [
    'convert_pytorch_checkpoint_to_tf2',
    'modeling_flax_pytorch_utils',
    'models.esm.openfold_utils',
]

def 		UpperCamelCase   ()	-> str:
	# This is to make sure the transformers module imported is the one in the repo.
	from transformers.utils import direct_transformers_import

	A__				:       Any	  =						direct_transformers_import(lowercase_   )

	A__				:       List[str]	  =						set(transformers._import_structure.keys()   )
	# This contains all the base keys of the _import_structure object defined in the init, but if the user is missing
	# some optional dependencies, they may not have all of them. Thus we read the init to read all additions and
	# (potentiall re-) add them.
	with open(os.path.join(lowercase_       , """__init__.py"""   )       , """r"""   ) as f:
		A__				:       str	  =						f.read()
	import_structure_keys.update(set(re.findall(r"""import_structure\[\"([^\"]*)\"\]"""       , lowercase_   )   )   )

	A__				:       Union[str, Any]	  =						[
	    module
	    for module in get_transformers_submodules()
	    if module not in IGNORE_SUBMODULES and module not in import_structure_keys
	]

	if len(lowercase_   ) > 0:
		A__				:       Dict	  =						"""\n""".join(f"""- {module}""" for module in module_not_registered   )
		raise ValueError(
		    """The following submodules are not properly registed in the main init of Transformers:\n"""
		    f"""{list_of_modules}\n"""
		    """Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value."""   )


if __name__ == "__main__":
	check_all_inits()
	check_submodules()



