




'''simple docstring'''


import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
				from cookiecutter.main import cookiecutter

				_snake_case : Union[str, Any]  				=						True
except ImportError:
				_snake_case : Tuple  				=						False

_snake_case : List[str]  				=						logging.get_logger(__name__)  # pylint: disable=invalid-name
def 						snake_case_       (UpperCamelCase			:    Namespace ):





							'''simple docstring'''


							return AddNewModelCommand(args.testing					,		args.testing_file					,		path=args.path )

class      A       (						_a       ):







							@staticmethod
							def   __lowerCAmelCase     (    lowerCAmelCase_			:    ArgumentParser   )     ->				Any:


														"""simple docstring"""


														_a      	=							parser.add_parser('''add-new-model'''   )
														add_new_model_parser.add_argument('''--testing'''				,     action='''store_true'''				,     help='''If in testing mode.'''   )
														add_new_model_parser.add_argument('''--testing_file'''				,     type=lowerCAmelCase_				,     help='''Configuration file on which to run.'''   )
														add_new_model_parser.add_argument(
														    '''--path'''				,     type=lowerCAmelCase_				,     help='''Path to cookiecutter. Should only be used for testing purposes.'''   )
														add_new_model_parser.set_defaults(func=lowerCAmelCase_   )







							def __init__(    self			:    Optional[Any]				,     lowerCAmelCase_			:    bool				,     lowerCAmelCase_			:    str				,     lowerCAmelCase_			:    List[str]=None				,     *lowerCAmelCase_			:    Tuple   )     ->				Optional[int]:


														"""simple docstring"""


														_a      	=							testing
														_a      	=							testing_file
														_a      	=							path







							def   __lowerCAmelCase     (    self			:    Union[str, Any]   )     ->				Dict:


														"""simple docstring"""


														warnings.warn(
														    '''The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. '''
														    '''It is not actively maintained anymore, so might give a result that won\'t pass all tests and quality '''
														    '''checks, you should use `transformers-cli add-new-model-like` instead.'''   )
														if not _has_cookiecutter:
																					raise ImportError(
																					    '''Model creation dependencies are required to use the `add_new_model` command. Install them by running '''
																					    '''the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n'''   )
														# Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
														_a      	=							[directory for directory in os.listdir() if '''cookiecutter-template-''' == directory[:22]]
														if len(lowerCAmelCase_   ) > 0:
																					raise ValueError(
																					    '''Several directories starting with `cookiecutter-template-` in current working directory. '''
																					    '''Please clean your directory by removing all folders starting with `cookiecutter-template-` or '''
																					    '''change your working directory.'''   )

														_a      	=							(
														    Path(lowerCAmelCase_   ).parent.parent.parent.parent if self._path is None else Path(self._path   ).parent.parent
														)
														_a      	=							path_to_transformer_root / '''templates''' / '''adding_a_new_model'''

														# Execute cookiecutter
														if not self._testing:
																					cookiecutter(str(lowerCAmelCase_   )   )
														else:
																					with open(self._testing_file				,     '''r'''   ) as configuration_file:
																												_a      	=							json.load(lowerCAmelCase_   )

																					cookiecutter(
																					    str(path_to_cookiecutter if self._path is None else self._path   )				,     no_input=lowerCAmelCase_				,     extra_context=lowerCAmelCase_				,     )

														_a      	=							[directory for directory in os.listdir() if '''cookiecutter-template-''' in directory[:22]][0]

														# Retrieve configuration
														with open(directory + '''/configuration.json'''				,     '''r'''   ) as configuration_file:
																					_a      	=							json.load(lowerCAmelCase_   )

														_a      	=							configuration['''lowercase_modelname''']
														_a      	=							configuration['''generate_tensorflow_pytorch_and_flax''']
														os.remove(F'{directory}/configuration.json'   )

														_a      	=							'''PyTorch''' in generate_tensorflow_pytorch_and_flax
														_a      	=							'''TensorFlow''' in generate_tensorflow_pytorch_and_flax
														_a      	=							'''Flax''' in generate_tensorflow_pytorch_and_flax

														_a      	=							F'{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}'
														os.makedirs(lowerCAmelCase_				,     exist_ok=lowerCAmelCase_   )
														os.makedirs(F'{path_to_transformer_root}/tests/models/{lowercase_model_name}'				,     exist_ok=lowerCAmelCase_   )

														# Tests require submodules as they have parent imports
														with open(F'{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py'				,     '''w'''   ):
																					pass

														shutil.move(
														    F'{directory}/__init__.py'				,     F'{model_dir}/__init__.py'				,     )
														shutil.move(
														    F'{directory}/configuration_{lowercase_model_name}.py'				,     F'{model_dir}/configuration_{lowercase_model_name}.py'				,     )

														def remove_copy_lines(lowerCAmelCase_			:    int   ):
																					with open(lowerCAmelCase_				,     '''r'''   ) as f:
																												_a      	=							f.readlines()
																					with open(lowerCAmelCase_				,     '''w'''   ) as f:
																												for line in lines:
																																			if "# Copied from transformers." not in line:
																																										f.write(lowerCAmelCase_   )

														if output_pytorch:
																					if not self._testing:
																												remove_copy_lines(F'{directory}/modeling_{lowercase_model_name}.py'   )

																					shutil.move(
																					    F'{directory}/modeling_{lowercase_model_name}.py'				,     F'{model_dir}/modeling_{lowercase_model_name}.py'				,     )

																					shutil.move(
																					    F'{directory}/test_modeling_{lowercase_model_name}.py'				,     F'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py'				,     )
														else:
																					os.remove(F'{directory}/modeling_{lowercase_model_name}.py'   )
																					os.remove(F'{directory}/test_modeling_{lowercase_model_name}.py'   )

														if output_tensorflow:
																					if not self._testing:
																												remove_copy_lines(F'{directory}/modeling_tf_{lowercase_model_name}.py'   )

																					shutil.move(
																					    F'{directory}/modeling_tf_{lowercase_model_name}.py'				,     F'{model_dir}/modeling_tf_{lowercase_model_name}.py'				,     )

																					shutil.move(
																					    F'{directory}/test_modeling_tf_{lowercase_model_name}.py'				,     F'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py'				,     )
														else:
																					os.remove(F'{directory}/modeling_tf_{lowercase_model_name}.py'   )
																					os.remove(F'{directory}/test_modeling_tf_{lowercase_model_name}.py'   )

														if output_flax:
																					if not self._testing:
																												remove_copy_lines(F'{directory}/modeling_flax_{lowercase_model_name}.py'   )

																					shutil.move(
																					    F'{directory}/modeling_flax_{lowercase_model_name}.py'				,     F'{model_dir}/modeling_flax_{lowercase_model_name}.py'				,     )

																					shutil.move(
																					    F'{directory}/test_modeling_flax_{lowercase_model_name}.py'				,     F'{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py'				,     )
														else:
																					os.remove(F'{directory}/modeling_flax_{lowercase_model_name}.py'   )
																					os.remove(F'{directory}/test_modeling_flax_{lowercase_model_name}.py'   )

														shutil.move(
														    F'{directory}/{lowercase_model_name}.md'				,     F'{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md'				,     )

														shutil.move(
														    F'{directory}/tokenization_{lowercase_model_name}.py'				,     F'{model_dir}/tokenization_{lowercase_model_name}.py'				,     )

														shutil.move(
														    F'{directory}/tokenization_fast_{lowercase_model_name}.py'				,     F'{model_dir}/tokenization_{lowercase_model_name}_fast.py'				,     )

														from os import fdopen, remove
														from shutil import copymode, move
														from tempfile import mkstemp

														def replace(lowerCAmelCase_			:    str				,     lowerCAmelCase_			:    str				,     lowerCAmelCase_			:    List[str]   ):
																					# Create temp file
																					_a	,				_a      	=							mkstemp()
																					_a      	=							False
																					with fdopen(lowerCAmelCase_				,     '''w'''   ) as new_file:
																												with open(lowerCAmelCase_   ) as old_file:
																																			for line in old_file:
																																										new_file.write(lowerCAmelCase_   )
																																										if line_to_copy_below in line:
																																																	_a      	=							True
																																																	for line_to_copy in lines_to_copy:
																																																								new_file.write(lowerCAmelCase_   )

																					if not line_found:
																												raise ValueError(F'Line {line_to_copy_below} was not found in file.'   )

																					# Copy the file permissions from the old file to the new file
																					copymode(lowerCAmelCase_				,     lowerCAmelCase_   )
																					# Remove original file
																					remove(lowerCAmelCase_   )
																					# Move new file
																					move(lowerCAmelCase_				,     lowerCAmelCase_   )

														def skip_units(lowerCAmelCase_			:    str   ):
																					return (
																					    ("generating PyTorch" in line and not output_pytorch)
																					    or ("generating TensorFlow" in line and not output_tensorflow)
																					    or ("generating Flax" in line and not output_flax)
																					)

														def replace_in_files(lowerCAmelCase_			:    Optional[Any]   ):
																					with open(lowerCAmelCase_   ) as datafile:
																												_a      	=							[]
																												_a      	=							False
																												_a      	=							False
																												for line in datafile:
																																			if "# To replace in: " in line and "##" not in line:
																																										_a      	=							line.split('''"'''   )[1]
																																										_a      	=							skip_units(lowerCAmelCase_   )
																																			elif "# Below: " in line and "##" not in line:
																																										_a      	=							line.split('''"'''   )[1]
																																										_a      	=							skip_units(lowerCAmelCase_   )
																																			elif "# End." in line and "##" not in line:
																																										if not skip_file and not skip_snippet:
																																																	replace(lowerCAmelCase_				,     lowerCAmelCase_				,     lowerCAmelCase_   )

																																										_a      	=							[]
																																			elif "# Replace with" in line and "##" not in line:
																																										_a      	=							[]
																																			elif "##" not in line:
																																										lines_to_copy.append(lowerCAmelCase_   )

																					remove(lowerCAmelCase_   )

														replace_in_files(F'{directory}/to_replace_{lowercase_model_name}.py'   )
														os.rmdir(lowerCAmelCase_   )




'''simple docstring'''





import re

import jax.numpy as jnp
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.random import PRNGKey

from ..utils import logging


UpperCamelCase_    						=       logging.get_logger(__name__)






def 				_lowerCAmelCase			(   __magic_name__						:     List[str]						)				->  Dict:
						lowercase :							List[str]    		=R'''\w+[.]\d+'''
						lowercase :							List[str]    		=re.findall(__magic_name__   , __magic_name__						)
						for pat in pats:
												lowercase :							Optional[int]    		=key.replace(__magic_name__   , '''_'''.join(pat.split('''.'''						)						)						)
						return key






def 				_lowerCAmelCase			(   __magic_name__						:     List[str]   , __magic_name__						:     Union[str, Any]   , __magic_name__						:     Dict						)				->  str:
						lowercase :							Dict    		=pt_tuple_key[:-1] + ('''scale''',)
						if (
						    any('''norm''' in str_ for str_ in pt_tuple_key						)
						    and (pt_tuple_key[-1] == "bias")
						    and (pt_tuple_key[:-1] + ("bias",) not in random_flax_state_dict)
						    and (pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict)
						):
												lowercase :							str    		=pt_tuple_key[:-1] + ('''scale''',)
												return renamed_pt_tuple_key, pt_tensor
						elif pt_tuple_key[-1] in ["weight", "gamma"] and pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict:
												lowercase :							str    		=pt_tuple_key[:-1] + ('''scale''',)
												return renamed_pt_tuple_key, pt_tensor

						# embedding
						if pt_tuple_key[-1] == "weight" and pt_tuple_key[:-1] + ("embedding",) in random_flax_state_dict:
												lowercase :							Dict    		=pt_tuple_key[:-1] + ('''embedding''',)
												return renamed_pt_tuple_key, pt_tensor

						# conv layer
						lowercase :							Tuple    		=pt_tuple_key[:-1] + ('''kernel''',)
						if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4:
												lowercase :							Tuple    		=pt_tensor.transpose(2   , 3   , 1   , 0						)
												return renamed_pt_tuple_key, pt_tensor

						# linear layer
						lowercase :							str    		=pt_tuple_key[:-1] + ('''kernel''',)
						if pt_tuple_key[-1] == "weight":
												lowercase :							Optional[Any]    		=pt_tensor.T
												return renamed_pt_tuple_key, pt_tensor

						# old PyTorch layer norm weight
						lowercase :							Dict    		=pt_tuple_key[:-1] + ('''weight''',)
						if pt_tuple_key[-1] == "gamma":
												return renamed_pt_tuple_key, pt_tensor

						# old PyTorch layer norm bias
						lowercase :							Union[str, Any]    		=pt_tuple_key[:-1] + ('''bias''',)
						if pt_tuple_key[-1] == "beta":
												return renamed_pt_tuple_key, pt_tensor

						return pt_tuple_key, pt_tensor







def 				_lowerCAmelCase			(   __magic_name__						:     int   , __magic_name__						:     Optional[Any]   , __magic_name__						:     Union[str, Any]=42						)				->  List[str]:
						# Step 1: Convert pytorch tensor to numpy
						lowercase :							Optional[Any]    		={k: v.numpy() for k, v in pt_state_dict.items()}

						# Step 2: Since the model is stateless, get random Flax params
						lowercase :							str    		=flax_model.init_weights(PRNGKey(__magic_name__						)						)

						lowercase :							Dict    		=flatten_dict(__magic_name__						)
						lowercase :							Dict    		={}

						# Need to change some parameters name to match Flax names
						for pt_key, pt_tensor in pt_state_dict.items():
												lowercase :							Dict    		=rename_key(__magic_name__						)
												lowercase :							Optional[int]    		=tuple(renamed_pt_key.split('''.'''						)						)

												# Correctly rename weight parameters
												lowercase		,	lowercase :							Any    		=rename_key_and_reshape_tensor(__magic_name__   , __magic_name__   , __magic_name__						)

												if flax_key in random_flax_state_dict:
																		if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
																								raise ValueError(
																								    f'''PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape '''
																								    f'''{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}.'''						)

        # also add unexpected weight so that warning is thrown
												lowercase :							Tuple    		=jnp.asarray(__magic_name__						)

						return unflatten_dict(__magic_name__						)





