


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


snake_case_						:			Optional[Any]													=							logging.get_logger(__name__)

snake_case_						:			Union[str, Any]													=							{'''vocab_file''': '''spiece.model'''}

snake_case_						:			List[Any]													=							{
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

snake_case_						:			Union[str, Any]													=							{
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
snake_case_						:			str													=							0
snake_case_						:			Tuple													=							1
snake_case_						:			List[str]													=							2
snake_case_						:			Optional[Any]													=							3
snake_case_						:			Any													=							4

class 						A__							(		UpperCamelCase__  ):
						UpperCAmelCase             =						VOCAB_FILES_NAMES
						UpperCAmelCase             =						PRETRAINED_VOCAB_FILES_MAP
						UpperCAmelCase             =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCAmelCase             =						"left"




						def __init__(       self					:      int				,    _a					:      Any				,    _a					:      Dict=False				,    _a					:      str=True				,    _a					:      List[Any]=False				,    _a					:      int="<s>"				,    _a					:      Union[str, Any]="</s>"				,    _a					:      str="<unk>"				,    _a					:      str="<sep>"				,    _a					:      List[Any]="<pad>"				,    _a					:      Tuple="<cls>"				,    _a					:      Union[str, Any]="<mask>"				,    _a					:      Union[str, Any]=["<eop>", "<eod>"]				,    _a					:      Optional[Dict[str, Any]] = None				,    **_a					:      Tuple				,    )							->     None:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =AddedToken(_a				,    lstrip=_a				,    rstrip=_a  ) if isinstance(_a				,    _a  ) else mask_token

										_SCREAMING_SNAKE_CASE			  ={} if sp_model_kwargs is None else sp_model_kwargs

										super().__init__(
										    do_lower_case=_a				,    remove_space=_a				,    keep_accents=_a				,    bos_token=_a				,    eos_token=_a				,    unk_token=_a				,    sep_token=_a				,    pad_token=_a				,    cls_token=_a				,    mask_token=_a				,    additional_special_tokens=_a				,    sp_model_kwargs=self.sp_model_kwargs				,    **_a				,    )

										_SCREAMING_SNAKE_CASE			  =3

										_SCREAMING_SNAKE_CASE			  =do_lower_case
										_SCREAMING_SNAKE_CASE			  =remove_space
										_SCREAMING_SNAKE_CASE			  =keep_accents
										_SCREAMING_SNAKE_CASE			  =vocab_file

										_SCREAMING_SNAKE_CASE			  =spm.SentencePieceProcessor(**self.sp_model_kwargs  )
										self.sp_model.Load(_a  )




						@property
						def 							__UpperCamelCase				(       self					:      str  )							->     Any:


										"""simple docstring"""





										return len(self.sp_model  )




						def 							__UpperCamelCase				(       self					:      Optional[Any]  )							->     Optional[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ={self.convert_ids_to_tokens(_a  ): i for i in range(self.vocab_size  )}
										vocab.update(self.added_tokens_encoder  )
										return vocab




						def __getstate__(       self					:      List[Any]  )							->     List[str]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.__dict__.copy()
										_SCREAMING_SNAKE_CASE			  =None
										return state




						def __setstate__(       self					:      Union[str, Any]				,    _a					:      str  )							->     Optional[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =d

										# for backward compatibility
										if not hasattr(self				,    '''sp_model_kwargs'''  ):
														_SCREAMING_SNAKE_CASE			  ={}

										_SCREAMING_SNAKE_CASE			  =spm.SentencePieceProcessor(**self.sp_model_kwargs  )
										self.sp_model.Load(self.vocab_file  )




						def 							__UpperCamelCase				(       self					:      Optional[Any]				,    _a					:      int  )							->     Optional[Any]:


										"""simple docstring"""





										if self.remove_space:
														_SCREAMING_SNAKE_CASE			  =''' '''.join(inputs.strip().split()  )
										else:
														_SCREAMING_SNAKE_CASE			  =inputs
										_SCREAMING_SNAKE_CASE			  =outputs.replace('''``'''				,    '''"'''  ).replace('''\'\''''				,    '''"'''  )

										if not self.keep_accents:
														_SCREAMING_SNAKE_CASE			  =unicodedata.normalize('''NFKD'''				,    _a  )
														_SCREAMING_SNAKE_CASE			  =''''''.join([c for c in outputs if not unicodedata.combining(_a  )]  )
										if self.do_lower_case:
														_SCREAMING_SNAKE_CASE			  =outputs.lower()

										return outputs




						def 							__UpperCamelCase				(       self					:      Optional[int]				,    _a					:      str  )							->     List[str]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.preprocess_text(_a  )
										_SCREAMING_SNAKE_CASE			  =self.sp_model.encode(_a				,    out_type=_a  )
										_SCREAMING_SNAKE_CASE			  =[]
										for piece in pieces:
														if len(_a  ) > 1 and piece[-1] == str(''','''  ) and piece[-2].isdigit():
																		_SCREAMING_SNAKE_CASE			  =self.sp_model.EncodeAsPieces(piece[:-1].replace(_a				,    ''''''  )  )
																		if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																						if len(cur_pieces[0]  ) == 1:
																										_SCREAMING_SNAKE_CASE			  =cur_pieces[1:]
																						else:
																										_SCREAMING_SNAKE_CASE			  =cur_pieces[0][1:]
																		cur_pieces.append(piece[-1]  )
																		new_pieces.extend(_a  )
														else:
																		new_pieces.append(_a  )

										return new_pieces




						def 							__UpperCamelCase				(       self					:      Optional[Any]				,    _a					:      Any  )							->     List[str]:


										"""simple docstring"""





										return self.sp_model.PieceToId(_a  )




						def 							__UpperCamelCase				(       self					:      Union[str, Any]				,    _a					:      str  )							->     Dict:


										"""simple docstring"""





										return self.sp_model.IdToPiece(_a  )




						def 							__UpperCamelCase				(       self					:      Dict				,    _a					:      Union[str, Any]  )							->     Union[str, Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =''''''.join(_a  ).replace(_a				,    ''' '''  ).strip()
										return out_string




						def 							__UpperCamelCase				(       self					:      str				,    _a					:      List[int]				,    _a					:      bool = False				,    _a					:      bool = None				,    _a					:      bool = True				,    **_a					:      Any				,    )							->     str:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =kwargs.pop('''use_source_tokenizer'''				,    _a  )

										_SCREAMING_SNAKE_CASE			  =self.convert_ids_to_tokens(_a				,    skip_special_tokens=_a  )

										# To avoid mixing byte-level and unicode for byte-level BPT
										# we need to build string separately for added tokens and byte-level tokens
										# cf. https://github.com/huggingface/transformers/issues/1133
										_SCREAMING_SNAKE_CASE			  =[]
										_SCREAMING_SNAKE_CASE			  =[]
										for token in filtered_tokens:
														if skip_special_tokens and token in self.all_special_ids:
																		continue
														if token in self.added_tokens_encoder:
																		if current_sub_text:
																						sub_texts.append(self.convert_tokens_to_string(_a  )  )
																						_SCREAMING_SNAKE_CASE			  =[]
																		sub_texts.append(_a  )
														else:
																		current_sub_text.append(_a  )
										if current_sub_text:
														sub_texts.append(self.convert_tokens_to_string(_a  )  )

										# Mimic the behavior of the Rust tokenizer:
										# By default, there are no spaces between special tokens
										_SCREAMING_SNAKE_CASE			  =''''''.join(_a  )

										_SCREAMING_SNAKE_CASE			  =(
										    clean_up_tokenization_spaces
										    if clean_up_tokenization_spaces is not None
										    else self.clean_up_tokenization_spaces
										)
										if clean_up_tokenization_spaces:
														_SCREAMING_SNAKE_CASE			  =self.clean_up_tokenization(_a  )
														return clean_text
										else:
														return text




						def 							__UpperCamelCase				(       self					:      Dict				,    _a					:      List[int]				,    _a					:      Optional[List[int]] = None  )							->     List[int]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =[self.sep_token_id]
										_SCREAMING_SNAKE_CASE			  =[self.cls_token_id]
										if token_ids_a is None:
														return token_ids_a + sep + cls
										return token_ids_a + sep + token_ids_a + sep + cls




						def 							__UpperCamelCase				(       self					:      Optional[Any]				,    _a					:      List[int]				,    _a					:      Optional[List[int]] = None				,    _a					:      bool = False  )							->     List[int]:


										"""simple docstring"""





										if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=_a				,    token_ids_a=_a				,    already_has_special_tokens=_a  )

										if token_ids_a is not None:
														return ([0] * len(_a  )) + [1] + ([0] * len(_a  )) + [1, 1]
										return ([0] * len(_a  )) + [1, 1]




						def 							__UpperCamelCase				(       self					:      str				,    _a					:      List[int]				,    _a					:      Optional[List[int]] = None  )							->     List[int]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =[self.sep_token_id]
										_SCREAMING_SNAKE_CASE			  =[2]

										if token_ids_a is None:
														return len(token_ids_a + sep  ) * [0] + cls_segment_id
										return len(token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1] + cls_segment_id




						def 							__UpperCamelCase				(       self					:      Optional[Any]				,    _a					:      str				,    _a					:      Optional[str] = None  )							->     Tuple[str]:


										"""simple docstring"""





										if not os.path.isdir(_a  ):
														logger.error(f"Vocabulary path ({save_directory}) should be a directory"  )
														return
										_SCREAMING_SNAKE_CASE			  =os.path.join(
										    _a				,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']  )

										if os.path.abspath(self.vocab_file  ) != os.path.abspath(_a  ) and os.path.isfile(self.vocab_file  ):
														copyfile(self.vocab_file				,    _a  )
										elif not os.path.isfile(self.vocab_file  ):
														with open(_a				,    '''wb'''  ) as fi:
																		_SCREAMING_SNAKE_CASE			  =self.sp_model.serialized_model_proto()
																		fi.write(_a  )

										return (out_vocab_file,)



import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
							from cookiecutter.main import cookiecutter

							snake_case_						:			str													=							True
except ImportError:
							snake_case_						:			Optional[Any]													=							False

snake_case_						:			int													=							logging.get_logger(__name__)  # pylint: disable=invalid-name







def 						lowerCamelCase(   a__):
				return AddNewModelCommand(args.testing	,args.testing_file	,path=args.path)


class 						A__							(		UpperCamelCase__  ):




						@staticmethod
						def 							__UpperCamelCase				(       _a					:      ArgumentParser  )							->     int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =parser.add_parser('''add-new-model'''  )
										add_new_model_parser.add_argument('''--testing'''				,    action='''store_true'''				,    help='''If in testing mode.'''  )
										add_new_model_parser.add_argument('''--testing_file'''				,    type=_a				,    help='''Configuration file on which to run.'''  )
										add_new_model_parser.add_argument(
										    '''--path'''				,    type=_a				,    help='''Path to cookiecutter. Should only be used for testing purposes.'''  )
										add_new_model_parser.set_defaults(func=_a  )




						def __init__(       self					:      List[str]				,    _a					:      bool				,    _a					:      str				,    _a					:      Any=None				,    *_a					:      Union[str, Any]  )							->     Any:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =testing
										_SCREAMING_SNAKE_CASE			  =testing_file
										_SCREAMING_SNAKE_CASE			  =path




						def 							__UpperCamelCase				(       self					:      List[Any]  )							->     int:


										"""simple docstring"""





										warnings.warn(
										    '''The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. '''
										    '''It is not actively maintained anymore, so might give a result that won\'t pass all tests and quality '''
										    '''checks, you should use `transformers-cli add-new-model-like` instead.'''  )
										if not _has_cookiecutter:
														raise ImportError(
														    '''Model creation dependencies are required to use the `add_new_model` command. Install them by running '''
														    '''the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n'''  )
										# Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
										_SCREAMING_SNAKE_CASE			  =[directory for directory in os.listdir() if '''cookiecutter-template-''' == directory[:22]]
										if len(_a  ) > 0:
														raise ValueError(
														    '''Several directories starting with `cookiecutter-template-` in current working directory. '''
														    '''Please clean your directory by removing all folders starting with `cookiecutter-template-` or '''
														    '''change your working directory.'''  )

										_SCREAMING_SNAKE_CASE			  =(
										    Path(_a  ).parent.parent.parent.parent if self._path is None else Path(self._path  ).parent.parent
										)
										_SCREAMING_SNAKE_CASE			  =path_to_transformer_root / '''templates''' / '''adding_a_new_model'''

										# Execute cookiecutter
										if not self._testing:
														cookiecutter(str(_a  )  )
										else:
														with open(self._testing_file				,    '''r'''  ) as configuration_file:
																		_SCREAMING_SNAKE_CASE			  =json.load(_a  )

														cookiecutter(
														    str(path_to_cookiecutter if self._path is None else self._path  )				,    no_input=_a				,    extra_context=_a				,    )

										_SCREAMING_SNAKE_CASE			  =[directory for directory in os.listdir() if '''cookiecutter-template-''' in directory[:22]][0]

										# Retrieve configuration
										with open(directory + '''/configuration.json'''				,    '''r'''  ) as configuration_file:
														_SCREAMING_SNAKE_CASE			  =json.load(_a  )

										_SCREAMING_SNAKE_CASE			  =configuration['''lowercase_modelname''']
										_SCREAMING_SNAKE_CASE			  =configuration['''generate_tensorflow_pytorch_and_flax''']
										os.remove(f"{directory}/configuration.json"  )

										_SCREAMING_SNAKE_CASE			  ='''PyTorch''' in generate_tensorflow_pytorch_and_flax
										_SCREAMING_SNAKE_CASE			  ='''TensorFlow''' in generate_tensorflow_pytorch_and_flax
										_SCREAMING_SNAKE_CASE			  ='''Flax''' in generate_tensorflow_pytorch_and_flax

										_SCREAMING_SNAKE_CASE			  =f"{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}"
										os.makedirs(_a				,    exist_ok=_a  )
										os.makedirs(f"{path_to_transformer_root}/tests/models/{lowercase_model_name}"				,    exist_ok=_a  )

										# Tests require submodules as they have parent imports
										with open(f"{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py"				,    '''w'''  ):
														pass

										shutil.move(
										    f"{directory}/__init__.py"				,    f"{model_dir}/__init__.py"				,    )
										shutil.move(
										    f"{directory}/configuration_{lowercase_model_name}.py"				,    f"{model_dir}/configuration_{lowercase_model_name}.py"				,    )

										def remove_copy_lines(_a					:      int  ):
														with open(_a				,    '''r'''  ) as f:
																		_SCREAMING_SNAKE_CASE			  =f.readlines()
														with open(_a				,    '''w'''  ) as f:
																		for line in lines:
																						if "# Copied from transformers." not in line:
																										f.write(_a  )

										if output_pytorch:
														if not self._testing:
																		remove_copy_lines(f"{directory}/modeling_{lowercase_model_name}.py"  )

														shutil.move(
														    f"{directory}/modeling_{lowercase_model_name}.py"				,    f"{model_dir}/modeling_{lowercase_model_name}.py"				,    )

														shutil.move(
														    f"{directory}/test_modeling_{lowercase_model_name}.py"				,    f"{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py"				,    )
										else:
														os.remove(f"{directory}/modeling_{lowercase_model_name}.py"  )
														os.remove(f"{directory}/test_modeling_{lowercase_model_name}.py"  )

										if output_tensorflow:
														if not self._testing:
																		remove_copy_lines(f"{directory}/modeling_tf_{lowercase_model_name}.py"  )

														shutil.move(
														    f"{directory}/modeling_tf_{lowercase_model_name}.py"				,    f"{model_dir}/modeling_tf_{lowercase_model_name}.py"				,    )

														shutil.move(
														    f"{directory}/test_modeling_tf_{lowercase_model_name}.py"				,    f"{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py"				,    )
										else:
														os.remove(f"{directory}/modeling_tf_{lowercase_model_name}.py"  )
														os.remove(f"{directory}/test_modeling_tf_{lowercase_model_name}.py"  )

										if output_flax:
														if not self._testing:
																		remove_copy_lines(f"{directory}/modeling_flax_{lowercase_model_name}.py"  )

														shutil.move(
														    f"{directory}/modeling_flax_{lowercase_model_name}.py"				,    f"{model_dir}/modeling_flax_{lowercase_model_name}.py"				,    )

														shutil.move(
														    f"{directory}/test_modeling_flax_{lowercase_model_name}.py"				,    f"{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py"				,    )
										else:
														os.remove(f"{directory}/modeling_flax_{lowercase_model_name}.py"  )
														os.remove(f"{directory}/test_modeling_flax_{lowercase_model_name}.py"  )

										shutil.move(
										    f"{directory}/{lowercase_model_name}.md"				,    f"{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md"				,    )

										shutil.move(
										    f"{directory}/tokenization_{lowercase_model_name}.py"				,    f"{model_dir}/tokenization_{lowercase_model_name}.py"				,    )

										shutil.move(
										    f"{directory}/tokenization_fast_{lowercase_model_name}.py"				,    f"{model_dir}/tokenization_{lowercase_model_name}_fast.py"				,    )

										from os import fdopen, remove
										from shutil import copymode, move
										from tempfile import mkstemp

										def replace(_a					:      str				,    _a					:      str				,    _a					:      List[str]  ):
														# Create temp file
														_SCREAMING_SNAKE_CASE  ,				_SCREAMING_SNAKE_CASE			  =mkstemp()
														_SCREAMING_SNAKE_CASE			  =False
														with fdopen(_a				,    '''w'''  ) as new_file:
																		with open(_a  ) as old_file:
																						for line in old_file:
																										new_file.write(_a  )
																										if line_to_copy_below in line:
																														_SCREAMING_SNAKE_CASE			  =True
																														for line_to_copy in lines_to_copy:
																																		new_file.write(_a  )

														if not line_found:
																		raise ValueError(f"Line {line_to_copy_below} was not found in file."  )

														# Copy the file permissions from the old file to the new file
														copymode(_a				,    _a  )
														# Remove original file
														remove(_a  )
														# Move new file
														move(_a				,    _a  )

										def skip_units(_a					:      str  ):
														return (
														    ("generating PyTorch" in line and not output_pytorch)
														    or ("generating TensorFlow" in line and not output_tensorflow)
														    or ("generating Flax" in line and not output_flax)
														)

										def replace_in_files(_a					:      Any  ):
														with open(_a  ) as datafile:
																		_SCREAMING_SNAKE_CASE			  =[]
																		_SCREAMING_SNAKE_CASE			  =False
																		_SCREAMING_SNAKE_CASE			  =False
																		for line in datafile:
																						if "# To replace in: " in line and "##" not in line:
																										_SCREAMING_SNAKE_CASE			  =line.split('''"'''  )[1]
																										_SCREAMING_SNAKE_CASE			  =skip_units(_a  )
																						elif "# Below: " in line and "##" not in line:
																										_SCREAMING_SNAKE_CASE			  =line.split('''"'''  )[1]
																										_SCREAMING_SNAKE_CASE			  =skip_units(_a  )
																						elif "# End." in line and "##" not in line:
																										if not skip_file and not skip_snippet:
																														replace(_a				,    _a				,    _a  )

																										_SCREAMING_SNAKE_CASE			  =[]
																						elif "# Replace with" in line and "##" not in line:
																										_SCREAMING_SNAKE_CASE			  =[]
																						elif "##" not in line:
																										lines_to_copy.append(_a  )

														remove(_a  )

										replace_in_files(f"{directory}/to_replace_{lowercase_model_name}.py"  )
										os.rmdir(_a  )