

'''simple docstring'''

import os
import re
import warnings
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer


if TYPE_CHECKING:
							from ...tokenization_utils_base import TextInput
from ...utils import logging


SCREAMING_SNAKE_CASE							:				Optional[int]     	=			logging.get_logger(__name__)

SCREAMING_SNAKE_CASE							:				Optional[int]     	=			{"vocab_file": "spiece.model"}

SCREAMING_SNAKE_CASE							:				Optional[int]     	=			{
    "vocab_file": {
        "t5-small": "https://huggingface.co/t5-small/resolve/main/spiece.model",
        "t5-base": "https://huggingface.co/t5-base/resolve/main/spiece.model",
        "t5-large": "https://huggingface.co/t5-large/resolve/main/spiece.model",
        "t5-3b": "https://huggingface.co/t5-3b/resolve/main/spiece.model",
        "t5-11b": "https://huggingface.co/t5-11b/resolve/main/spiece.model",
    }
}


# TODO(PVP) - this should be removed in Transformers v5
SCREAMING_SNAKE_CASE							:				List[str]     	=			{
    "t5-small": 512,
    "t5-base": 512,
    "t5-large": 512,
    "t5-3b": 512,
    "t5-11b": 512,
}

SCREAMING_SNAKE_CASE							:				Any     	=			"▁"






class    snake_case     (      lowercase_		):






				"""simple docstring"""
				_a    	=     VOCAB_FILES_NAMES
				_a    	=     PRETRAINED_VOCAB_FILES_MAP
				_a    	=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				_a    	=     ["""input_ids""", """attention_mask"""]



				def __init__(    self,     _lowercase,     _lowercase="</s>",     _lowercase="<unk>",     _lowercase="<pad>",     _lowercase=100,     _lowercase=None,     _lowercase = None,     _lowercase=True,     **_lowercase,     )     ->							None:
											# Add extra_ids to the special token list
											if extra_ids > 0 and additional_special_tokens is None:
																		SCREAMING_SNAKE_CASE_				=				[f"""<extra_id_{i}>""" for i in range(_lowercase       )]
											elif extra_ids > 0 and additional_special_tokens is not None:
																		# Check that we have the right number of extra_id special tokens
																		SCREAMING_SNAKE_CASE_				=				len(set(filter(lambda _lowercase       : bool('extra_id' in str(_lowercase       )       ),     _lowercase       )       )       )
																		if extra_tokens != extra_ids:
																									raise ValueError(
																									    f"""Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"""
																									    ' provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids'
																									    ' tokens'       )
											if legacy:
																		logger.warning_once(
																		    f"""You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to"""
																		    ' read the related pull request available at https://github.com/huggingface/transformers/pull/24565'       )

											SCREAMING_SNAKE_CASE_				=				legacy
											SCREAMING_SNAKE_CASE_				=				{} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    eos_token=_lowercase,     unk_token=_lowercase,     pad_token=_lowercase,     extra_ids=_lowercase,     additional_special_tokens=_lowercase,     sp_model_kwargs=self.sp_model_kwargs,     legacy=_lowercase,     **_lowercase,     )

											SCREAMING_SNAKE_CASE_				=				vocab_file
											SCREAMING_SNAKE_CASE_				=				extra_ids

											SCREAMING_SNAKE_CASE_				=				spm.SentencePieceProcessor(**self.sp_model_kwargs       )
											self.sp_model.Load(_lowercase       )



				@staticmethod
				def   a__					(    _lowercase,     _lowercase,     _lowercase       )     ->							Union[str, Any]:
											if pretrained_model_name_or_path in TaTokenizer.max_model_input_sizes:
																		SCREAMING_SNAKE_CASE_				=				TaTokenizer.max_model_input_sizes[pretrained_model_name_or_path]
																		if init_max_model_length is not None and init_max_model_length != max_model_length:
																									return init_max_model_length
																		elif init_max_model_length is None:
																									warnings.warn(
																									    'This tokenizer was incorrectly instantiated with a model max length of'
																									    f""" {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this"""
																									    ' behavior is kept to avoid breaking backwards compatibility when padding/encoding with'
																									    ' `truncation is True`.\n- Be aware that you SHOULD NOT rely on'
																									    f""" {pretrained_model_name_or_path} automatically truncating your input to"""
																									    f""" {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences"""
																									    f""" longer than {deprecated_max_model_length} you can either instantiate this tokenizer with"""
																									    ' `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please'
																									    ' instantiate this tokenizer with `model_max_length` set to your preferred value.',     _lowercase,     )

											return max_model_length



				@property
				def   a__					(    self       )     ->							Dict:
											return self.sp_model.get_piece_size() + self._extra_ids



				def   a__					(    self       )     ->							Any:
											SCREAMING_SNAKE_CASE_				=				{self.convert_ids_to_tokens(_lowercase       ): i for i in range(self.vocab_size       )}
											vocab.update(self.added_tokens_encoder       )
											return vocab



				def   a__					(    self,     _lowercase,     _lowercase = None,     _lowercase = False       )     ->							List[int]:
											if already_has_special_tokens:
																		return super().get_special_tokens_mask(
																		    token_ids_a=_lowercase,     token_ids_a=_lowercase,     already_has_special_tokens=_lowercase       )

											# normal case: some special tokens
											if token_ids_a is None:
																		return ([0] * len(_lowercase       )) + [1]
											return ([0] * len(_lowercase       )) + [1] + ([0] * len(_lowercase       )) + [1]



				def   a__					(    self       )     ->							str:
											return list(
											    set(filter(lambda _lowercase       : bool(re.search(R'<extra_id_\d+>',     _lowercase       )       ) is not None,     self.additional_special_tokens       )       )       )



				def   a__					(    self       )     ->							Union[str, Any]:
											return [self._convert_token_to_id(_lowercase       ) for token in self.get_sentinel_tokens()]



				def   a__					(    self,     _lowercase       )     ->							List[int]:
											if len(_lowercase       ) > 0 and token_ids[-1] == self.eos_token_id:
																		warnings.warn(
																		    f"""This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated"""
																		    ' eos tokens being added.'       )
																		return token_ids
											else:
																		return token_ids + [self.eos_token_id]



				def   a__					(    self,     _lowercase,     _lowercase = None       )     ->							List[int]:
											SCREAMING_SNAKE_CASE_				=				[self.eos_token_id]

											if token_ids_a is None:
																		return len(token_ids_a + eos       ) * [0]
											return len(token_ids_a + eos + token_ids_a + eos       ) * [0]



				def   a__					(    self,     _lowercase,     _lowercase = None       )     ->							List[int]:
											SCREAMING_SNAKE_CASE_				=				self._add_eos_if_not_present(_lowercase       )
											if token_ids_a is None:
																		return token_ids_a
											else:
																		SCREAMING_SNAKE_CASE_				=				self._add_eos_if_not_present(_lowercase       )
																		return token_ids_a + token_ids_a



				def __getstate__(    self       )     ->							Union[str, Any]:
											SCREAMING_SNAKE_CASE_				=				self.__dict__.copy()
											SCREAMING_SNAKE_CASE_				=				None
											return state



				def __setstate__(    self,     _lowercase       )     ->							Any:
											SCREAMING_SNAKE_CASE_				=				d

											# for backward compatibility
											if not hasattr(self,     'sp_model_kwargs'       ):
																		SCREAMING_SNAKE_CASE_				=				{}

											SCREAMING_SNAKE_CASE_				=				spm.SentencePieceProcessor(**self.sp_model_kwargs       )
											self.sp_model.Load(self.vocab_file       )



				def   a__					(    self,     _lowercase,     **_lowercase       )     ->							List[str]:
											# Replace the SPIECE_UNDERLINE with a space to make sure SPIECE_UNDERLINE is only used at
											# the beginning of the text
											if not self.legacy:
																		SCREAMING_SNAKE_CASE_				=				SPIECE_UNDERLINE + text.replace(_lowercase,     ' '       )
											return super().tokenize(_lowercase,     **_lowercase       )



				def   a__					(    self,     _lowercase,     **_lowercase       )     ->							int:
											if not self.legacy:
																		SCREAMING_SNAKE_CASE_				=				text.startswith(_lowercase       )
																		if is_first:
																									SCREAMING_SNAKE_CASE_				=				text[1:]

											SCREAMING_SNAKE_CASE_				=				self.sp_model.encode(_lowercase,     out_type=_lowercase       )

											if not self.legacy and not is_first and not text.startswith(' '       ) and tokens[0].startswith(_lowercase       ):
																		SCREAMING_SNAKE_CASE_				=				([tokens[0][1:]] if len(tokens[0]       ) > 1 else []) + tokens[1:]
											return tokens



				def   a__					(    self,     _lowercase       )     ->							int:
											if token.startswith('<extra_id_'       ):
																		SCREAMING_SNAKE_CASE_				=				re.match(R'<extra_id_(\d+)>',     _lowercase       )
																		SCREAMING_SNAKE_CASE_				=				int(match.group(1       )       )
																		return self.vocab_size - num - 1
											return self.sp_model.piece_to_id(_lowercase       )



				def   a__					(    self,     _lowercase       )     ->							int:
											if index < self.sp_model.get_piece_size():
																		SCREAMING_SNAKE_CASE_				=				self.sp_model.IdToPiece(_lowercase       )
											else:
																		SCREAMING_SNAKE_CASE_				=				f"""<extra_id_{self.vocab_size - 1 - index}>"""
											return token



				def   a__					(    self,     _lowercase       )     ->							str:
											SCREAMING_SNAKE_CASE_				=				[]
											SCREAMING_SNAKE_CASE_				=				''
											SCREAMING_SNAKE_CASE_				=				False
											for token in tokens:
																		# make sure that special tokens are not decoded using sentencepiece model
																		if token in self.all_special_tokens:
																									if not prev_is_special:
																																out_string += " "
																									out_string += self.sp_model.decode(_lowercase       ) + token
																									SCREAMING_SNAKE_CASE_				=				True
																									SCREAMING_SNAKE_CASE_				=				[]
																		else:
																									current_sub_tokens.append(_lowercase       )
																									SCREAMING_SNAKE_CASE_				=				False
											out_string += self.sp_model.decode(_lowercase       )
											return out_string.strip()





				def   a__					(    self,     _lowercase,     _lowercase = None       )     ->							Tuple[str]:
											if not os.path.isdir(_lowercase       ):
																		logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""       )
																		return
											SCREAMING_SNAKE_CASE_				=				os.path.join(
											    _lowercase,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']       )

											if os.path.abspath(self.vocab_file       ) != os.path.abspath(_lowercase       ) and os.path.isfile(self.vocab_file       ):
																		copyfile(self.vocab_file,     _lowercase       )
											elif not os.path.isfile(self.vocab_file       ):
																		with open(_lowercase,     'wb'       ) as fi:
																									SCREAMING_SNAKE_CASE_				=				self.sp_model.serialized_model_proto()
																									fi.write(_lowercase       )

											return (out_vocab_file,)








'''simple docstring'''

import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
							from cookiecutter.main import cookiecutter

							SCREAMING_SNAKE_CASE							:				Dict     	=			True
except ImportError:
							SCREAMING_SNAKE_CASE							:				Optional[Any]     	=			False

SCREAMING_SNAKE_CASE							:				str     	=			logging.get_logger(__name__)  # pylint: disable=invalid-name




def 							_UpperCamelCase				(   lowerCAmelCase__:					Namespace   )      -> Optional[Any]:
							return AddNewModelCommand(args.testing		,args.testing_file		,path=args.path   )






class    snake_case     (      lowercase_		):






				"""simple docstring"""



				@staticmethod
				def   a__					(    _lowercase       )     ->							Tuple:
											SCREAMING_SNAKE_CASE_				=				parser.add_parser('add-new-model'       )
											add_new_model_parser.add_argument('--testing',     action='store_true',     help='If in testing mode.'       )
											add_new_model_parser.add_argument('--testing_file',     type=_lowercase,     help='Configuration file on which to run.'       )
											add_new_model_parser.add_argument(
											    '--path',     type=_lowercase,     help='Path to cookiecutter. Should only be used for testing purposes.'       )
											add_new_model_parser.set_defaults(func=_lowercase       )



				def __init__(    self,     _lowercase,     _lowercase,     _lowercase=None,     *_lowercase       )     ->							Optional[Any]:
											SCREAMING_SNAKE_CASE_				=				testing
											SCREAMING_SNAKE_CASE_				=				testing_file
											SCREAMING_SNAKE_CASE_				=				path





				def   a__					(    self       )     ->							Optional[int]:
											warnings.warn(
											    'The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. '
											    'It is not actively maintained anymore, so might give a result that won\'t pass all tests and quality '
											    'checks, you should use `transformers-cli add-new-model-like` instead.'       )
											if not _has_cookiecutter:
																		raise ImportError(
																		    'Model creation dependencies are required to use the `add_new_model` command. Install them by running '
																		    'the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n'       )
											# Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
											SCREAMING_SNAKE_CASE_				=				[directory for directory in os.listdir() if 'cookiecutter-template-' == directory[:22]]
											if len(_lowercase       ) > 0:
																		raise ValueError(
																		    'Several directories starting with `cookiecutter-template-` in current working directory. '
																		    'Please clean your directory by removing all folders starting with `cookiecutter-template-` or '
																		    'change your working directory.'       )

											SCREAMING_SNAKE_CASE_				=				(
											    Path(_lowercase       ).parent.parent.parent.parent if self._path is None else Path(self._path       ).parent.parent
											)
											SCREAMING_SNAKE_CASE_				=				path_to_transformer_root / 'templates' / 'adding_a_new_model'

											# Execute cookiecutter
											if not self._testing:
																		cookiecutter(str(_lowercase       )       )
											else:
																		with open(self._testing_file,     'r'       ) as configuration_file:
																									SCREAMING_SNAKE_CASE_				=				json.load(_lowercase       )

																		cookiecutter(
																		    str(path_to_cookiecutter if self._path is None else self._path       ),     no_input=_lowercase,     extra_context=_lowercase,     )

											SCREAMING_SNAKE_CASE_				=				[directory for directory in os.listdir() if 'cookiecutter-template-' in directory[:22]][0]

											# Retrieve configuration
											with open(directory + '/configuration.json',     'r'       ) as configuration_file:
																		SCREAMING_SNAKE_CASE_				=				json.load(_lowercase       )

											SCREAMING_SNAKE_CASE_				=				configuration['lowercase_modelname']
											SCREAMING_SNAKE_CASE_				=				configuration['generate_tensorflow_pytorch_and_flax']
											os.remove(f"""{directory}/configuration.json"""       )

											SCREAMING_SNAKE_CASE_				=				'PyTorch' in generate_tensorflow_pytorch_and_flax
											SCREAMING_SNAKE_CASE_				=				'TensorFlow' in generate_tensorflow_pytorch_and_flax
											SCREAMING_SNAKE_CASE_				=				'Flax' in generate_tensorflow_pytorch_and_flax

											SCREAMING_SNAKE_CASE_				=				f"""{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}"""
											os.makedirs(_lowercase,     exist_ok=_lowercase       )
											os.makedirs(f"""{path_to_transformer_root}/tests/models/{lowercase_model_name}""",     exist_ok=_lowercase       )

											# Tests require submodules as they have parent imports
											with open(f"""{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py""",     'w'       ):
																		pass

											shutil.move(
											    f"""{directory}/__init__.py""",     f"""{model_dir}/__init__.py""",     )
											shutil.move(
											    f"""{directory}/configuration_{lowercase_model_name}.py""",     f"""{model_dir}/configuration_{lowercase_model_name}.py""",     )

											def remove_copy_lines(_lowercase       ):
																		with open(_lowercase,     'r'       ) as f:
																									SCREAMING_SNAKE_CASE_				=				f.readlines()
																		with open(_lowercase,     'w'       ) as f:
																									for line in lines:
																																if "# Copied from transformers." not in line:
																																							f.write(_lowercase       )

											if output_pytorch:
																		if not self._testing:
																									remove_copy_lines(f"""{directory}/modeling_{lowercase_model_name}.py"""       )

																		shutil.move(
																		    f"""{directory}/modeling_{lowercase_model_name}.py""",     f"""{model_dir}/modeling_{lowercase_model_name}.py""",     )

																		shutil.move(
																		    f"""{directory}/test_modeling_{lowercase_model_name}.py""",     f"""{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py""",     )
											else:
																		os.remove(f"""{directory}/modeling_{lowercase_model_name}.py"""       )
																		os.remove(f"""{directory}/test_modeling_{lowercase_model_name}.py"""       )

											if output_tensorflow:
																		if not self._testing:
																									remove_copy_lines(f"""{directory}/modeling_tf_{lowercase_model_name}.py"""       )

																		shutil.move(
																		    f"""{directory}/modeling_tf_{lowercase_model_name}.py""",     f"""{model_dir}/modeling_tf_{lowercase_model_name}.py""",     )

																		shutil.move(
																		    f"""{directory}/test_modeling_tf_{lowercase_model_name}.py""",     f"""{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py""",     )
											else:
																		os.remove(f"""{directory}/modeling_tf_{lowercase_model_name}.py"""       )
																		os.remove(f"""{directory}/test_modeling_tf_{lowercase_model_name}.py"""       )

											if output_flax:
																		if not self._testing:
																									remove_copy_lines(f"""{directory}/modeling_flax_{lowercase_model_name}.py"""       )

																		shutil.move(
																		    f"""{directory}/modeling_flax_{lowercase_model_name}.py""",     f"""{model_dir}/modeling_flax_{lowercase_model_name}.py""",     )

																		shutil.move(
																		    f"""{directory}/test_modeling_flax_{lowercase_model_name}.py""",     f"""{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py""",     )
											else:
																		os.remove(f"""{directory}/modeling_flax_{lowercase_model_name}.py"""       )
																		os.remove(f"""{directory}/test_modeling_flax_{lowercase_model_name}.py"""       )

											shutil.move(
											    f"""{directory}/{lowercase_model_name}.md""",     f"""{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md""",     )

											shutil.move(
											    f"""{directory}/tokenization_{lowercase_model_name}.py""",     f"""{model_dir}/tokenization_{lowercase_model_name}.py""",     )

											shutil.move(
											    f"""{directory}/tokenization_fast_{lowercase_model_name}.py""",     f"""{model_dir}/tokenization_{lowercase_model_name}_fast.py""",     )

											from os import fdopen, remove
											from shutil import copymode, move
											from tempfile import mkstemp

											def replace(_lowercase,     _lowercase,     _lowercase       ):
																		# Create temp file
																		SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				=				mkstemp()
																		SCREAMING_SNAKE_CASE_				=				False
																		with fdopen(_lowercase,     'w'       ) as new_file:
																									with open(_lowercase       ) as old_file:
																																for line in old_file:
																																							new_file.write(_lowercase       )
																																							if line_to_copy_below in line:
																																														SCREAMING_SNAKE_CASE_				=				True
																																														for line_to_copy in lines_to_copy:
																																																					new_file.write(_lowercase       )

																		if not line_found:
																									raise ValueError(f"""Line {line_to_copy_below} was not found in file."""       )

																		# Copy the file permissions from the old file to the new file
																		copymode(_lowercase,     _lowercase       )
																		# Remove original file
																		remove(_lowercase       )
																		# Move new file
																		move(_lowercase,     _lowercase       )

											def skip_units(_lowercase       ):
																		return (
																		    ("generating PyTorch" in line and not output_pytorch)
																		    or ("generating TensorFlow" in line and not output_tensorflow)
																		    or ("generating Flax" in line and not output_flax)
																		)

											def replace_in_files(_lowercase       ):
																		with open(_lowercase       ) as datafile:
																									SCREAMING_SNAKE_CASE_				=				[]
																									SCREAMING_SNAKE_CASE_				=				False
																									SCREAMING_SNAKE_CASE_				=				False
																									for line in datafile:
																																if "# To replace in: " in line and "##" not in line:
																																							SCREAMING_SNAKE_CASE_				=				line.split('"'       )[1]
																																							SCREAMING_SNAKE_CASE_				=				skip_units(_lowercase       )
																																elif "# Below: " in line and "##" not in line:
																																							SCREAMING_SNAKE_CASE_				=				line.split('"'       )[1]
																																							SCREAMING_SNAKE_CASE_				=				skip_units(_lowercase       )
																																elif "# End." in line and "##" not in line:
																																							if not skip_file and not skip_snippet:
																																														replace(_lowercase,     _lowercase,     _lowercase       )

																																							SCREAMING_SNAKE_CASE_				=				[]
																																elif "# Replace with" in line and "##" not in line:
																																							SCREAMING_SNAKE_CASE_				=				[]
																																elif "##" not in line:
																																							lines_to_copy.append(_lowercase       )

																		remove(_lowercase       )

											replace_in_files(f"""{directory}/to_replace_{lowercase_model_name}.py"""       )
											os.rmdir(_lowercase       )





