

'''simple docstring'''

import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin






class    snake_case     (      lowercase_,						unittest.TestCase		):






				"""simple docstring"""
				_a    	=     TransfoXLTokenizer
				_a    	=     False
				_a    	=     False



				def   a__					(    self       )     ->							int:
											super().setUp()

											SCREAMING_SNAKE_CASE_				=				[
											    '<unk>',
											    '[CLS]',
											    '[SEP]',
											    'want',
											    'unwanted',
											    'wa',
											    'un',
											    'running',
											    ',',
											    'low',
											    'l',
											]
											SCREAMING_SNAKE_CASE_				=				os.path.join(self.tmpdirname,     VOCAB_FILES_NAMES['vocab_file']       )
											with open(self.vocab_file,     'w',     encoding='utf-8'       ) as vocab_writer:
																		vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]       )       )



				def   a__					(    self,     **_lowercase       )     ->							Optional[Any]:
											SCREAMING_SNAKE_CASE_				=				True
											return TransfoXLTokenizer.from_pretrained(self.tmpdirname,     **_lowercase       )



				def   a__					(    self,     _lowercase       )     ->							Any:
											SCREAMING_SNAKE_CASE_				=				'<unk> UNwanted , running'
											SCREAMING_SNAKE_CASE_				=				'<unk> unwanted, running'
											return input_text, output_text



				def   a__					(    self       )     ->							Optional[Any]:
											SCREAMING_SNAKE_CASE_				=				TransfoXLTokenizer(vocab_file=self.vocab_file,     lower_case=_lowercase       )

											SCREAMING_SNAKE_CASE_				=				tokenizer.tokenize('<unk> UNwanted , running'       )
											self.assertListEqual(_lowercase,     ['<unk>', 'unwanted', ',', 'running']       )

											self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase       ),     [0, 4, 8, 7]       )



				def   a__					(    self       )     ->							List[Any]:
											SCREAMING_SNAKE_CASE_				=				TransfoXLTokenizer(lower_case=_lowercase       )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '       ),     ['hello', '!', 'how', 'are', 'you', '?']       )



				def   a__					(    self       )     ->							str:
											SCREAMING_SNAKE_CASE_				=				TransfoXLTokenizer(lower_case=_lowercase       )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '       ),     ['HeLLo', '!', 'how', 'Are', 'yoU', '?']       )



				def   a__					(    self       )     ->							List[Any]:
											SCREAMING_SNAKE_CASE_				=				TransfoXLTokenizer(lower_case=_lowercase       )
											SCREAMING_SNAKE_CASE_				=				'Hello (bracket) and side-scrolled [and] Henry\'s $5,000 with 3.34 m. What\'s up!?'
											SCREAMING_SNAKE_CASE_				=				[
											    'Hello',
											    '(',
											    'bracket',
											    ')',
											    'and',
											    'side',
											    '@-@',
											    'scrolled',
											    '[',
											    'and',
											    ']',
											    'Henry',
											    '\'s',
											    '$',
											    '5',
											    '@,@',
											    '000',
											    'with',
											    '3',
											    '@.@',
											    '34',
											    'm',
											    '.',
											    'What',
											    '\'s',
											    'up',
											    '!',
											    '?',
											]

											self.assertListEqual(tokenizer.tokenize(_lowercase       ),     _lowercase       )

											self.assertEqual(tokenizer.convert_tokens_to_string(_lowercase       ),     _lowercase       )





				def   a__					(    self       )     ->							str:
											SCREAMING_SNAKE_CASE_				=				self.get_tokenizer()
											SCREAMING_SNAKE_CASE_				=				len(_lowercase       )

											tokenizer.add_tokens(['new1', 'new2']       )
											tokenizer.move_added_token('new1',     1       )

											# Check that moved token is not copied (duplicate)
											self.assertEqual(len(_lowercase       ),     original_len + 2       )
											# Check that token is moved to specified id
											self.assertEqual(tokenizer.encode('new1'       ),     [1]       )
											self.assertEqual(tokenizer.decode([1]       ),     'new1'       )








'''simple docstring'''

import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
							from cookiecutter.main import cookiecutter

							SCREAMING_SNAKE_CASE							:				Dict     	=			True
except ImportError:
							SCREAMING_SNAKE_CASE							:				Optional[Any]     	=			False

SCREAMING_SNAKE_CASE							:				str     	=			logging.get_logger(__name__)  # pylint: disable=invalid-name




def 							_UpperCamelCase				(   lowerCAmelCase__:					Namespace   )      -> Optional[Any]:
							return AddNewModelCommand(args.testing		,args.testing_file		,path=args.path   )






class    snake_case     (      lowercase_		):






				"""simple docstring"""



				@staticmethod
				def   a__					(    _lowercase       )     ->							Tuple:
											SCREAMING_SNAKE_CASE_				=				parser.add_parser('add-new-model'       )
											add_new_model_parser.add_argument('--testing',     action='store_true',     help='If in testing mode.'       )
											add_new_model_parser.add_argument('--testing_file',     type=_lowercase,     help='Configuration file on which to run.'       )
											add_new_model_parser.add_argument(
											    '--path',     type=_lowercase,     help='Path to cookiecutter. Should only be used for testing purposes.'       )
											add_new_model_parser.set_defaults(func=_lowercase       )



				def __init__(    self,     _lowercase,     _lowercase,     _lowercase=None,     *_lowercase       )     ->							Optional[Any]:
											SCREAMING_SNAKE_CASE_				=				testing
											SCREAMING_SNAKE_CASE_				=				testing_file
											SCREAMING_SNAKE_CASE_				=				path





				def   a__					(    self       )     ->							Optional[int]:
											warnings.warn(
											    'The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. '
											    'It is not actively maintained anymore, so might give a result that won\'t pass all tests and quality '
											    'checks, you should use `transformers-cli add-new-model-like` instead.'       )
											if not _has_cookiecutter:
																		raise ImportError(
																		    'Model creation dependencies are required to use the `add_new_model` command. Install them by running '
																		    'the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n'       )
											# Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
											SCREAMING_SNAKE_CASE_				=				[directory for directory in os.listdir() if 'cookiecutter-template-' == directory[:22]]
											if len(_lowercase       ) > 0:
																		raise ValueError(
																		    'Several directories starting with `cookiecutter-template-` in current working directory. '
																		    'Please clean your directory by removing all folders starting with `cookiecutter-template-` or '
																		    'change your working directory.'       )

											SCREAMING_SNAKE_CASE_				=				(
											    Path(_lowercase       ).parent.parent.parent.parent if self._path is None else Path(self._path       ).parent.parent
											)
											SCREAMING_SNAKE_CASE_				=				path_to_transformer_root / 'templates' / 'adding_a_new_model'

											# Execute cookiecutter
											if not self._testing:
																		cookiecutter(str(_lowercase       )       )
											else:
																		with open(self._testing_file,     'r'       ) as configuration_file:
																									SCREAMING_SNAKE_CASE_				=				json.load(_lowercase       )

																		cookiecutter(
																		    str(path_to_cookiecutter if self._path is None else self._path       ),     no_input=_lowercase,     extra_context=_lowercase,     )

											SCREAMING_SNAKE_CASE_				=				[directory for directory in os.listdir() if 'cookiecutter-template-' in directory[:22]][0]

											# Retrieve configuration
											with open(directory + '/configuration.json',     'r'       ) as configuration_file:
																		SCREAMING_SNAKE_CASE_				=				json.load(_lowercase       )

											SCREAMING_SNAKE_CASE_				=				configuration['lowercase_modelname']
											SCREAMING_SNAKE_CASE_				=				configuration['generate_tensorflow_pytorch_and_flax']
											os.remove(f"""{directory}/configuration.json"""       )

											SCREAMING_SNAKE_CASE_				=				'PyTorch' in generate_tensorflow_pytorch_and_flax
											SCREAMING_SNAKE_CASE_				=				'TensorFlow' in generate_tensorflow_pytorch_and_flax
											SCREAMING_SNAKE_CASE_				=				'Flax' in generate_tensorflow_pytorch_and_flax

											SCREAMING_SNAKE_CASE_				=				f"""{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}"""
											os.makedirs(_lowercase,     exist_ok=_lowercase       )
											os.makedirs(f"""{path_to_transformer_root}/tests/models/{lowercase_model_name}""",     exist_ok=_lowercase       )

											# Tests require submodules as they have parent imports
											with open(f"""{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py""",     'w'       ):
																		pass

											shutil.move(
											    f"""{directory}/__init__.py""",     f"""{model_dir}/__init__.py""",     )
											shutil.move(
											    f"""{directory}/configuration_{lowercase_model_name}.py""",     f"""{model_dir}/configuration_{lowercase_model_name}.py""",     )

											def remove_copy_lines(_lowercase       ):
																		with open(_lowercase,     'r'       ) as f:
																									SCREAMING_SNAKE_CASE_				=				f.readlines()
																		with open(_lowercase,     'w'       ) as f:
																									for line in lines:
																																if "# Copied from transformers." not in line:
																																							f.write(_lowercase       )

											if output_pytorch:
																		if not self._testing:
																									remove_copy_lines(f"""{directory}/modeling_{lowercase_model_name}.py"""       )

																		shutil.move(
																		    f"""{directory}/modeling_{lowercase_model_name}.py""",     f"""{model_dir}/modeling_{lowercase_model_name}.py""",     )

																		shutil.move(
																		    f"""{directory}/test_modeling_{lowercase_model_name}.py""",     f"""{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py""",     )
											else:
																		os.remove(f"""{directory}/modeling_{lowercase_model_name}.py"""       )
																		os.remove(f"""{directory}/test_modeling_{lowercase_model_name}.py"""       )

											if output_tensorflow:
																		if not self._testing:
																									remove_copy_lines(f"""{directory}/modeling_tf_{lowercase_model_name}.py"""       )

																		shutil.move(
																		    f"""{directory}/modeling_tf_{lowercase_model_name}.py""",     f"""{model_dir}/modeling_tf_{lowercase_model_name}.py""",     )

																		shutil.move(
																		    f"""{directory}/test_modeling_tf_{lowercase_model_name}.py""",     f"""{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py""",     )
											else:
																		os.remove(f"""{directory}/modeling_tf_{lowercase_model_name}.py"""       )
																		os.remove(f"""{directory}/test_modeling_tf_{lowercase_model_name}.py"""       )

											if output_flax:
																		if not self._testing:
																									remove_copy_lines(f"""{directory}/modeling_flax_{lowercase_model_name}.py"""       )

																		shutil.move(
																		    f"""{directory}/modeling_flax_{lowercase_model_name}.py""",     f"""{model_dir}/modeling_flax_{lowercase_model_name}.py""",     )

																		shutil.move(
																		    f"""{directory}/test_modeling_flax_{lowercase_model_name}.py""",     f"""{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py""",     )
											else:
																		os.remove(f"""{directory}/modeling_flax_{lowercase_model_name}.py"""       )
																		os.remove(f"""{directory}/test_modeling_flax_{lowercase_model_name}.py"""       )

											shutil.move(
											    f"""{directory}/{lowercase_model_name}.md""",     f"""{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md""",     )

											shutil.move(
											    f"""{directory}/tokenization_{lowercase_model_name}.py""",     f"""{model_dir}/tokenization_{lowercase_model_name}.py""",     )

											shutil.move(
											    f"""{directory}/tokenization_fast_{lowercase_model_name}.py""",     f"""{model_dir}/tokenization_{lowercase_model_name}_fast.py""",     )

											from os import fdopen, remove
											from shutil import copymode, move
											from tempfile import mkstemp

											def replace(_lowercase,     _lowercase,     _lowercase       ):
																		# Create temp file
																		SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				=				mkstemp()
																		SCREAMING_SNAKE_CASE_				=				False
																		with fdopen(_lowercase,     'w'       ) as new_file:
																									with open(_lowercase       ) as old_file:
																																for line in old_file:
																																							new_file.write(_lowercase       )
																																							if line_to_copy_below in line:
																																														SCREAMING_SNAKE_CASE_				=				True
																																														for line_to_copy in lines_to_copy:
																																																					new_file.write(_lowercase       )

																		if not line_found:
																									raise ValueError(f"""Line {line_to_copy_below} was not found in file."""       )

																		# Copy the file permissions from the old file to the new file
																		copymode(_lowercase,     _lowercase       )
																		# Remove original file
																		remove(_lowercase       )
																		# Move new file
																		move(_lowercase,     _lowercase       )

											def skip_units(_lowercase       ):
																		return (
																		    ("generating PyTorch" in line and not output_pytorch)
																		    or ("generating TensorFlow" in line and not output_tensorflow)
																		    or ("generating Flax" in line and not output_flax)
																		)

											def replace_in_files(_lowercase       ):
																		with open(_lowercase       ) as datafile:
																									SCREAMING_SNAKE_CASE_				=				[]
																									SCREAMING_SNAKE_CASE_				=				False
																									SCREAMING_SNAKE_CASE_				=				False
																									for line in datafile:
																																if "# To replace in: " in line and "##" not in line:
																																							SCREAMING_SNAKE_CASE_				=				line.split('"'       )[1]
																																							SCREAMING_SNAKE_CASE_				=				skip_units(_lowercase       )
																																elif "# Below: " in line and "##" not in line:
																																							SCREAMING_SNAKE_CASE_				=				line.split('"'       )[1]
																																							SCREAMING_SNAKE_CASE_				=				skip_units(_lowercase       )
																																elif "# End." in line and "##" not in line:
																																							if not skip_file and not skip_snippet:
																																														replace(_lowercase,     _lowercase,     _lowercase       )

																																							SCREAMING_SNAKE_CASE_				=				[]
																																elif "# Replace with" in line and "##" not in line:
																																							SCREAMING_SNAKE_CASE_				=				[]
																																elif "##" not in line:
																																							lines_to_copy.append(_lowercase       )

																		remove(_lowercase       )

											replace_in_files(f"""{directory}/to_replace_{lowercase_model_name}.py"""       )
											os.rmdir(_lowercase       )





