

'''simple docstring'''

import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class    snake_case     (      lowercase_,						unittest.TestCase		):






				"""simple docstring"""
				_a    	=     CodeGenTokenizer
				_a    	=     CodeGenTokenizerFast
				_a    	=     True
				_a    	=     {"""add_prefix_space""": True}
				_a    	=     False



				def   a__					(    self       )     ->							int:
											super().setUp()

											# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
											SCREAMING_SNAKE_CASE_				=				[
											    'l',
											    'o',
											    'w',
											    'e',
											    'r',
											    's',
											    't',
											    'i',
											    'd',
											    'n',
											    '\u0120',
											    '\u0120l',
											    '\u0120n',
											    '\u0120lo',
											    '\u0120low',
											    'er',
											    '\u0120lowest',
											    '\u0120newer',
											    '\u0120wider',
											    '<unk>',
											    '<|endoftext|>',
											]
											SCREAMING_SNAKE_CASE_				=				dict(zip(_lowercase,     range(len(_lowercase       )       )       )       )
											SCREAMING_SNAKE_CASE_				=				['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
											SCREAMING_SNAKE_CASE_				=				{'unk_token': '<unk>'}

											SCREAMING_SNAKE_CASE_				=				os.path.join(self.tmpdirname,     VOCAB_FILES_NAMES['vocab_file']       )
											SCREAMING_SNAKE_CASE_				=				os.path.join(self.tmpdirname,     VOCAB_FILES_NAMES['merges_file']       )
											with open(self.vocab_file,     'w',     encoding='utf-8'       ) as fp:
																		fp.write(json.dumps(_lowercase       ) + '\n'       )
											with open(self.merges_file,     'w',     encoding='utf-8'       ) as fp:
																		fp.write('\n'.join(_lowercase       )       )



				def   a__					(    self,     **_lowercase       )     ->							List[Any]:
											kwargs.update(self.special_tokens_map       )
											return CodeGenTokenizer.from_pretrained(self.tmpdirname,     **_lowercase       )



				def   a__					(    self,     **_lowercase       )     ->							Tuple:
											kwargs.update(self.special_tokens_map       )
											return CodeGenTokenizerFast.from_pretrained(self.tmpdirname,     **_lowercase       )



				def   a__					(    self,     _lowercase       )     ->							int:
											SCREAMING_SNAKE_CASE_				=				'lower newer'
											SCREAMING_SNAKE_CASE_				=				'lower newer'
											return input_text, output_text



				def   a__					(    self       )     ->							Any:
											SCREAMING_SNAKE_CASE_				=				CodeGenTokenizer(self.vocab_file,     self.merges_file,     **self.special_tokens_map       )
											SCREAMING_SNAKE_CASE_				=				'lower newer'
											SCREAMING_SNAKE_CASE_				=				['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
											SCREAMING_SNAKE_CASE_				=				tokenizer.tokenize(_lowercase,     add_prefix_space=_lowercase       )
											self.assertListEqual(_lowercase,     _lowercase       )

											SCREAMING_SNAKE_CASE_				=				tokens + [tokenizer.unk_token]
											SCREAMING_SNAKE_CASE_				=				[14, 15, 10, 9, 3, 2, 15, 19]
											self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase       ),     _lowercase       )



				def   a__					(    self       )     ->							int:
											if not self.test_rust_tokenizer:
																		return

											SCREAMING_SNAKE_CASE_				=				self.get_tokenizer()
											SCREAMING_SNAKE_CASE_				=				self.get_rust_tokenizer(add_prefix_space=_lowercase       )

											SCREAMING_SNAKE_CASE_				=				'lower newer'

											# Testing tokenization
											SCREAMING_SNAKE_CASE_				=				tokenizer.tokenize(_lowercase,     add_prefix_space=_lowercase       )
											SCREAMING_SNAKE_CASE_				=				rust_tokenizer.tokenize(_lowercase       )
											self.assertListEqual(_lowercase,     _lowercase       )

											# Testing conversion to ids without special tokens
											SCREAMING_SNAKE_CASE_				=				tokenizer.encode(_lowercase,     add_special_tokens=_lowercase,     add_prefix_space=_lowercase       )
											SCREAMING_SNAKE_CASE_				=				rust_tokenizer.encode(_lowercase,     add_special_tokens=_lowercase       )
											self.assertListEqual(_lowercase,     _lowercase       )

											# Testing conversion to ids with special tokens
											SCREAMING_SNAKE_CASE_				=				self.get_rust_tokenizer(add_prefix_space=_lowercase       )
											SCREAMING_SNAKE_CASE_				=				tokenizer.encode(_lowercase,     add_prefix_space=_lowercase       )
											SCREAMING_SNAKE_CASE_				=				rust_tokenizer.encode(_lowercase       )
											self.assertListEqual(_lowercase,     _lowercase       )

											# Testing the unknown token
											SCREAMING_SNAKE_CASE_				=				tokens + [rust_tokenizer.unk_token]
											SCREAMING_SNAKE_CASE_				=				[14, 15, 10, 9, 3, 2, 15, 19]
											self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(_lowercase       ),     _lowercase       )



				def   a__					(    self,     *_lowercase,     **_lowercase       )     ->							List[Any]:
											# It's very difficult to mix/test pretokenization with byte-level
											# And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
											pass



				def   a__					(    self,     _lowercase=15       )     ->							Tuple:
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
																									SCREAMING_SNAKE_CASE_				=				self.rust_tokenizer_class.from_pretrained(_lowercase,     **_lowercase       )

																									# Simple input
																									SCREAMING_SNAKE_CASE_				=				'This is a simple input'
																									SCREAMING_SNAKE_CASE_				=				['This is a simple input 1', 'This is a simple input 2']
																									SCREAMING_SNAKE_CASE_				=				('This is a simple input', 'This is a pair')
																									SCREAMING_SNAKE_CASE_				=				[
																									    ('This is a simple input 1', 'This is a simple input 2'),
																									    ('This is a simple pair 1', 'This is a simple pair 2'),
																									]

																									# Simple input tests
																									self.assertRaises(_lowercase,     tokenizer_r.encode,     _lowercase,     max_length=_lowercase,     padding='max_length'       )

																									# Simple input
																									self.assertRaises(_lowercase,     tokenizer_r.encode_plus,     _lowercase,     max_length=_lowercase,     padding='max_length'       )

																									# Simple input
																									self.assertRaises(
																									    _lowercase,     tokenizer_r.batch_encode_plus,     _lowercase,     max_length=_lowercase,     padding='max_length',     )

																									# Pair input
																									self.assertRaises(_lowercase,     tokenizer_r.encode,     _lowercase,     max_length=_lowercase,     padding='max_length'       )

																									# Pair input
																									self.assertRaises(_lowercase,     tokenizer_r.encode_plus,     _lowercase,     max_length=_lowercase,     padding='max_length'       )

																									# Pair input
																									self.assertRaises(
																									    _lowercase,     tokenizer_r.batch_encode_plus,     _lowercase,     max_length=_lowercase,     padding='max_length',     )



				def   a__					(    self       )     ->							Optional[Any]:
											SCREAMING_SNAKE_CASE_				=				CodeGenTokenizer.from_pretrained(self.tmpdirname,     pad_token='<pad>'       )

											# Simple input
											SCREAMING_SNAKE_CASE_				=				'This is a simple input'
											SCREAMING_SNAKE_CASE_				=				['This is a simple input looooooooong', 'This is a simple input']
											SCREAMING_SNAKE_CASE_				=				('This is a simple input', 'This is a pair')
											SCREAMING_SNAKE_CASE_				=				[
											    ('This is a simple input loooooong', 'This is a simple input'),
											    ('This is a simple pair loooooong', 'This is a simple pair'),
											]

											SCREAMING_SNAKE_CASE_				=				tokenizer.pad_token_id

											SCREAMING_SNAKE_CASE_				=				tokenizer(_lowercase,     padding='max_length',     max_length=30,     return_tensors='np'       )
											SCREAMING_SNAKE_CASE_				=				tokenizer(_lowercase,     padding=_lowercase,     truncate=_lowercase,     return_tensors='np'       )
											SCREAMING_SNAKE_CASE_				=				tokenizer(*_lowercase,     padding='max_length',     max_length=60,     return_tensors='np'       )
											SCREAMING_SNAKE_CASE_				=				tokenizer(_lowercase,     padding=_lowercase,     truncate=_lowercase,     return_tensors='np'       )

											# s
											# test single string max_length padding
											self.assertEqual(out_s['input_ids'].shape[-1],     30       )
											self.assertTrue(pad_token_id in out_s['input_ids']       )
											self.assertTrue(0 in out_s['attention_mask']       )

											# s2
											# test automatic padding
											self.assertEqual(out_sa['input_ids'].shape[-1],     33       )
											# long slice doesn't have padding
											self.assertFalse(pad_token_id in out_sa['input_ids'][0]       )
											self.assertFalse(0 in out_sa['attention_mask'][0]       )
											# short slice does have padding
											self.assertTrue(pad_token_id in out_sa['input_ids'][1]       )
											self.assertTrue(0 in out_sa['attention_mask'][1]       )

											# p
											# test single pair max_length padding
											self.assertEqual(out_p['input_ids'].shape[-1],     60       )
											self.assertTrue(pad_token_id in out_p['input_ids']       )
											self.assertTrue(0 in out_p['attention_mask']       )

											# p2
											# test automatic padding pair
											self.assertEqual(out_pa['input_ids'].shape[-1],     52       )
											# long slice pair doesn't have padding
											self.assertFalse(pad_token_id in out_pa['input_ids'][0]       )
											self.assertFalse(0 in out_pa['attention_mask'][0]       )
											# short slice pair does have padding
											self.assertTrue(pad_token_id in out_pa['input_ids'][1]       )
											self.assertTrue(0 in out_pa['attention_mask'][1]       )



				def   a__					(    self       )     ->							List[Any]:
											SCREAMING_SNAKE_CASE_				=				'$$$'
											SCREAMING_SNAKE_CASE_				=				CodeGenTokenizer.from_pretrained(self.tmpdirname,     bos_token=_lowercase,     add_bos_token=_lowercase       )

											SCREAMING_SNAKE_CASE_				=				'This is a simple input'
											SCREAMING_SNAKE_CASE_				=				['This is a simple input 1', 'This is a simple input 2']

											SCREAMING_SNAKE_CASE_				=				tokenizer.bos_token_id

											SCREAMING_SNAKE_CASE_				=				tokenizer(_lowercase       )
											SCREAMING_SNAKE_CASE_				=				tokenizer(_lowercase       )

											self.assertEqual(out_s.input_ids[0],     _lowercase       )
											self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids       )       )

											SCREAMING_SNAKE_CASE_				=				tokenizer.decode(out_s.input_ids       )
											SCREAMING_SNAKE_CASE_				=				tokenizer.batch_decode(out_sa.input_ids       )

											self.assertEqual(decode_s.split()[0],     _lowercase       )
											self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa       )       )



				@slow
				def   a__					(    self       )     ->							Dict:
											SCREAMING_SNAKE_CASE_				=				CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono'       )

											SCREAMING_SNAKE_CASE_				=				'\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
											SCREAMING_SNAKE_CASE_				=				'\nif len_a > len_b:      result = a\nelse:      result = b'

											SCREAMING_SNAKE_CASE_				=				tokenizer.encode(_lowercase       )
											SCREAMING_SNAKE_CASE_				=				['^#', re.escape('<|endoftext|>'       ), '^\'\'\'', '^"""', '\n\n\n']
											SCREAMING_SNAKE_CASE_				=				tokenizer.decode(_lowercase,     truncate_before_pattern=_lowercase       )
											self.assertEqual(_lowercase,     _lowercase       )





				def   a__					(    self       )     ->							Any:
											pass








'''simple docstring'''

import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
							from cookiecutter.main import cookiecutter

							SCREAMING_SNAKE_CASE							:				Dict     	=			True
except ImportError:
							SCREAMING_SNAKE_CASE							:				Optional[Any]     	=			False

SCREAMING_SNAKE_CASE							:				str     	=			logging.get_logger(__name__)  # pylint: disable=invalid-name




def 							_UpperCamelCase				(   lowerCAmelCase__:					Namespace   )      -> Optional[Any]:
							return AddNewModelCommand(args.testing		,args.testing_file		,path=args.path   )






class    snake_case     (      lowercase_		):






				"""simple docstring"""



				@staticmethod
				def   a__					(    _lowercase       )     ->							Tuple:
											SCREAMING_SNAKE_CASE_				=				parser.add_parser('add-new-model'       )
											add_new_model_parser.add_argument('--testing',     action='store_true',     help='If in testing mode.'       )
											add_new_model_parser.add_argument('--testing_file',     type=_lowercase,     help='Configuration file on which to run.'       )
											add_new_model_parser.add_argument(
											    '--path',     type=_lowercase,     help='Path to cookiecutter. Should only be used for testing purposes.'       )
											add_new_model_parser.set_defaults(func=_lowercase       )



				def __init__(    self,     _lowercase,     _lowercase,     _lowercase=None,     *_lowercase       )     ->							Optional[Any]:
											SCREAMING_SNAKE_CASE_				=				testing
											SCREAMING_SNAKE_CASE_				=				testing_file
											SCREAMING_SNAKE_CASE_				=				path





				def   a__					(    self       )     ->							Optional[int]:
											warnings.warn(
											    'The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. '
											    'It is not actively maintained anymore, so might give a result that won\'t pass all tests and quality '
											    'checks, you should use `transformers-cli add-new-model-like` instead.'       )
											if not _has_cookiecutter:
																		raise ImportError(
																		    'Model creation dependencies are required to use the `add_new_model` command. Install them by running '
																		    'the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n'       )
											# Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
											SCREAMING_SNAKE_CASE_				=				[directory for directory in os.listdir() if 'cookiecutter-template-' == directory[:22]]
											if len(_lowercase       ) > 0:
																		raise ValueError(
																		    'Several directories starting with `cookiecutter-template-` in current working directory. '
																		    'Please clean your directory by removing all folders starting with `cookiecutter-template-` or '
																		    'change your working directory.'       )

											SCREAMING_SNAKE_CASE_				=				(
											    Path(_lowercase       ).parent.parent.parent.parent if self._path is None else Path(self._path       ).parent.parent
											)
											SCREAMING_SNAKE_CASE_				=				path_to_transformer_root / 'templates' / 'adding_a_new_model'

											# Execute cookiecutter
											if not self._testing:
																		cookiecutter(str(_lowercase       )       )
											else:
																		with open(self._testing_file,     'r'       ) as configuration_file:
																									SCREAMING_SNAKE_CASE_				=				json.load(_lowercase       )

																		cookiecutter(
																		    str(path_to_cookiecutter if self._path is None else self._path       ),     no_input=_lowercase,     extra_context=_lowercase,     )

											SCREAMING_SNAKE_CASE_				=				[directory for directory in os.listdir() if 'cookiecutter-template-' in directory[:22]][0]

											# Retrieve configuration
											with open(directory + '/configuration.json',     'r'       ) as configuration_file:
																		SCREAMING_SNAKE_CASE_				=				json.load(_lowercase       )

											SCREAMING_SNAKE_CASE_				=				configuration['lowercase_modelname']
											SCREAMING_SNAKE_CASE_				=				configuration['generate_tensorflow_pytorch_and_flax']
											os.remove(f"""{directory}/configuration.json"""       )

											SCREAMING_SNAKE_CASE_				=				'PyTorch' in generate_tensorflow_pytorch_and_flax
											SCREAMING_SNAKE_CASE_				=				'TensorFlow' in generate_tensorflow_pytorch_and_flax
											SCREAMING_SNAKE_CASE_				=				'Flax' in generate_tensorflow_pytorch_and_flax

											SCREAMING_SNAKE_CASE_				=				f"""{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}"""
											os.makedirs(_lowercase,     exist_ok=_lowercase       )
											os.makedirs(f"""{path_to_transformer_root}/tests/models/{lowercase_model_name}""",     exist_ok=_lowercase       )

											# Tests require submodules as they have parent imports
											with open(f"""{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py""",     'w'       ):
																		pass

											shutil.move(
											    f"""{directory}/__init__.py""",     f"""{model_dir}/__init__.py""",     )
											shutil.move(
											    f"""{directory}/configuration_{lowercase_model_name}.py""",     f"""{model_dir}/configuration_{lowercase_model_name}.py""",     )

											def remove_copy_lines(_lowercase       ):
																		with open(_lowercase,     'r'       ) as f:
																									SCREAMING_SNAKE_CASE_				=				f.readlines()
																		with open(_lowercase,     'w'       ) as f:
																									for line in lines:
																																if "# Copied from transformers." not in line:
																																							f.write(_lowercase       )

											if output_pytorch:
																		if not self._testing:
																									remove_copy_lines(f"""{directory}/modeling_{lowercase_model_name}.py"""       )

																		shutil.move(
																		    f"""{directory}/modeling_{lowercase_model_name}.py""",     f"""{model_dir}/modeling_{lowercase_model_name}.py""",     )

																		shutil.move(
																		    f"""{directory}/test_modeling_{lowercase_model_name}.py""",     f"""{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py""",     )
											else:
																		os.remove(f"""{directory}/modeling_{lowercase_model_name}.py"""       )
																		os.remove(f"""{directory}/test_modeling_{lowercase_model_name}.py"""       )

											if output_tensorflow:
																		if not self._testing:
																									remove_copy_lines(f"""{directory}/modeling_tf_{lowercase_model_name}.py"""       )

																		shutil.move(
																		    f"""{directory}/modeling_tf_{lowercase_model_name}.py""",     f"""{model_dir}/modeling_tf_{lowercase_model_name}.py""",     )

																		shutil.move(
																		    f"""{directory}/test_modeling_tf_{lowercase_model_name}.py""",     f"""{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py""",     )
											else:
																		os.remove(f"""{directory}/modeling_tf_{lowercase_model_name}.py"""       )
																		os.remove(f"""{directory}/test_modeling_tf_{lowercase_model_name}.py"""       )

											if output_flax:
																		if not self._testing:
																									remove_copy_lines(f"""{directory}/modeling_flax_{lowercase_model_name}.py"""       )

																		shutil.move(
																		    f"""{directory}/modeling_flax_{lowercase_model_name}.py""",     f"""{model_dir}/modeling_flax_{lowercase_model_name}.py""",     )

																		shutil.move(
																		    f"""{directory}/test_modeling_flax_{lowercase_model_name}.py""",     f"""{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py""",     )
											else:
																		os.remove(f"""{directory}/modeling_flax_{lowercase_model_name}.py"""       )
																		os.remove(f"""{directory}/test_modeling_flax_{lowercase_model_name}.py"""       )

											shutil.move(
											    f"""{directory}/{lowercase_model_name}.md""",     f"""{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md""",     )

											shutil.move(
											    f"""{directory}/tokenization_{lowercase_model_name}.py""",     f"""{model_dir}/tokenization_{lowercase_model_name}.py""",     )

											shutil.move(
											    f"""{directory}/tokenization_fast_{lowercase_model_name}.py""",     f"""{model_dir}/tokenization_{lowercase_model_name}_fast.py""",     )

											from os import fdopen, remove
											from shutil import copymode, move
											from tempfile import mkstemp

											def replace(_lowercase,     _lowercase,     _lowercase       ):
																		# Create temp file
																		SCREAMING_SNAKE_CASE_				,    SCREAMING_SNAKE_CASE_				=				mkstemp()
																		SCREAMING_SNAKE_CASE_				=				False
																		with fdopen(_lowercase,     'w'       ) as new_file:
																									with open(_lowercase       ) as old_file:
																																for line in old_file:
																																							new_file.write(_lowercase       )
																																							if line_to_copy_below in line:
																																														SCREAMING_SNAKE_CASE_				=				True
																																														for line_to_copy in lines_to_copy:
																																																					new_file.write(_lowercase       )

																		if not line_found:
																									raise ValueError(f"""Line {line_to_copy_below} was not found in file."""       )

																		# Copy the file permissions from the old file to the new file
																		copymode(_lowercase,     _lowercase       )
																		# Remove original file
																		remove(_lowercase       )
																		# Move new file
																		move(_lowercase,     _lowercase       )

											def skip_units(_lowercase       ):
																		return (
																		    ("generating PyTorch" in line and not output_pytorch)
																		    or ("generating TensorFlow" in line and not output_tensorflow)
																		    or ("generating Flax" in line and not output_flax)
																		)

											def replace_in_files(_lowercase       ):
																		with open(_lowercase       ) as datafile:
																									SCREAMING_SNAKE_CASE_				=				[]
																									SCREAMING_SNAKE_CASE_				=				False
																									SCREAMING_SNAKE_CASE_				=				False
																									for line in datafile:
																																if "# To replace in: " in line and "##" not in line:
																																							SCREAMING_SNAKE_CASE_				=				line.split('"'       )[1]
																																							SCREAMING_SNAKE_CASE_				=				skip_units(_lowercase       )
																																elif "# Below: " in line and "##" not in line:
																																							SCREAMING_SNAKE_CASE_				=				line.split('"'       )[1]
																																							SCREAMING_SNAKE_CASE_				=				skip_units(_lowercase       )
																																elif "# End." in line and "##" not in line:
																																							if not skip_file and not skip_snippet:
																																														replace(_lowercase,     _lowercase,     _lowercase       )

																																							SCREAMING_SNAKE_CASE_				=				[]
																																elif "# Replace with" in line and "##" not in line:
																																							SCREAMING_SNAKE_CASE_				=				[]
																																elif "##" not in line:
																																							lines_to_copy.append(_lowercase       )

																		remove(_lowercase       )

											replace_in_files(f"""{directory}/to_replace_{lowercase_model_name}.py"""       )
											os.rmdir(_lowercase       )





