


import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE							    =							get_tests_dir('fixtures/test_sentencepiece.model')


@require_sentencepiece
class     __UpperCAmelCase							(       __A ,					unittest.TestCase							):


     """simple docstring"""
     _lowerCamelCase   =      XLMProphetNetTokenizer
     _lowerCamelCase   =      False
     _lowerCamelCase   =      True


     def 	snake_case_  (	self    ):
        super().setUp()

        # We have a SentencePiece fixture for testing
        __a					=  XLMProphetNetTokenizer(__A  ,       keep_accents=__A    )
        tokenizer.save_pretrained(self.tmpdirname    )


     def 	snake_case_  (	self    ):
        __a					=  """[PAD]"""
        __a					=  0

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(__A    )  ,       __A    )
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(__A    )  ,       __A    )


     def 	snake_case_  (	self    ):
        __a					=  list(self.get_tokenizer().get_vocab().keys()    )

        self.assertEqual(vocab_keys[0]  ,       """[PAD]"""    )
        self.assertEqual(vocab_keys[1]  ,       """[CLS]"""    )
        self.assertEqual(vocab_keys[-1]  ,       """j"""    )
        self.assertEqual(len(__A    )  ,       1012    )


     def 	snake_case_  (	self    ):
        self.assertEqual(self.get_tokenizer().vocab_size  ,       1012    )


     def 	snake_case_  (	self    ):
        __a					=  XLMProphetNetTokenizer(__A  ,       keep_accents=__A    )

        __a					=  tokenizer.tokenize("""This is a test"""    )
        self.assertListEqual(__A  ,       ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]    )

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(__A    )  ,       [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]  ,       )

        __a					=  tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
        self.assertListEqual(
            __A  ,       [
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """9""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """s""",
                """é""",
                """.""",
            ]  ,       )
        __a					=  tokenizer.convert_tokens_to_ids(__A    )
        self.assertListEqual(
            __A  ,       [
                value + tokenizer.fairseq_offset
                for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
            ]  ,       )

        __a					=  tokenizer.convert_ids_to_tokens(__A    )
        self.assertListEqual(
            __A  ,       [
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """[UNK]""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """s""",
                """[UNK]""",
                """.""",
            ]  ,       )


     @cached_property
     def 	snake_case_  (	self    ):
        return XLMProphetNetTokenizer.from_pretrained("""microsoft/xprophetnet-large-wiki100-cased"""    )


     @slow
     def 	snake_case_  (	self    ):
        __a					=  """Hello World!"""
        __a					=  [35389, 6672, 49, 2]
        self.assertListEqual(__A  ,       self.big_tokenizer.encode(__A    )    )


     @slow
     def 	snake_case_  (	self    ):
        # fmt: off
        __a					=  {"""input_ids""": [[11073, 82783, 18, 26, 82783, 549, 51540, 248, 17209, 1301, 217, 20, 215186, 1325, 147, 17209, 1301, 217, 20, 56370, 53, 122020, 20, 16477, 27, 87355, 4548, 20, 4728, 78392, 17, 159969, 18, 26, 24491, 629, 15, 538, 22704, 5439, 15, 2788, 24491, 9885, 15, 43534, 605, 15, 814, 18403, 33200, 29, 15, 43534, 24458, 12410, 111, 24966, 83669, 9637, 144068, 26, 850, 22346, 27, 147, 24966, 83669, 83490, 26, 39113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 122020, 115785, 34, 816, 1339, 46887, 18, 147, 53905, 1951, 42238, 41170, 17732, 834, 436, 15, 27523, 98733, 217, 147, 5542, 4981, 930, 17347, 16, 2], [20091, 629, 94, 82786, 58, 490, 20, 1528, 84, 53905, 344, 80592, 110128, 18822, 5267, 1306, 62, 152537, 308, 7997, 401, 124427, 549, 35442, 225, 109, 15055, 25748, 147, 7119, 43712, 34, 767, 135366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63784, 119466, 17, 147808, 88214, 18, 656, 81, 32, 3296, 10280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=__A  ,       model_name="""microsoft/xprophetnet-large-wiki100-cased"""  ,       revision="""1acad1643ddd54a44df6a1b797ada8373685d90e"""  ,       )










'''simple docstring'''






import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
					from cookiecutter.main import cookiecutter

					lowerCAmelCase									=   True
except ImportError:
					lowerCAmelCase									=   False

lowerCAmelCase									=   logging.get_logger(__name__)  # pylint: disable=invalid-name






def   __A    (				a_    :	Namespace						):
		return AddNewModelCommand(args.testing   ,args.testing_file   ,path=args.path						)

class 		lowerCamelCase	(   _A							):



	@staticmethod
	def 	_lowerCamelCase     ( a_						):
			lowerCAmelCase  :     Optional[int]       						= parser.add_parser("add-new-model"						)
			add_new_model_parser.add_argument("--testing"		,   action="store_true"		,   help="If in testing mode."						)
			add_new_model_parser.add_argument("--testing_file"		,   type=a_		,   help="Configuration file on which to run."						)
			add_new_model_parser.add_argument(
			    "--path"		,   type=a_		,   help="Path to cookiecutter. Should only be used for testing purposes."						)
			add_new_model_parser.set_defaults(func=a_						)



	def __init__( self		,   a_		,   a_		,   a_=None		,   *a_						):
			lowerCAmelCase  :     Any       						= testing
			lowerCAmelCase  :     str       						= testing_file
			lowerCAmelCase  :     Optional[int]       						= path





	def 	_lowerCamelCase     ( self						):
			warnings.warn(
			    "The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. "
			    "It is not actively maintained anymore, so might give a result that won't pass all tests and quality "
			    "checks, you should use `transformers-cli add-new-model-like` instead."						)
			if not _has_cookiecutter:
					raise ImportError(
					    "Model creation dependencies are required to use the `add_new_model` command. Install them by running "
					    "the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n"						)
			# Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
			lowerCAmelCase  :     Optional[Any]       						= [directory for directory in os.listdir() if "cookiecutter-template-" == directory[:22]]
			if len(a_						) > 0:
					raise ValueError(
					    "Several directories starting with `cookiecutter-template-` in current working directory. "
					    "Please clean your directory by removing all folders starting with `cookiecutter-template-` or "
					    "change your working directory."						)

			lowerCAmelCase  :     List[str]       						= (
			    Path(a_						).parent.parent.parent.parent if self._path is None else Path(self._path						).parent.parent
			)
			lowerCAmelCase  :     str       						= path_to_transformer_root / "templates" / "adding_a_new_model"

			# Execute cookiecutter
			if not self._testing:
					cookiecutter(str(a_						)						)
			else:
					with open(self._testing_file		,   "r"						) as configuration_file:
							lowerCAmelCase  :     Dict       						= json.load(a_						)

					cookiecutter(
					    str(path_to_cookiecutter if self._path is None else self._path						)		,   no_input=a_		,   extra_context=a_		,   )

			lowerCAmelCase  :     Optional[Any]       						= [directory for directory in os.listdir() if "cookiecutter-template-" in directory[:22]][0]

			# Retrieve configuration
			with open(directory + "/configuration.json"		,   "r"						) as configuration_file:
					lowerCAmelCase  :     str       						= json.load(a_						)

			lowerCAmelCase  :     List[str]       						= configuration["lowercase_modelname"]
			lowerCAmelCase  :     List[Any]       						= configuration["generate_tensorflow_pytorch_and_flax"]
			os.remove(F'''{directory}/configuration.json'''						)

			lowerCAmelCase  :     Optional[int]       						= "PyTorch" in generate_tensorflow_pytorch_and_flax
			lowerCAmelCase  :     Optional[int]       						= "TensorFlow" in generate_tensorflow_pytorch_and_flax
			lowerCAmelCase  :     Optional[Any]       						= "Flax" in generate_tensorflow_pytorch_and_flax

			lowerCAmelCase  :     List[str]       						= F'''{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}'''
			os.makedirs(a_		,   exist_ok=a_						)
			os.makedirs(F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}'''		,   exist_ok=a_						)

			# Tests require submodules as they have parent imports
			with open(F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py'''		,   "w"						):
					pass

			shutil.move(
			    F'''{directory}/__init__.py'''		,   F'''{model_dir}/__init__.py'''		,   )
			shutil.move(
			    F'''{directory}/configuration_{lowercase_model_name}.py'''		,   F'''{model_dir}/configuration_{lowercase_model_name}.py'''		,   )

			def remove_copy_lines(a_						):
					with open(a_		,   "r"						) as f:
							lowerCAmelCase  :     Dict       						= f.readlines()
					with open(a_		,   "w"						) as f:
							for line in lines:
									if "# Copied from transformers." not in line:
											f.write(a_						)

			if output_pytorch:
					if not self._testing:
							remove_copy_lines(F'''{directory}/modeling_{lowercase_model_name}.py'''						)

					shutil.move(
					    F'''{directory}/modeling_{lowercase_model_name}.py'''		,   F'''{model_dir}/modeling_{lowercase_model_name}.py'''		,   )

					shutil.move(
					    F'''{directory}/test_modeling_{lowercase_model_name}.py'''		,   F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py'''		,   )
			else:
					os.remove(F'''{directory}/modeling_{lowercase_model_name}.py'''						)
					os.remove(F'''{directory}/test_modeling_{lowercase_model_name}.py'''						)

			if output_tensorflow:
					if not self._testing:
							remove_copy_lines(F'''{directory}/modeling_tf_{lowercase_model_name}.py'''						)

					shutil.move(
					    F'''{directory}/modeling_tf_{lowercase_model_name}.py'''		,   F'''{model_dir}/modeling_tf_{lowercase_model_name}.py'''		,   )

					shutil.move(
					    F'''{directory}/test_modeling_tf_{lowercase_model_name}.py'''		,   F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py'''		,   )
			else:
					os.remove(F'''{directory}/modeling_tf_{lowercase_model_name}.py'''						)
					os.remove(F'''{directory}/test_modeling_tf_{lowercase_model_name}.py'''						)

			if output_flax:
					if not self._testing:
							remove_copy_lines(F'''{directory}/modeling_flax_{lowercase_model_name}.py'''						)

					shutil.move(
					    F'''{directory}/modeling_flax_{lowercase_model_name}.py'''		,   F'''{model_dir}/modeling_flax_{lowercase_model_name}.py'''		,   )

					shutil.move(
					    F'''{directory}/test_modeling_flax_{lowercase_model_name}.py'''		,   F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py'''		,   )
			else:
					os.remove(F'''{directory}/modeling_flax_{lowercase_model_name}.py'''						)
					os.remove(F'''{directory}/test_modeling_flax_{lowercase_model_name}.py'''						)

			shutil.move(
			    F'''{directory}/{lowercase_model_name}.md'''		,   F'''{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md'''		,   )

			shutil.move(
			    F'''{directory}/tokenization_{lowercase_model_name}.py'''		,   F'''{model_dir}/tokenization_{lowercase_model_name}.py'''		,   )

			shutil.move(
			    F'''{directory}/tokenization_fast_{lowercase_model_name}.py'''		,   F'''{model_dir}/tokenization_{lowercase_model_name}_fast.py'''		,   )

			from os import fdopen, remove
			from shutil import copymode, move
			from tempfile import mkstemp

			def replace(a_		,   a_		,   a_						):
					# Create temp file
					lowerCAmelCase						,		lowerCAmelCase  :     Tuple       						= mkstemp()
					lowerCAmelCase  :     int       						= False
					with fdopen(a_		,   "w"						) as new_file:
							with open(a_						) as old_file:
									for line in old_file:
											new_file.write(a_						)
											if line_to_copy_below in line:
													lowerCAmelCase  :     Optional[int]       						= True
													for line_to_copy in lines_to_copy:
															new_file.write(a_						)

					if not line_found:
							raise ValueError(F'''Line {line_to_copy_below} was not found in file.'''						)

					# Copy the file permissions from the old file to the new file
					copymode(a_		,   a_						)
					# Remove original file
					remove(a_						)
					# Move new file
					move(a_		,   a_						)

			def skip_units(a_						):
					return (
					    ("generating PyTorch" in line and not output_pytorch)
					    or ("generating TensorFlow" in line and not output_tensorflow)
					    or ("generating Flax" in line and not output_flax)
					)

			def replace_in_files(a_						):
					with open(a_						) as datafile:
							lowerCAmelCase  :     Dict       						= []
							lowerCAmelCase  :     Optional[int]       						= False
							lowerCAmelCase  :     List[Any]       						= False
							for line in datafile:
									if "# To replace in: " in line and "##" not in line:
											lowerCAmelCase  :     Optional[int]       						= line.split("\""						)[1]
											lowerCAmelCase  :     Tuple       						= skip_units(a_						)
									elif "# Below: " in line and "##" not in line:
											lowerCAmelCase  :     Any       						= line.split("\""						)[1]
											lowerCAmelCase  :     List[str]       						= skip_units(a_						)
									elif "# End." in line and "##" not in line:
											if not skip_file and not skip_snippet:
													replace(a_		,   a_		,   a_						)

											lowerCAmelCase  :     List[str]       						= []
									elif "# Replace with" in line and "##" not in line:
											lowerCAmelCase  :     Any       						= []
									elif "##" not in line:
											lines_to_copy.append(a_						)

					remove(a_						)

			replace_in_files(F'''{directory}/to_replace_{lowercase_model_name}.py'''						)
			os.rmdir(a_						)



