



"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


UpperCamelCase     =							{
    """configuration_mega""": ["""MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP""", """MegaConfig""", """MegaOnnxConfig"""],
}

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			UpperCamelCase     =							[
			    """MEGA_PRETRAINED_MODEL_ARCHIVE_LIST""",
			    """MegaForCausalLM""",
			    """MegaForMaskedLM""",
			    """MegaForMultipleChoice""",
			    """MegaForQuestionAnswering""",
			    """MegaForSequenceClassification""",
			    """MegaForTokenClassification""",
			    """MegaModel""",
			    """MegaPreTrainedModel""",
			]

if TYPE_CHECKING:
			from .configuration_mega import MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP, MegaConfig, MegaOnnxConfig

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_mega import (
						    MEGA_PRETRAINED_MODEL_ARCHIVE_LIST,
						    MegaForCausalLM,
						    MegaForMaskedLM,
						    MegaForMultipleChoice,
						    MegaForQuestionAnswering,
						    MegaForSequenceClassification,
						    MegaForTokenClassification,
						    MegaModel,
						    MegaPreTrainedModel,
						)

else:
			import sys

			UpperCamelCase     =							_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)











import os
import unittest

from transformers.models.bartpho.tokenization_bartpho import VOCAB_FILES_NAMES, BartphoTokenizer
from transformers.testing_utils import get_tests_dir

from ...test_tokenization_common import TokenizerTesterMixin


_A					:				Dict				=			get_tests_dir("""fixtures/test_sentencepiece_bpe.model""")



class 					__snake_case	(  __SCREAMING_SNAKE_CASE			,     unittest.TestCase    ):


    '''simple docstring'''







    lowerCamelCase__    :		Any				=	BartphoTokenizer
    lowerCamelCase__    :		Tuple				=	False
    lowerCamelCase__    :		Dict				=	True



    def  lowercase_					(       self     ):





        '''simple docstring'''

        super().setUp()

        SCREAMING_SNAKE_CASE__											=			['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']
        SCREAMING_SNAKE_CASE__											=			dict(zip(A_   ,				range(len(A_     )     )     )     )
        SCREAMING_SNAKE_CASE__											=			{'''unk_token''': '''<unk>'''}

        SCREAMING_SNAKE_CASE__											=			os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''monolingual_vocab_file''']     )
        with open(self.monolingual_vocab_file   ,				'''w'''   ,				encoding='''utf-8'''     ) as fp:
            for token in vocab_tokens:
                fp.write(f'''{token} {vocab_tokens[token]}\n'''     )

        SCREAMING_SNAKE_CASE__											=			BartphoTokenizer(A_   ,				self.monolingual_vocab_file   ,				**self.special_tokens_map     )
        tokenizer.save_pretrained(self.tmpdirname     )



    def  lowercase_					(       self   ,				**A_     ):





        '''simple docstring'''

        kwargs.update(self.special_tokens_map     )
        return BartphoTokenizer.from_pretrained(self.tmpdirname   ,				**A_     )



    def  lowercase_					(       self   ,				A_     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			'''This is a là test'''
        SCREAMING_SNAKE_CASE__											=			'''This is a<unk><unk> test'''
        return input_text, output_text




    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			BartphoTokenizer(A_   ,				self.monolingual_vocab_file   ,				**self.special_tokens_map     )
        SCREAMING_SNAKE_CASE__											=			'''This is a là test'''
        SCREAMING_SNAKE_CASE__											=			'''▁This ▁is ▁a ▁l à ▁t est'''.split()
        SCREAMING_SNAKE_CASE__											=			tokenizer.tokenize(A_     )
        self.assertListEqual(A_   ,				A_     )

        SCREAMING_SNAKE_CASE__											=			tokens + [tokenizer.unk_token]
        SCREAMING_SNAKE_CASE__											=			[4, 5, 6, 3, 3, 7, 8, 3]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(A_     )   ,				A_     )
