



import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import MaMaaaTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)
from transformers.utils import is_sentencepiece_available


if is_sentencepiece_available():
				from transformers.models.mam_aaa.tokenization_mam_aaa import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


if is_sentencepiece_available():
				__snake_case     :Tuple			       =get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
				from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__snake_case     :Optional[int]			       =128022
__snake_case     :int			       =128028



@require_sentencepiece
class        lowerCAmelCase__					(  _lowerCamelCase ,				unittest.TestCase       ):
				A_   :     Optional[int]								=     MaMaaaTokenizer
				A_   :     str								=     False
				A_   :     Union[str, Any]								=     False
				A_   :     Dict								=     True


				def __UpperCamelCase						(							self      :  int       )		->     Optional[Any]:
											super().setUp()

											A      						=    ['</s>', '<unk>', '▁This', '▁is', '▁a', '▁t', 'est', '\u0120', '<pad>']
											A      						=    dict(zip(__UpperCamelCase						, range(len(__UpperCamelCase       )       )       )       )
											A      						=    Path(self.tmpdirname       )
											save_json(__UpperCamelCase						, save_dir / VOCAB_FILES_NAMES['vocab_file']       )
											if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
																		copyfile(__UpperCamelCase						, save_dir / VOCAB_FILES_NAMES['spm_file']       )

											A      						=    MaMaaaTokenizer.from_pretrained(self.tmpdirname       )
											tokenizer.save_pretrained(self.tmpdirname       )


				def __UpperCamelCase						(							self      :  Optional[int]						, **__UpperCamelCase      :  List[str]       )		->     List[str]:
											return MaMaaaTokenizer.from_pretrained(self.tmpdirname						, **__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Optional[Any]						, __UpperCamelCase      :  Dict       )		->     int:
											return (
											    "This is a test",
											    "This is a test",
											)


				def __UpperCamelCase						(							self      :  str       )		->     int:
											A      						=    '</s>'
											A      						=    0

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(__UpperCamelCase       )						, __UpperCamelCase       )
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(__UpperCamelCase       )						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  int       )		->     Union[str, Any]:
											A      						=    self.get_tokenizer()
											A      						=    list(tokenizer.get_vocab().keys()       )

											self.assertEqual(vocab_keys[0]						, '</s>'       )
											self.assertEqual(vocab_keys[1]						, '<unk>'       )
											self.assertEqual(vocab_keys[-1]						, '<s>'       )
											self.assertEqual(len(__UpperCamelCase       )						, tokenizer.vocab_size + len(tokenizer.get_added_vocab()       )       )


				@unittest.skip('Skip this test while all models are still to be uploaded.'       )
				def __UpperCamelCase						(							self      :  List[str]       )		->     Dict:
											pass


				def __UpperCamelCase						(							self      :  List[str]       )		->     str:
											A      						=    self.get_tokenizer()

											A      						=    tokenizer.tokenize('This is a test'       )
											self.assertListEqual(__UpperCamelCase						, ['▁This', '▁is', '▁a', '▁t', 'est']       )

											self.assertListEqual(
											    tokenizer.convert_tokens_to_ids(__UpperCamelCase       )						, [2, 3, 4, 5, 6]						, )

											A      						=    tokenizer.convert_ids_to_tokens([2, 3, 4, 5, 6]       )
											self.assertListEqual(__UpperCamelCase						, ['▁This', '▁is', '▁a', '▁t', 'est']       )

											A      						=    tokenizer.convert_tokens_to_string(__UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, 'This is a test'       )


				@slow
				def __UpperCamelCase						(							self      :  str       )		->     Optional[Any]:
											# fmt: off
											A      						=    {'input_ids': [[128_022, 110_108, 397, 11, 38_272, 2_247, 124_811, 285, 18_105, 1_586, 207, 7, 39_534, 4_428, 397, 1_019, 18_105, 1_586, 207, 7, 41_337, 16_786, 241, 7, 20_214, 17, 125_690, 10_398, 7, 44_378, 58_069, 68_342, 7_798, 7_343, 11, 299, 33_310, 4, 158, 37_350, 94_077, 4_569, 299, 33_310, 90, 4, 52_840, 290, 4, 31_270, 112, 299, 682, 4, 52_840, 39_953, 14_079, 193, 52_519, 90_894, 17_894, 120_697, 11, 40_445, 551, 17, 1_019, 52_519, 90_894, 17_756, 963, 11, 40_445, 480, 17, 9_792, 1_120, 5_173, 1_393, 6_240, 16_786, 241, 120_996, 28, 1_245, 1_393, 118_240, 11_123, 1_019, 93_612, 2_691, 10_618, 98_058, 120_409, 1_928, 279, 4, 40_683, 367, 178, 207, 1_019, 103, 103_121, 506, 65_296, 5, 2], [128_022, 21_217, 367, 117, 125_450, 128, 719, 7, 7_308, 40, 93_612, 12_669, 1_116, 16_704, 71, 17_785, 3_699, 15_592, 35, 144, 9_584, 241, 11_943, 713, 950, 799, 2_247, 88_427, 150, 149, 118_813, 120_706, 1_019, 106_906, 81_518, 28, 1_224, 22_799, 397, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [128_022, 1_658, 123_311, 5_155, 5_578, 4_722, 279, 14_947, 2_366, 1_120, 1_197, 14, 1_348, 9_232, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=__UpperCamelCase						, model_name='facebook/m2m100_418M'						, revision='c168bae485c864188cf9aa0e4108b0b6934dc91e'						, )





@require_torch
@require_sentencepiece
@require_tokenizers
class        lowerCAmelCase__					(  unittest.TestCase       ):
				A_   :     Optional[Any]								=     'facebook/m2m100_418M'
				A_   :     Dict								=     [
				    'In my opinion, there are two levels of response from the French government.',
				    'NSA Affair Emphasizes Complete Lack of Debate on Intelligence',
				]
				A_   :     List[Any]								=     [
				    'Selon moi, il y a deux niveaux de réponse de la part du gouvernement français.',
				    'L\'affaire NSA souligne l\'absence totale de débat sur le renseignement',
				]

				# fmt: off
				A_   :     Dict								=     [EN_CODE, 5_9_3, 1_9_4_9, 1_1_5_7_8_1, 4, 7_1_5_8_6, 4_2_3_4, 6_0_6_3_3, 1_2_6_2_3_3, 4_3_2, 1_2_3_8_0_8, 1_5_5_9_2, 1_1_9_7, 1_1_7_1_3_2, 1_2_0_6_1_8, 5, 2]


				@classmethod
				def __UpperCamelCase						(							cls      :  Tuple       )		->     Union[str, Any]:
											A      						=    MaMaaaTokenizer.from_pretrained(
											    cls.checkpoint_name						, src_lang='en'						, tgt_lang='fr'       )
											A      						=    1
											return cls


				def __UpperCamelCase						(							self      :  Tuple       )		->     Tuple:
											self.assertEqual(self.tokenizer.get_lang_id('ar'       )						, 128_006       )
											self.assertEqual(self.tokenizer.get_lang_id('en'       )						, 128_022       )
											self.assertEqual(self.tokenizer.get_lang_id('ro'       )						, 128_076       )
											self.assertEqual(self.tokenizer.get_lang_id('mr'       )						, 128_063       )


				def __UpperCamelCase						(							self      :  List[str]       )		->     List[Any]:
											A      						=    self.tokenizer.get_vocab()
											self.assertEqual(len(__UpperCamelCase       )						, self.tokenizer.vocab_size       )
											self.assertEqual(vocab['<unk>']						, 3       )
											self.assertIn(self.tokenizer.get_lang_token('en'       )						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Any       )		->     Dict:
											A      						=    'en'
											A      						=    self.tokenizer.batch_encode_plus(self.src_text       ).input_ids[0]
											self.assertListEqual(self.expected_src_tokens						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  List[Any]       )		->     Dict:
											self.assertIn(__UpperCamelCase						, self.tokenizer.all_special_ids       )
											# fmt: off
											A      						=    [FR_CODE, 5_364, 82, 8_642, 4, 294, 47, 8, 14_028, 136, 3_286, 9_706, 6, 90_797, 6, 144_012, 162, 88_128, 30_061, 5, 2]
											# fmt: on
											A      						=    self.tokenizer.decode(__UpperCamelCase						, skip_special_tokens=__UpperCamelCase       )
											A      						=    self.tokenizer.decode(generated_ids[1:]						, skip_special_tokens=__UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertNotIn(self.tokenizer.eos_token						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  int       )		->     Any:
											A      						=    tempfile.mkdtemp()
											A      						=    self.tokenizer.lang_token_to_id
											self.tokenizer.save_pretrained(__UpperCamelCase       )
											A      						=    MaMaaaTokenizer.from_pretrained(__UpperCamelCase       )
											self.assertDictEqual(new_tok.lang_token_to_id						, __UpperCamelCase       )


				@require_torch
				def __UpperCamelCase						(							self      :  Any       )		->     str:
											A      						=    'en'
											A      						=    'fr'

											A      						=    self.tokenizer(self.src_text						, text_target=self.tgt_text						, padding=__UpperCamelCase						, return_tensors='pt'       )

											A      						=    shift_tokens_right(
											    batch['labels']						, self.tokenizer.pad_token_id						, self.tokenizer.eos_token_id       )

											for k in batch:
																		A      						=    batch[k].tolist()
											# batch = {k: v.tolist() for k,v in batch.items()}
											# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
											# batch.decoder_inputs_ids[0][0] ==
											assert batch.input_ids[1][0] == EN_CODE
											assert batch.input_ids[1][-1] == 2
											assert batch.labels[1][0] == FR_CODE
											assert batch.labels[1][-1] == 2
											assert batch.decoder_input_ids[1][:2] == [2, FR_CODE]


				@require_torch
				def __UpperCamelCase						(							self      :  Any       )		->     List[Any]:
											A      						=    'mr'
											self.assertListEqual(self.tokenizer.prefix_tokens						, [self.tokenizer.get_lang_id('mr'       )]       )
											self.assertListEqual(self.tokenizer.suffix_tokens						, [self.tokenizer.eos_token_id]       )

											A      						=    'zh'
											self.assertListEqual(self.tokenizer.prefix_tokens						, [self.tokenizer.get_lang_id('zh'       )]       )
											self.assertListEqual(self.tokenizer.suffix_tokens						, [self.tokenizer.eos_token_id]       )


				@require_torch
				def __UpperCamelCase						(							self      :  Optional[Any]       )		->     str:
											A      						=    'mr'
											self.tokenizer._switch_to_target_mode()
											self.assertListEqual(self.tokenizer.prefix_tokens						, [self.tokenizer.get_lang_id('mr'       )]       )
											self.assertListEqual(self.tokenizer.suffix_tokens						, [self.tokenizer.eos_token_id]       )
											self.tokenizer._switch_to_input_mode()
											self.assertListEqual(self.tokenizer.prefix_tokens						, [self.tokenizer.get_lang_id(self.tokenizer.src_lang       )]       )

											A      						=    'zh'
											self.tokenizer._switch_to_target_mode()
											self.assertListEqual(self.tokenizer.prefix_tokens						, [self.tokenizer.get_lang_id('zh'       )]       )
											self.assertListEqual(self.tokenizer.suffix_tokens						, [self.tokenizer.eos_token_id]       )
											self.tokenizer._switch_to_input_mode()
											self.assertListEqual(self.tokenizer.prefix_tokens						, [self.tokenizer.get_lang_id(self.tokenizer.src_lang       )]       )


				@require_torch
				def __UpperCamelCase						(							self      :  str       )		->     Optional[int]:
											A      						=    self.tokenizer._build_translation_inputs('A test'						, return_tensors='pt'						, src_lang='en'						, tgt_lang='ar'       )

											self.assertEqual(
											    nested_simplify(__UpperCamelCase       )						, {
											        # en_XX, A, test, EOS
											        'input_ids': [[128_022, 58, 4_183, 2]],
											        'attention_mask': [[1, 1, 1, 1]],
											        # ar_AR
											        'forced_bos_token_id': 128_006,
											    }						, )




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
				from test_module.custom_tokenization_fast import CustomTokenizerFast



class        lowerCAmelCase__					(  unittest.TestCase       ):


				def __UpperCamelCase						(							self      :  Any       )		->     Tuple:
											# A mock response for an HTTP head request to emulate server down
											A      						=    mock.Mock()
											A      						=    500
											A      						=    {}
											A      						=    HTTPError
											A      						=    {}

											# Download this model to make sure it's in the cache.
											A      						=    BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'       )

											# Under the mock environment we get a 500 error when trying to reach the tokenizer.
											with mock.patch('requests.Session.request'						, return_value=__UpperCamelCase       ) as mock_head:
																		A      						=    BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'       )
																		# This check we did call the fake head request
																		mock_head.assert_called()


				@require_tokenizers
				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     Union[str, Any]:
											# A mock response for an HTTP head request to emulate server down
											A      						=    mock.Mock()
											A      						=    500
											A      						=    {}
											A      						=    HTTPError
											A      						=    {}

											# Download this model to make sure it's in the cache.
											A      						=    GPTaTokenizerFast.from_pretrained('gpt2'       )

											# Under the mock environment we get a 500 error when trying to reach the tokenizer.
											with mock.patch('requests.Session.request'						, return_value=__UpperCamelCase       ) as mock_head:
																		A      						=    GPTaTokenizerFast.from_pretrained('gpt2'       )
																		# This check we did call the fake head request
																		mock_head.assert_called()


				def __UpperCamelCase						(							self      :  Optional[Any]       )		->     List[str]:
											# This test is for deprecated behavior and can be removed in v5
											try:
																		A      						=    tempfile.mktemp()
																		with open(__UpperCamelCase						, 'wb'       ) as f:
																									http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'						, __UpperCamelCase       )

																		A      						=    AlbertTokenizer.from_pretrained(__UpperCamelCase       )
											finally:
																		os.remove(__UpperCamelCase       )

											# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
											# the current folder and have the right name.
											if os.path.isfile('tokenizer.json'       ):
																		# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
																		return
											try:
																		with open('tokenizer.json'						, 'wb'       ) as f:
																									http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'						, __UpperCamelCase       )
																		A      						=    AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'       )
																		# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
																		self.assertEqual(tokenizer.vocab_size						, 1_000       )
																		# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

											finally:
																		os.remove('tokenizer.json'       )


				def __UpperCamelCase						(							self      :  str       )		->     int:
											# This test is for deprecated behavior and can be removed in v5
											A      						=    AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'       )





@is_staging_test
class        lowerCAmelCase__					(  unittest.TestCase       ):
				A_   :     str								=     ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']


				@classmethod
				def __UpperCamelCase						(							cls      :  Tuple       )		->     Dict:
											A      						=    TOKEN
											HfFolder.save_token(__UpperCamelCase       )


				@classmethod
				def __UpperCamelCase						(							cls      :  Any       )		->     Dict:
											try:
																		delete_repo(token=cls._token						, repo_id='test-tokenizer'       )
											except HTTPError:
																		pass

											try:
																		delete_repo(token=cls._token						, repo_id='valid_org/test-tokenizer-org'       )
											except HTTPError:
																		pass

											try:
																		delete_repo(token=cls._token						, repo_id='test-dynamic-tokenizer'       )
											except HTTPError:
																		pass


				def __UpperCamelCase						(							self      :  Optional[Any]       )		->     List[str]:
											with tempfile.TemporaryDirectory() as tmp_dir:
																		A      						=    os.path.join(__UpperCamelCase						, 'vocab.txt'       )
																		with open(__UpperCamelCase						, 'w'						, encoding='utf-8'       ) as vocab_writer:
																									vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]       )       )
																		A      						=    BertTokenizer(__UpperCamelCase       )

											tokenizer.push_to_hub('test-tokenizer'						, use_auth_token=self._token       )
											A      						=    BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''       )
											self.assertDictEqual(new_tokenizer.vocab						, tokenizer.vocab       )

											# Reset repo
											delete_repo(token=self._token						, repo_id='test-tokenizer'       )

											# Push to hub via save_pretrained
											with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(__UpperCamelCase						, repo_id='test-tokenizer'						, push_to_hub=__UpperCamelCase						, use_auth_token=self._token       )

											A      						=    BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''       )
											self.assertDictEqual(new_tokenizer.vocab						, tokenizer.vocab       )


				def __UpperCamelCase						(							self      :  List[str]       )		->     Tuple:
											with tempfile.TemporaryDirectory() as tmp_dir:
																		A      						=    os.path.join(__UpperCamelCase						, 'vocab.txt'       )
																		with open(__UpperCamelCase						, 'w'						, encoding='utf-8'       ) as vocab_writer:
																									vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]       )       )
																		A      						=    BertTokenizer(__UpperCamelCase       )

											tokenizer.push_to_hub('valid_org/test-tokenizer-org'						, use_auth_token=self._token       )
											A      						=    BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'       )
											self.assertDictEqual(new_tokenizer.vocab						, tokenizer.vocab       )

											# Reset repo
											delete_repo(token=self._token						, repo_id='valid_org/test-tokenizer-org'       )

											# Push to hub via save_pretrained
											with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(
																		    __UpperCamelCase						, repo_id='valid_org/test-tokenizer-org'						, push_to_hub=__UpperCamelCase						, use_auth_token=self._token       )

											A      						=    BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'       )
											self.assertDictEqual(new_tokenizer.vocab						, tokenizer.vocab       )


				@require_tokenizers
				def __UpperCamelCase						(							self      :  Any       )		->     Union[str, Any]:
											CustomTokenizer.register_for_auto_class()
											with tempfile.TemporaryDirectory() as tmp_dir:
																		A      						=    os.path.join(__UpperCamelCase						, 'vocab.txt'       )
																		with open(__UpperCamelCase						, 'w'						, encoding='utf-8'       ) as vocab_writer:
																									vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]       )       )
																		A      						=    CustomTokenizer(__UpperCamelCase       )

											# No fast custom tokenizer
											tokenizer.push_to_hub('test-dynamic-tokenizer'						, use_auth_token=self._token       )

											A      						=    AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''						, trust_remote_code=__UpperCamelCase       )
											# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
											self.assertEqual(tokenizer.__class__.__name__						, 'CustomTokenizer'       )

											# Fast and slow custom tokenizer
											CustomTokenizerFast.register_for_auto_class()
											with tempfile.TemporaryDirectory() as tmp_dir:
																		A      						=    os.path.join(__UpperCamelCase						, 'vocab.txt'       )
																		with open(__UpperCamelCase						, 'w'						, encoding='utf-8'       ) as vocab_writer:
																									vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]       )       )

																		A      						=    BertTokenizerFast.from_pretrained(__UpperCamelCase       )
																		bert_tokenizer.save_pretrained(__UpperCamelCase       )
																		A      						=    CustomTokenizerFast.from_pretrained(__UpperCamelCase       )

											tokenizer.push_to_hub('test-dynamic-tokenizer'						, use_auth_token=self._token       )

											A      						=    AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''						, trust_remote_code=__UpperCamelCase       )
											# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
											self.assertEqual(tokenizer.__class__.__name__						, 'CustomTokenizerFast'       )
											A      						=    AutoTokenizer.from_pretrained(
											    f'''{USER}/test-dynamic-tokenizer'''						, use_fast=__UpperCamelCase						, trust_remote_code=__UpperCamelCase       )
											# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
											self.assertEqual(tokenizer.__class__.__name__						, 'CustomTokenizer'       )





class        lowerCAmelCase__					(  unittest.TestCase       ):


				def __UpperCamelCase						(							self      :  List[Any]       )		->     Any:
											A      						=    Trie()
											trie.add('Hello 友達'       )
											self.assertEqual(trie.data						, {'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}       )
											trie.add('Hello'       )
											trie.data
											self.assertEqual(trie.data						, {'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}       )


				def __UpperCamelCase						(							self      :  Dict       )		->     Tuple:
											A      						=    Trie()
											self.assertEqual(trie.split('[CLS] This is a extra_id_100'       )						, ['[CLS] This is a extra_id_100']       )
											trie.add('[CLS]'       )
											trie.add('extra_id_1'       )
											trie.add('extra_id_100'       )
											self.assertEqual(trie.split('[CLS] This is a extra_id_100'       )						, ['[CLS]', ' This is a ', 'extra_id_100']       )


				def __UpperCamelCase						(							self      :  Any       )		->     Optional[int]:
											A      						=    Trie()
											trie.add('A'       )
											self.assertEqual(trie.split('ABC'       )						, ['A', 'BC']       )
											self.assertEqual(trie.split('BCA'       )						, ['BC', 'A']       )


				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     str:
											A      						=    Trie()
											trie.add('TOKEN]'       )
											trie.add('[SPECIAL_TOKEN]'       )
											self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'       )						, ['This is something ', '[SPECIAL_TOKEN]']       )


				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     Tuple:
											A      						=    Trie()
											trie.add('A'       )
											trie.add('P'       )
											trie.add('[SPECIAL_TOKEN]'       )
											self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'       )						, ['This is something ', '[SPECIAL_TOKEN]']       )


				def __UpperCamelCase						(							self      :  List[str]       )		->     Tuple:
											A      						=    Trie()
											trie.add('AB'       )
											trie.add('B'       )
											trie.add('C'       )
											self.assertEqual(trie.split('ABC'       )						, ['AB', 'C']       )


				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     List[Any]:
											A      						=    Trie()
											trie.add('ABC'       )
											trie.add('B'       )
											trie.add('CD'       )
											self.assertEqual(trie.split('ABCD'       )						, ['ABC', 'D']       )


				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     List[Any]:
											# Even if the offsets are wrong, we necessarily output correct string
											# parts.
											A      						=    Trie()
											A      						=    trie.cut_text('ABC'						, [0, 0, 2, 1, 2, 3]       )
											self.assertEqual(__UpperCamelCase						, ['AB', 'C']       )