
import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_A     =  get_tests_dir("fixtures/test_sentencepiece.model")





@require_sentencepiece
@require_tokenizers
class 			_lowerCAmelCase      (			__a      ,   unittest.TestCase ):
				_lowercase    	=XLNetTokenizer
				_lowercase    	=XLNetTokenizerFast
				_lowercase    	=True
				_lowercase    	=True




				def   __a		(      self  )       ->   Optional[int]:
							super().setUp()

							# We have a SentencePiece fixture for testing
							lowerCAmelCase_      			=			XLNetTokenizer(_UpperCamelCase       ,       keep_accents=_UpperCamelCase  )
							tokenizer.sanitize_special_tokens()
							tokenizer.save_pretrained(self.tmpdirname  )




				def   __a		(      self  )       ->   List[Any]:
							lowerCAmelCase_      			=			"<s>"
							lowerCAmelCase_      			=			1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCamelCase  )       ,       _UpperCamelCase  )
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCamelCase  )       ,       _UpperCamelCase  )




				def   __a		(      self  )       ->   Any:
							lowerCAmelCase_      			=			list(self.get_tokenizer().get_vocab().keys()  )

							self.assertEqual(vocab_keys[0]       ,       "<unk>"  )
							self.assertEqual(vocab_keys[1]       ,       "<s>"  )
							self.assertEqual(vocab_keys[-1]       ,       "<eod>"  )
							self.assertEqual(len(_UpperCamelCase  )       ,       1_006  )




				def   __a		(      self  )       ->   List[str]:
							self.assertEqual(self.get_tokenizer().vocab_size       ,       1_000  )




				def   __a		(      self  )       ->   Any:
							lowerCAmelCase_      			=			XLNetTokenizer(_UpperCamelCase       ,       keep_accents=_UpperCamelCase  )

							lowerCAmelCase_      			=			tokenizer.tokenize("This is a test"  )
							self.assertListEqual(_UpperCamelCase       ,       ["▁This", "▁is", "▁a", "▁t", "est"]  )

							self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase  )       ,       [285, 46, 10, 170, 382]  )

							lowerCAmelCase_      			=			tokenizer.tokenize("I was born in 92000, and this is falsé."  )
							self.assertListEqual(
							    _UpperCamelCase       ,       [
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "9",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "s",
							        "é",
							        ".",
							    ]       ,       )
							lowerCAmelCase_      			=			tokenizer.convert_tokens_to_ids(_UpperCamelCase  )
							self.assertListEqual(_UpperCamelCase       ,       [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]  )

							lowerCAmelCase_      			=			tokenizer.convert_ids_to_tokens(_UpperCamelCase  )
							self.assertListEqual(
							    _UpperCamelCase       ,       [
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "<unk>",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "s",
							        "<unk>",
							        ".",
							    ]       ,       )




				def   __a		(      self  )       ->   Optional[Any]:
							lowerCAmelCase_      			=			XLNetTokenizer(_UpperCamelCase       ,       do_lower_case=_UpperCamelCase  )
							lowerCAmelCase_      			=			tokenizer.tokenize("I was born in 92000, and this is falsé."  )
							self.assertListEqual(
							    _UpperCamelCase       ,       [
							        SPIECE_UNDERLINE + "",
							        "i",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "9",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "se",
							        ".",
							    ]       ,       )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"  )       ,       ["▁he", "ll", "o"]  )




				def   __a		(      self  )       ->   Tuple:
							lowerCAmelCase_      			=			XLNetTokenizer(_UpperCamelCase       ,       do_lower_case=_UpperCamelCase  )
							lowerCAmelCase_      			=			tokenizer.tokenize("I was born in 92000, and this is falsé."  )
							self.assertListEqual(
							    _UpperCamelCase       ,       [
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "9",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "se",
							        ".",
							    ]       ,       )




				@slow
				def   __a		(      self  )       ->   str:
							lowerCAmelCase_      			=			XLNetTokenizer.from_pretrained("xlnet-base-cased"  )

							lowerCAmelCase_      			=			tokenizer.encode("sequence builders"       ,       add_special_tokens=_UpperCamelCase  )
							lowerCAmelCase_      			=			tokenizer.encode("multi-sequence build"       ,       add_special_tokens=_UpperCamelCase  )

							lowerCAmelCase_      			=			tokenizer.build_inputs_with_special_tokens(_UpperCamelCase  )
							lowerCAmelCase_      			=			tokenizer.build_inputs_with_special_tokens(_UpperCamelCase       ,       _UpperCamelCase  )

							assert encoded_sentence == text + [4, 3]
							assert encoded_pair == text + [4] + text_a + [4, 3]




				@slow
				def   __a		(      self  )       ->   Union[str, Any]:
							# fmt: off
							lowerCAmelCase_      			=			{"input_ids": [[17, 21_442, 270, 17, 10, 14_645, 318, 34, 17, 4_546, 3_145, 787, 13, 7_752, 22_018, 23, 21, 17, 4_546, 3_145, 787, 13, 3_352, 14_431, 13, 5_500, 11, 1_176, 580, 13, 16_819, 4_797, 23, 17, 10, 17_135, 658, 19, 457, 7_932, 13, 184, 19, 3_154, 17_135, 6_468, 19, 1_404, 12_269, 19, 4_229, 5_356, 16_264, 46, 19, 17, 20_545, 10_395, 9, 9, 9, 11, 28, 6_421, 9_531, 20_729, 17, 10, 353, 17_022, 11, 21, 6_421, 9_531, 16_949, 17, 10, 11_509, 753, 11, 33, 95, 2_421, 7_385, 956, 14_431, 2_626, 25, 842, 7_385, 4_836, 21, 1_429, 2_272, 9_855, 3_120, 161, 24_738, 19, 13_203, 658, 218, 787, 21, 430, 18_482, 847, 2_637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22_178, 27, 1_064, 22, 956, 13, 11_101, 1_429, 5_854, 24_313, 18_953, 40, 422, 24_366, 68, 1_758, 37, 10_483, 14_257, 31, 207, 263, 21, 203, 3_773, 25, 71, 9_735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2_049, 3_442, 17, 13_894, 3_380, 23, 95, 18, 17_634, 2_288, 9, 4, 3]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=_UpperCamelCase       ,       model_name="xlnet-base-cased"       ,       revision="c841166438c31ec7ca9a106dee7bb312b73ae511"       ,       )





import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
			from test_module.custom_tokenization_fast import CustomTokenizerFast





class 			_lowerCAmelCase      (			unittest.TestCase ):




				def   __a		(      self  )       ->   str:
							# A mock response for an HTTP head request to emulate server down
							lowerCAmelCase_      			=			mock.Mock()
							lowerCAmelCase_      			=			500
							lowerCAmelCase_      			=			{}
							lowerCAmelCase_      			=			HTTPError
							lowerCAmelCase_      			=			{}

							# Download this model to make sure it's in the cache.
							lowerCAmelCase_      			=			BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"  )

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch("requests.Session.request"       ,       return_value=_UpperCamelCase  ) as mock_head:
										lowerCAmelCase_      			=			BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"  )
										# This check we did call the fake head request
										mock_head.assert_called()




				@require_tokenizers
				def   __a		(      self  )       ->   int:
							# A mock response for an HTTP head request to emulate server down
							lowerCAmelCase_      			=			mock.Mock()
							lowerCAmelCase_      			=			500
							lowerCAmelCase_      			=			{}
							lowerCAmelCase_      			=			HTTPError
							lowerCAmelCase_      			=			{}

							# Download this model to make sure it's in the cache.
							lowerCAmelCase_      			=			GPTaTokenizerFast.from_pretrained("gpt2"  )

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch("requests.Session.request"       ,       return_value=_UpperCamelCase  ) as mock_head:
										lowerCAmelCase_      			=			GPTaTokenizerFast.from_pretrained("gpt2"  )
										# This check we did call the fake head request
										mock_head.assert_called()




				def   __a		(      self  )       ->   List[Any]:
							# This test is for deprecated behavior and can be removed in v5
							try:
										lowerCAmelCase_      			=			tempfile.mktemp()
										with open(_UpperCamelCase       ,       "wb"  ) as f:
													http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"       ,       _UpperCamelCase  )

										lowerCAmelCase_      			=			AlbertTokenizer.from_pretrained(_UpperCamelCase  )
							finally:
										os.remove(_UpperCamelCase  )

							# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
							# the current folder and have the right name.
							if os.path.isfile("tokenizer.json"  ):
										# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
										return
							try:
										with open("tokenizer.json"       ,       "wb"  ) as f:
													http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"       ,       _UpperCamelCase  )
										lowerCAmelCase_      			=			AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"  )
										# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
										self.assertEqual(tokenizer.vocab_size       ,       1_000  )
										# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

							finally:
										os.remove("tokenizer.json"  )




				def   __a		(      self  )       ->   Union[str, Any]:
							# This test is for deprecated behavior and can be removed in v5
							lowerCAmelCase_      			=			AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"  )





@is_staging_test
class 			_lowerCAmelCase      (			unittest.TestCase ):
				_lowercase    	=['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']




				@classmethod
				def   __a		(      cls  )       ->   Tuple:
							lowerCAmelCase_      			=			TOKEN
							HfFolder.save_token(_UpperCamelCase  )




				@classmethod
				def   __a		(      cls  )       ->   Tuple:
							try:
										delete_repo(token=cls._token       ,       repo_id="test-tokenizer"  )
							except HTTPError:
										pass

							try:
										delete_repo(token=cls._token       ,       repo_id="valid_org/test-tokenizer-org"  )
							except HTTPError:
										pass

							try:
										delete_repo(token=cls._token       ,       repo_id="test-dynamic-tokenizer"  )
							except HTTPError:
										pass




				def   __a		(      self  )       ->   int:
							with tempfile.TemporaryDirectory() as tmp_dir:
										lowerCAmelCase_      			=			os.path.join(_UpperCamelCase       ,       "vocab.txt"  )
										with open(_UpperCamelCase       ,       "w"       ,       encoding="utf-8"  ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]  )  )
										lowerCAmelCase_      			=			BertTokenizer(_UpperCamelCase  )

							tokenizer.push_to_hub("test-tokenizer"       ,       use_auth_token=self._token  )
							lowerCAmelCase_      			=			BertTokenizer.from_pretrained(f"""{USER}/test-tokenizer"""  )
							self.assertDictEqual(new_tokenizer.vocab       ,       tokenizer.vocab  )

							# Reset repo
							delete_repo(token=self._token       ,       repo_id="test-tokenizer"  )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(_UpperCamelCase       ,       repo_id="test-tokenizer"       ,       push_to_hub=_UpperCamelCase       ,       use_auth_token=self._token  )

							lowerCAmelCase_      			=			BertTokenizer.from_pretrained(f"""{USER}/test-tokenizer"""  )
							self.assertDictEqual(new_tokenizer.vocab       ,       tokenizer.vocab  )




				def   __a		(      self  )       ->   Optional[int]:
							with tempfile.TemporaryDirectory() as tmp_dir:
										lowerCAmelCase_      			=			os.path.join(_UpperCamelCase       ,       "vocab.txt"  )
										with open(_UpperCamelCase       ,       "w"       ,       encoding="utf-8"  ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]  )  )
										lowerCAmelCase_      			=			BertTokenizer(_UpperCamelCase  )

							tokenizer.push_to_hub("valid_org/test-tokenizer-org"       ,       use_auth_token=self._token  )
							lowerCAmelCase_      			=			BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"  )
							self.assertDictEqual(new_tokenizer.vocab       ,       tokenizer.vocab  )

							# Reset repo
							delete_repo(token=self._token       ,       repo_id="valid_org/test-tokenizer-org"  )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(
										    _UpperCamelCase       ,       repo_id="valid_org/test-tokenizer-org"       ,       push_to_hub=_UpperCamelCase       ,       use_auth_token=self._token  )

							lowerCAmelCase_      			=			BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"  )
							self.assertDictEqual(new_tokenizer.vocab       ,       tokenizer.vocab  )




				@require_tokenizers
				def   __a		(      self  )       ->   Optional[Any]:
							CustomTokenizer.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
										lowerCAmelCase_      			=			os.path.join(_UpperCamelCase       ,       "vocab.txt"  )
										with open(_UpperCamelCase       ,       "w"       ,       encoding="utf-8"  ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]  )  )
										lowerCAmelCase_      			=			CustomTokenizer(_UpperCamelCase  )

							# No fast custom tokenizer
							tokenizer.push_to_hub("test-dynamic-tokenizer"       ,       use_auth_token=self._token  )

							lowerCAmelCase_      			=			AutoTokenizer.from_pretrained(f"""{USER}/test-dynamic-tokenizer"""       ,       trust_remote_code=_UpperCamelCase  )
							# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__       ,       "CustomTokenizer"  )

							# Fast and slow custom tokenizer
							CustomTokenizerFast.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
										lowerCAmelCase_      			=			os.path.join(_UpperCamelCase       ,       "vocab.txt"  )
										with open(_UpperCamelCase       ,       "w"       ,       encoding="utf-8"  ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]  )  )

										lowerCAmelCase_      			=			BertTokenizerFast.from_pretrained(_UpperCamelCase  )
										bert_tokenizer.save_pretrained(_UpperCamelCase  )
										lowerCAmelCase_      			=			CustomTokenizerFast.from_pretrained(_UpperCamelCase  )

							tokenizer.push_to_hub("test-dynamic-tokenizer"       ,       use_auth_token=self._token  )

							lowerCAmelCase_      			=			AutoTokenizer.from_pretrained(f"""{USER}/test-dynamic-tokenizer"""       ,       trust_remote_code=_UpperCamelCase  )
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__       ,       "CustomTokenizerFast"  )
							lowerCAmelCase_      			=			AutoTokenizer.from_pretrained(
							    f"""{USER}/test-dynamic-tokenizer"""       ,       use_fast=_UpperCamelCase       ,       trust_remote_code=_UpperCamelCase  )
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__       ,       "CustomTokenizer"  )





class 			_lowerCAmelCase      (			unittest.TestCase ):




				def   __a		(      self  )       ->   Any:
							lowerCAmelCase_      			=			Trie()
							trie.add("Hello 友達"  )
							self.assertEqual(trie.data       ,       {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}  )
							trie.add("Hello"  )
							trie.data
							self.assertEqual(trie.data       ,       {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}  )




				def   __a		(      self  )       ->   Optional[int]:
							lowerCAmelCase_      			=			Trie()
							self.assertEqual(trie.split("[CLS] This is a extra_id_100"  )       ,       ["[CLS] This is a extra_id_100"]  )
							trie.add("[CLS]"  )
							trie.add("extra_id_1"  )
							trie.add("extra_id_100"  )
							self.assertEqual(trie.split("[CLS] This is a extra_id_100"  )       ,       ["[CLS]", " This is a ", "extra_id_100"]  )




				def   __a		(      self  )       ->   str:
							lowerCAmelCase_      			=			Trie()
							trie.add("A"  )
							self.assertEqual(trie.split("ABC"  )       ,       ["A", "BC"]  )
							self.assertEqual(trie.split("BCA"  )       ,       ["BC", "A"]  )




				def   __a		(      self  )       ->   int:
							lowerCAmelCase_      			=			Trie()
							trie.add("TOKEN]"  )
							trie.add("[SPECIAL_TOKEN]"  )
							self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"  )       ,       ["This is something ", "[SPECIAL_TOKEN]"]  )




				def   __a		(      self  )       ->   int:
							lowerCAmelCase_      			=			Trie()
							trie.add("A"  )
							trie.add("P"  )
							trie.add("[SPECIAL_TOKEN]"  )
							self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"  )       ,       ["This is something ", "[SPECIAL_TOKEN]"]  )




				def   __a		(      self  )       ->   str:
							lowerCAmelCase_      			=			Trie()
							trie.add("AB"  )
							trie.add("B"  )
							trie.add("C"  )
							self.assertEqual(trie.split("ABC"  )       ,       ["AB", "C"]  )




				def   __a		(      self  )       ->   List[Any]:
							lowerCAmelCase_      			=			Trie()
							trie.add("ABC"  )
							trie.add("B"  )
							trie.add("CD"  )
							self.assertEqual(trie.split("ABCD"  )       ,       ["ABC", "D"]  )




				def   __a		(      self  )       ->   List[Any]:
							# Even if the offsets are wrong, we necessarily output correct string
							# parts.
							lowerCAmelCase_      			=			Trie()
							lowerCAmelCase_      			=			trie.cut_text("ABC"       ,       [0, 0, 2, 1, 2, 3]  )
							self.assertEqual(_UpperCamelCase       ,       ["AB", "C"]  )



