






import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class     _lowerCAmelCase(	UpperCAmelCase_							,     unittest.TestCase ):





       """simple docstring"""







       a    :		List[Any]									=RoCBertTokenizer
       a    :		Any									=None
       a    :		List[Any]									=False
       a    :		int									=True
       a    :		List[Any]									=filter_non_english





       def       _a   ( self							):
         super().setUp()

         UpperCamelCase_: Dict   				=    ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', '你', '好', '是', '谁', 'a', 'b', 'c', 'd']
         UpperCamelCase_: str   				=    {}
         UpperCamelCase_: List[str]   				=    {}
         for i, value in enumerate(_lowerCamelCase							):
           UpperCamelCase_: str   				=    i
           UpperCamelCase_: Tuple   				=    i
         UpperCamelCase_: Optional[int]   				=    os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES['vocab_file']							)
         UpperCamelCase_: Tuple   				=    os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES['word_shape_file']							)
         UpperCamelCase_: Any   				=    os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES['word_pronunciation_file']							)
         with open(self.vocab_file  ,      'w'  ,      encoding='utf-8'							) as vocab_writer:
           vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]							)							)
         with open(self.word_shape_file  ,      'w'  ,      encoding='utf-8'							) as word_shape_writer:
           json.dump(_lowerCamelCase  ,      _lowerCamelCase  ,      ensure_ascii=_lowerCamelCase							)
         with open(self.word_pronunciation_file  ,      'w'  ,      encoding='utf-8'							) as word_pronunciation_writer:
           json.dump(_lowerCamelCase  ,      _lowerCamelCase  ,      ensure_ascii=_lowerCamelCase							)





       def       _a   ( self							):
         UpperCamelCase_: Optional[int]   				=    self.tokenizer_class(self.vocab_file  ,      self.word_shape_file  ,      self.word_pronunciation_file							)

         UpperCamelCase_: Tuple   				=    tokenizer.tokenize('你好[SEP]你是谁'							)
         self.assertListEqual(_lowerCamelCase  ,      ['你', '好', '[SEP]', '你', '是', '谁']							)
         self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase							)  ,      [5, 6, 2, 5, 7, 8]							)
         self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(_lowerCamelCase							)  ,      [5, 6, 2, 5, 7, 8]							)
         self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(_lowerCamelCase							)  ,      [5, 6, 2, 5, 7, 8]							)





       def       _a   ( self							):
         UpperCamelCase_: Optional[int]   				=    RoCBertBasicTokenizer()

         self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'							)  ,      ['ah', '\u535A', '\u63A8', 'zz']							)





       def       _a   ( self							):
         UpperCamelCase_: Union[str, Any]   				=    RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase							)

         self.assertListEqual(
             tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '							)  ,      ['hello', '!', 'how', 'are', 'you', '?']							)
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)  ,      ['hello']							)





       def       _a   ( self							):
         UpperCamelCase_: List[Any]   				=    RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase  ,      strip_accents=_lowerCamelCase							)

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)  ,      ['hällo', '!', 'how', 'are', 'you', '?']							)
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)  ,      ['h\u00E9llo']							)





       def       _a   ( self							):
         UpperCamelCase_: str   				=    RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase  ,      strip_accents=_lowerCamelCase							)

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)  ,      ['hallo', '!', 'how', 'are', 'you', '?']							)
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)  ,      ['hello']							)





       def       _a   ( self							):
         UpperCamelCase_: str   				=    RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase							)

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)  ,      ['hallo', '!', 'how', 'are', 'you', '?']							)
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)  ,      ['hello']							)





       def       _a   ( self							):
         UpperCamelCase_: Optional[Any]   				=    RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase							)

         self.assertListEqual(
             tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '							)  ,      ['HeLLo', '!', 'how', 'Are', 'yoU', '?']							)





       def       _a   ( self							):
         UpperCamelCase_: Optional[Any]   				=    RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase  ,      strip_accents=_lowerCamelCase							)

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)  ,      ['HäLLo', '!', 'how', 'Are', 'yoU', '?']							)





       def       _a   ( self							):
         UpperCamelCase_: Union[str, Any]   				=    RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase  ,      strip_accents=_lowerCamelCase							)

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)  ,      ['HaLLo', '!', 'how', 'Are', 'yoU', '?']							)





       def       _a   ( self							):
         UpperCamelCase_: List[str]   				=    RoCBertBasicTokenizer(do_lower_case=_lowerCamelCase  ,      never_split=['[UNK]']							)

         self.assertListEqual(
             tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'							)  ,      ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']							)





       def       _a   ( self							):
         UpperCamelCase_: Any   				=    ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

         UpperCamelCase_: Union[str, Any]   				=    {}
         for i, token in enumerate(_lowerCamelCase							):
           UpperCamelCase_: str   				=    i
         UpperCamelCase_: Optional[int]   				=    RoCBertWordpieceTokenizer(vocab=_lowerCamelCase  ,      unk_token='[UNK]'							)

         self.assertListEqual(tokenizer.tokenize(''							)  ,      []							)

         self.assertListEqual(tokenizer.tokenize('unwanted running'							)  ,      ['un', '##want', '##ed', 'runn', '##ing']							)

         self.assertListEqual(tokenizer.tokenize('unwantedX running'							)  ,      ['[UNK]', 'runn', '##ing']							)





       def       _a   ( self							):
         self.assertTrue(_is_whitespace(' '							)							)
         self.assertTrue(_is_whitespace('\t'							)							)
         self.assertTrue(_is_whitespace('\r'							)							)
         self.assertTrue(_is_whitespace('\n'							)							)
         self.assertTrue(_is_whitespace('\u00A0'							)							)

         self.assertFalse(_is_whitespace('A'							)							)
         self.assertFalse(_is_whitespace('-'							)							)





       def       _a   ( self							):
         self.assertTrue(_is_control('\u0005'							)							)

         self.assertFalse(_is_control('A'							)							)
         self.assertFalse(_is_control(' '							)							)
         self.assertFalse(_is_control('\t'							)							)
         self.assertFalse(_is_control('\r'							)							)





       def       _a   ( self							):
         self.assertTrue(_is_punctuation('-'							)							)
         self.assertTrue(_is_punctuation('$'							)							)
         self.assertTrue(_is_punctuation('`'							)							)
         self.assertTrue(_is_punctuation('.'							)							)

         self.assertFalse(_is_punctuation('A'							)							)
         self.assertFalse(_is_punctuation(' '							)							)





       def       _a   ( self							):
         UpperCamelCase_: str   				=    self.get_tokenizer()

         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(_lowerCamelCase							) for t in ['Test', '\xad', 'test']]  ,      [['[UNK]'], [], ['[UNK]']]							)

         if self.test_rust_tokenizer:
           UpperCamelCase_: Any   				=    self.get_rust_tokenizer()
           self.assertListEqual(
               [rust_tokenizer.tokenize(_lowerCamelCase							) for t in ['Test', '\xad', 'test']]  ,      [['[UNK]'], [], ['[UNK]']]							)





       def       _a   ( self							):
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
             UpperCamelCase_: Optional[int]   				=    self.rust_tokenizer_class.from_pretrained(_lowerCamelCase  ,      **_lowerCamelCase							)

             UpperCamelCase_: int   				=    f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
             UpperCamelCase_: Optional[int]   				=    tokenizer_r.encode_plus(
                 _lowerCamelCase  ,      return_attention_mask=_lowerCamelCase  ,      return_token_type_ids=_lowerCamelCase  ,      return_offsets_mapping=_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase  ,      )

             UpperCamelCase_: Optional[int]   				=    tokenizer_r.do_lower_case if hasattr(_lowerCamelCase  ,      'do_lower_case'							) else False
             UpperCamelCase_: Tuple   				=    (
                 [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), 'A'),
                     ((1, 2), ','),
                     ((3, 5), 'na'),
                     ((5, 6), '##ï'),
                     ((6, 8), '##ve'),
                     ((9, 1_5), tokenizer_r.mask_token),
                     ((1_6, 2_1), 'Allen'),
                     ((2_1, 2_3), '##NL'),
                     ((2_3, 2_4), '##P'),
                     ((2_5, 3_3), 'sentence'),
                     ((3_3, 3_4), '.'),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
                 if not do_lower_case
                 else [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), 'a'),
                     ((1, 2), ','),
                     ((3, 8), 'naive'),
                     ((9, 1_5), tokenizer_r.mask_token),
                     ((1_6, 2_1), 'allen'),
                     ((2_1, 2_3), '##nl'),
                     ((2_3, 2_4), '##p'),
                     ((2_5, 3_3), 'sentence'),
                     ((3_3, 3_4), '.'),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
             )

             self.assertEqual(
                 [e[1] for e in expected_results]  ,      tokenizer_r.convert_ids_to_tokens(tokens['input_ids']							)							)
             self.assertEqual([e[0] for e in expected_results]  ,      tokens['offset_mapping']							)





       def       _a   ( self							):
         UpperCamelCase_: Any   				=    ['的', '人', '有']
         UpperCamelCase_: List[str]   				=    ''.join(_lowerCamelCase							)
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
             UpperCamelCase_: Dict   				=    True
             UpperCamelCase_: List[Any]   				=    self.tokenizer_class.from_pretrained(_lowerCamelCase  ,      **_lowerCamelCase							)
             UpperCamelCase_: Dict   				=    self.rust_tokenizer_class.from_pretrained(_lowerCamelCase  ,      **_lowerCamelCase							)

             UpperCamelCase_: int   				=    tokenizer_p.encode(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             UpperCamelCase_: Dict   				=    tokenizer_r.encode(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)

             UpperCamelCase_: Optional[int]   				=    tokenizer_r.convert_ids_to_tokens(_lowerCamelCase							)
             UpperCamelCase_: Any   				=    tokenizer_p.convert_ids_to_tokens(_lowerCamelCase							)

             # it is expected that each Chinese character is not preceded by "##"
             self.assertListEqual(_lowerCamelCase  ,      _lowerCamelCase							)
             self.assertListEqual(_lowerCamelCase  ,      _lowerCamelCase							)

             UpperCamelCase_: Optional[int]   				=    False
             UpperCamelCase_: Union[str, Any]   				=    self.rust_tokenizer_class.from_pretrained(_lowerCamelCase  ,      **_lowerCamelCase							)
             UpperCamelCase_: Any   				=    self.tokenizer_class.from_pretrained(_lowerCamelCase  ,      **_lowerCamelCase							)

             UpperCamelCase_: Optional[int]   				=    tokenizer_r.encode(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)
             UpperCamelCase_: Optional[int]   				=    tokenizer_p.encode(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)

             UpperCamelCase_: Optional[int]   				=    tokenizer_r.convert_ids_to_tokens(_lowerCamelCase							)
             UpperCamelCase_: List[str]   				=    tokenizer_p.convert_ids_to_tokens(_lowerCamelCase							)

             # it is expected that only the first Chinese character is not preceded by "##".
             UpperCamelCase_: str   				=    [
                 f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_lowerCamelCase							)
             ]
             self.assertListEqual(_lowerCamelCase  ,      _lowerCamelCase							)
             self.assertListEqual(_lowerCamelCase  ,      _lowerCamelCase							)





       @slow
       def       _a   ( self							):
         UpperCamelCase_: List[str]   				=    self.tokenizer_class(self.vocab_file  ,      self.word_shape_file  ,      self.word_pronunciation_file							)

         UpperCamelCase_: Optional[Any]   				=    tokenizer.encode('你好'  ,      add_special_tokens=_lowerCamelCase							)
         UpperCamelCase_: Union[str, Any]   				=    tokenizer.encode('你是谁'  ,      add_special_tokens=_lowerCamelCase							)

         UpperCamelCase_: List[str]   				=    tokenizer.build_inputs_with_special_tokens(_lowerCamelCase							)
         UpperCamelCase_: Dict   				=    tokenizer.build_inputs_with_special_tokens(_lowerCamelCase  ,      _lowerCamelCase							)

         assert encoded_sentence == [1] + text + [2]
         assert encoded_pair == [1] + text + [2] + text_a + [2]





       def       _a   ( self							):
         UpperCamelCase_: Any   				=    self.get_tokenizers(do_lower_case=_lowerCamelCase							)
         for tokenizer in tokenizers:
           with self.subTest(f'''{tokenizer.__class__.__name__}'''							):
             UpperCamelCase_: Union[str, Any]   				=    '你好，你是谁'
             UpperCamelCase_: List[Any]   				=    tokenizer.tokenize(_lowerCamelCase							)
             UpperCamelCase_: Optional[int]   				=    tokenizer.convert_tokens_to_ids(_lowerCamelCase							)
             UpperCamelCase_: List[Any]   				=    tokenizer.convert_tokens_to_shape_ids(_lowerCamelCase							)
             UpperCamelCase_: Union[str, Any]   				=    tokenizer.convert_tokens_to_pronunciation_ids(_lowerCamelCase							)
             UpperCamelCase_: Any   				=    tokenizer.prepare_for_model(
                 _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)

             UpperCamelCase_: Tuple   				=    tokenizer.encode_plus(_lowerCamelCase  ,      add_special_tokens=_lowerCamelCase							)

             self.assertEqual(_lowerCamelCase  ,      _lowerCamelCase							)







import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
   from test_module.custom_tokenization_fast import CustomTokenizerFast




class     _lowerCAmelCase(	unittest.TestCase ):





       """simple docstring"""







       def       _a   ( self							):
         # A mock response for an HTTP head request to emulate server down
         UpperCamelCase_: Any   				=    mock.Mock()
         UpperCamelCase_: Dict   				=    5_0_0
         UpperCamelCase_: Any   				=    {}
         UpperCamelCase_: Tuple   				=    HTTPError
         UpperCamelCase_: List[str]   				=    {}

         # Download this model to make sure it's in the cache.
         UpperCamelCase_: int   				=    BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'							)

         # Under the mock environment we get a 500 error when trying to reach the tokenizer.
         with mock.patch('requests.Session.request'  ,      return_value=_lowerCamelCase							) as mock_head:
           UpperCamelCase_: Optional[int]   				=    BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'							)
           # This check we did call the fake head request
           mock_head.assert_called()





       @require_tokenizers
       def       _a   ( self							):
         # A mock response for an HTTP head request to emulate server down
         UpperCamelCase_: Union[str, Any]   				=    mock.Mock()
         UpperCamelCase_: Union[str, Any]   				=    5_0_0
         UpperCamelCase_: str   				=    {}
         UpperCamelCase_: List[str]   				=    HTTPError
         UpperCamelCase_: Optional[int]   				=    {}

         # Download this model to make sure it's in the cache.
         UpperCamelCase_: List[str]   				=    GPTaTokenizerFast.from_pretrained('gpt2'							)

         # Under the mock environment we get a 500 error when trying to reach the tokenizer.
         with mock.patch('requests.Session.request'  ,      return_value=_lowerCamelCase							) as mock_head:
           UpperCamelCase_: str   				=    GPTaTokenizerFast.from_pretrained('gpt2'							)
           # This check we did call the fake head request
           mock_head.assert_called()





       def       _a   ( self							):
         # This test is for deprecated behavior and can be removed in v5
         try:
           UpperCamelCase_: Optional[int]   				=    tempfile.mktemp()
           with open(_lowerCamelCase  ,      'wb'							) as f:
             http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'  ,      _lowerCamelCase							)

           UpperCamelCase_: Tuple   				=    AlbertTokenizer.from_pretrained(_lowerCamelCase							)
         finally:
           os.remove(_lowerCamelCase							)

         # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
         # the current folder and have the right name.
         if os.path.isfile('tokenizer.json'							):
           # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
           return
         try:
           with open('tokenizer.json'  ,      'wb'							) as f:
             http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'  ,      _lowerCamelCase							)
           UpperCamelCase_: List[str]   				=    AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'							)
           # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
           self.assertEqual(tokenizer.vocab_size  ,      1_0_0_0							)
           # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

         finally:
           os.remove('tokenizer.json'							)





       def       _a   ( self							):
         # This test is for deprecated behavior and can be removed in v5
         UpperCamelCase_: Any   				=    AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'							)







@is_staging_test
class     _lowerCAmelCase(	unittest.TestCase ):





       """simple docstring"""







       a    :		Dict									=['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']





       @classmethod
       def       _a   ( cls							):
         UpperCamelCase_: Optional[int]   				=    TOKEN
         HfFolder.save_token(_lowerCamelCase							)





       @classmethod
       def       _a   ( cls							):
         try:
           delete_repo(token=cls._token  ,      repo_id='test-tokenizer'							)
         except HTTPError:
           pass

         try:
           delete_repo(token=cls._token  ,      repo_id='valid_org/test-tokenizer-org'							)
         except HTTPError:
           pass

         try:
           delete_repo(token=cls._token  ,      repo_id='test-dynamic-tokenizer'							)
         except HTTPError:
           pass





       def       _a   ( self							):
         with tempfile.TemporaryDirectory() as tmp_dir:
           UpperCamelCase_: List[Any]   				=    os.path.join(_lowerCamelCase  ,      'vocab.txt'							)
           with open(_lowerCamelCase  ,      'w'  ,      encoding='utf-8'							) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]							)							)
           UpperCamelCase_: int   				=    BertTokenizer(_lowerCamelCase							)

         tokenizer.push_to_hub('test-tokenizer'  ,      use_auth_token=self._token							)
         UpperCamelCase_: Union[str, Any]   				=    BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''							)
         self.assertDictEqual(new_tokenizer.vocab  ,      tokenizer.vocab							)

         # Reset repo
         delete_repo(token=self._token  ,      repo_id='test-tokenizer'							)

         # Push to hub via save_pretrained
         with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(_lowerCamelCase  ,      repo_id='test-tokenizer'  ,      push_to_hub=_lowerCamelCase  ,      use_auth_token=self._token							)

         UpperCamelCase_: List[str]   				=    BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''							)
         self.assertDictEqual(new_tokenizer.vocab  ,      tokenizer.vocab							)





       def       _a   ( self							):
         with tempfile.TemporaryDirectory() as tmp_dir:
           UpperCamelCase_: Optional[Any]   				=    os.path.join(_lowerCamelCase  ,      'vocab.txt'							)
           with open(_lowerCamelCase  ,      'w'  ,      encoding='utf-8'							) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]							)							)
           UpperCamelCase_: Union[str, Any]   				=    BertTokenizer(_lowerCamelCase							)

         tokenizer.push_to_hub('valid_org/test-tokenizer-org'  ,      use_auth_token=self._token							)
         UpperCamelCase_: Dict   				=    BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'							)
         self.assertDictEqual(new_tokenizer.vocab  ,      tokenizer.vocab							)

         # Reset repo
         delete_repo(token=self._token  ,      repo_id='valid_org/test-tokenizer-org'							)

         # Push to hub via save_pretrained
         with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(
               _lowerCamelCase  ,      repo_id='valid_org/test-tokenizer-org'  ,      push_to_hub=_lowerCamelCase  ,      use_auth_token=self._token							)

         UpperCamelCase_: Optional[Any]   				=    BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'							)
         self.assertDictEqual(new_tokenizer.vocab  ,      tokenizer.vocab							)





       @require_tokenizers
       def       _a   ( self							):
         CustomTokenizer.register_for_auto_class()
         with tempfile.TemporaryDirectory() as tmp_dir:
           UpperCamelCase_: Optional[Any]   				=    os.path.join(_lowerCamelCase  ,      'vocab.txt'							)
           with open(_lowerCamelCase  ,      'w'  ,      encoding='utf-8'							) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]							)							)
           UpperCamelCase_: Optional[int]   				=    CustomTokenizer(_lowerCamelCase							)

         # No fast custom tokenizer
         tokenizer.push_to_hub('test-dynamic-tokenizer'  ,      use_auth_token=self._token							)

         UpperCamelCase_: str   				=    AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''  ,      trust_remote_code=_lowerCamelCase							)
         # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
         self.assertEqual(tokenizer.__class__.__name__  ,      'CustomTokenizer'							)

         # Fast and slow custom tokenizer
         CustomTokenizerFast.register_for_auto_class()
         with tempfile.TemporaryDirectory() as tmp_dir:
           UpperCamelCase_: Optional[Any]   				=    os.path.join(_lowerCamelCase  ,      'vocab.txt'							)
           with open(_lowerCamelCase  ,      'w'  ,      encoding='utf-8'							) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]							)							)

           UpperCamelCase_: Dict   				=    BertTokenizerFast.from_pretrained(_lowerCamelCase							)
           bert_tokenizer.save_pretrained(_lowerCamelCase							)
           UpperCamelCase_: List[str]   				=    CustomTokenizerFast.from_pretrained(_lowerCamelCase							)

         tokenizer.push_to_hub('test-dynamic-tokenizer'  ,      use_auth_token=self._token							)

         UpperCamelCase_: Tuple   				=    AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''  ,      trust_remote_code=_lowerCamelCase							)
         # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
         self.assertEqual(tokenizer.__class__.__name__  ,      'CustomTokenizerFast'							)
         UpperCamelCase_: int   				=    AutoTokenizer.from_pretrained(
             f'''{USER}/test-dynamic-tokenizer'''  ,      use_fast=_lowerCamelCase  ,      trust_remote_code=_lowerCamelCase							)
         # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
         self.assertEqual(tokenizer.__class__.__name__  ,      'CustomTokenizer'							)







class     _lowerCAmelCase(	unittest.TestCase ):





       """simple docstring"""







       def       _a   ( self							):
         UpperCamelCase_: Dict   				=    Trie()
         trie.add('Hello 友達'							)
         self.assertEqual(trie.data  ,      {'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}							)
         trie.add('Hello'							)
         trie.data
         self.assertEqual(trie.data  ,      {'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}							)





       def       _a   ( self							):
         UpperCamelCase_: Optional[int]   				=    Trie()
         self.assertEqual(trie.split('[CLS] This is a extra_id_100'							)  ,      ['[CLS] This is a extra_id_100']							)
         trie.add('[CLS]'							)
         trie.add('extra_id_1'							)
         trie.add('extra_id_100'							)
         self.assertEqual(trie.split('[CLS] This is a extra_id_100'							)  ,      ['[CLS]', ' This is a ', 'extra_id_100']							)





       def       _a   ( self							):
         UpperCamelCase_: int   				=    Trie()
         trie.add('A'							)
         self.assertEqual(trie.split('ABC'							)  ,      ['A', 'BC']							)
         self.assertEqual(trie.split('BCA'							)  ,      ['BC', 'A']							)





       def       _a   ( self							):
         UpperCamelCase_: str   				=    Trie()
         trie.add('TOKEN]'							)
         trie.add('[SPECIAL_TOKEN]'							)
         self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'							)  ,      ['This is something ', '[SPECIAL_TOKEN]']							)





       def       _a   ( self							):
         UpperCamelCase_: Union[str, Any]   				=    Trie()
         trie.add('A'							)
         trie.add('P'							)
         trie.add('[SPECIAL_TOKEN]'							)
         self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'							)  ,      ['This is something ', '[SPECIAL_TOKEN]']							)





       def       _a   ( self							):
         UpperCamelCase_: List[str]   				=    Trie()
         trie.add('AB'							)
         trie.add('B'							)
         trie.add('C'							)
         self.assertEqual(trie.split('ABC'							)  ,      ['AB', 'C']							)





       def       _a   ( self							):
         UpperCamelCase_: List[str]   				=    Trie()
         trie.add('ABC'							)
         trie.add('B'							)
         trie.add('CD'							)
         self.assertEqual(trie.split('ABCD'							)  ,      ['ABC', 'D']							)





       def       _a   ( self							):
         # Even if the offsets are wrong, we necessarily output correct string
         # parts.
         UpperCamelCase_: Union[str, Any]   				=    Trie()
         UpperCamelCase_: Any   				=    trie.cut_text('ABC'  ,      [0, 0, 2, 1, 2, 3]							)
         self.assertEqual(_lowerCamelCase  ,      ['AB', 'C']							)