


"""simple docstring"""



import unittest

from transformers import DebertaConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      DebertaForMaskedLM,
      DebertaForQuestionAnswering,
      DebertaForSequenceClassification,
      DebertaForTokenClassification,
      DebertaModel,
  )
  from transformers.models.deberta.modeling_deberta import DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST







class _lowerCAmelCase							( snake_case_      ):







   def __init__(      self		,      UpperCamelCase__		,      UpperCamelCase__=13		,      UpperCamelCase__=7		,      UpperCamelCase__=True		,      UpperCamelCase__=True		,      UpperCamelCase__=True		,      UpperCamelCase__=True		,      UpperCamelCase__=99		,      UpperCamelCase__=32		,      UpperCamelCase__=5		,      UpperCamelCase__=4		,      UpperCamelCase__=37		,      UpperCamelCase__="gelu"		,      UpperCamelCase__=0.1		,      UpperCamelCase__=0.1		,      UpperCamelCase__=512		,      UpperCamelCase__=16		,      UpperCamelCase__=2		,      UpperCamelCase__=0.02		,      UpperCamelCase__=False		,      UpperCamelCase__=True		,      UpperCamelCase__="None"		,      UpperCamelCase__=3		,      UpperCamelCase__=4		,      UpperCamelCase__=None		,      )      ->						List[Any]:

       '''simple docstring'''




       snake_case       : Tuple    =       parent
       snake_case       : Any    =       batch_size
       snake_case       : Optional[int]    =       seq_length
       snake_case       : Dict    =       is_training
       snake_case       : List[str]    =       use_input_mask
       snake_case       : Optional[int]    =       use_token_type_ids
       snake_case       : int    =       use_labels
       snake_case       : int    =       vocab_size
       snake_case       : str    =       hidden_size
       snake_case       : Optional[int]    =       num_hidden_layers
       snake_case       : Any    =       num_attention_heads
       snake_case       : List[Any]    =       intermediate_size
       snake_case       : Optional[int]    =       hidden_act
       snake_case       : Any    =       hidden_dropout_prob
       snake_case       : Any    =       attention_probs_dropout_prob
       snake_case       : Dict    =       max_position_embeddings
       snake_case       : Dict    =       type_vocab_size
       snake_case       : Any    =       type_sequence_label_size
       snake_case       : Dict    =       initializer_range
       snake_case       : List[str]    =       num_labels
       snake_case       : Dict    =       num_choices
       snake_case       : Optional[Any]    =       relative_attention
       snake_case       : List[Any]    =       position_biased_input
       snake_case       : int    =       pos_att_type
       snake_case       : Tuple    =       scope







   def 					lowerCamelCase						(      self		)      ->						Dict:

       '''simple docstring'''




       snake_case       : Optional[int]    =       ids_tensor([self.batch_size, self.seq_length]		,      self.vocab_size		)

       snake_case       : Dict    =       None
       if self.use_input_mask:
           snake_case       : List[str]    =       ids_tensor([self.batch_size, self.seq_length]		,      vocab_size=2		)

       snake_case       : Any    =       None
       if self.use_token_type_ids:
           snake_case       : int    =       ids_tensor([self.batch_size, self.seq_length]		,      self.type_vocab_size		)

       snake_case       : List[Any]    =       None
       snake_case       : Any    =       None
       snake_case       : Tuple    =       None
       if self.use_labels:
           snake_case       : Optional[Any]    =       ids_tensor([self.batch_size]		,      self.type_sequence_label_size		)
           snake_case       : Optional[Any]    =       ids_tensor([self.batch_size, self.seq_length]		,      self.num_labels		)
           snake_case       : str    =       ids_tensor([self.batch_size]		,      self.num_choices		)

       snake_case       : int    =       self.get_config()

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







   def 					lowerCamelCase						(      self		)      ->						str:

       '''simple docstring'''




       return DebertaConfig(
           vocab_size=self.vocab_size		,      hidden_size=self.hidden_size		,      num_hidden_layers=self.num_hidden_layers		,      num_attention_heads=self.num_attention_heads		,      intermediate_size=self.intermediate_size		,      hidden_act=self.hidden_act		,      hidden_dropout_prob=self.hidden_dropout_prob		,      attention_probs_dropout_prob=self.attention_probs_dropout_prob		,      max_position_embeddings=self.max_position_embeddings		,      type_vocab_size=self.type_vocab_size		,      initializer_range=self.initializer_range		,      relative_attention=self.relative_attention		,      position_biased_input=self.position_biased_input		,      pos_att_type=self.pos_att_type		,      )







   def 					lowerCamelCase						(      self		)      ->						Union[str, Any]:

       '''simple docstring'''




       snake_case       : List[Any]    =       self.get_config()
       snake_case       : List[Any]    =       300
       return config







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		)      ->						Any:

       '''simple docstring'''




       self.parent.assertListEqual(list(result.loss.size()		)		,      []		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		)      ->						List[str]:

       '''simple docstring'''




       snake_case       : List[Any]    =       DebertaModel(config=UpperCamelCase__		)
       model.to(UpperCamelCase__		)
       model.eval()
       snake_case       : Optional[Any]    =       model(UpperCamelCase__		,      attention_mask=UpperCamelCase__		,      token_type_ids=UpperCamelCase__		)[0]
       snake_case       : Any    =       model(UpperCamelCase__		,      token_type_ids=UpperCamelCase__		)[0]
       snake_case       : Union[str, Any]    =       model(UpperCamelCase__		)[0]

       self.parent.assertListEqual(list(sequence_output.size()		)		,      [self.batch_size, self.seq_length, self.hidden_size]		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		)      ->						Optional[int]:

       '''simple docstring'''




       snake_case       : Optional[int]    =       DebertaForMaskedLM(config=UpperCamelCase__		)
       model.to(UpperCamelCase__		)
       model.eval()
       snake_case       : Dict    =       model(UpperCamelCase__		,      attention_mask=UpperCamelCase__		,      token_type_ids=UpperCamelCase__		,      labels=UpperCamelCase__		)

       self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.seq_length, self.vocab_size)		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		)      ->						Tuple:

       '''simple docstring'''




       snake_case       : Tuple    =       self.num_labels
       snake_case       : str    =       DebertaForSequenceClassification(UpperCamelCase__		)
       model.to(UpperCamelCase__		)
       model.eval()
       snake_case       : Any    =       model(UpperCamelCase__		,      attention_mask=UpperCamelCase__		,      token_type_ids=UpperCamelCase__		,      labels=UpperCamelCase__		)
       self.parent.assertListEqual(list(result.logits.size()		)		,      [self.batch_size, self.num_labels]		)
       self.check_loss_output(UpperCamelCase__		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		)      ->						List[Any]:

       '''simple docstring'''




       snake_case       : Dict    =       self.num_labels
       snake_case       : Union[str, Any]    =       DebertaForTokenClassification(config=UpperCamelCase__		)
       model.to(UpperCamelCase__		)
       model.eval()
       snake_case       : int    =       model(UpperCamelCase__		,      attention_mask=UpperCamelCase__		,      token_type_ids=UpperCamelCase__		,      labels=UpperCamelCase__		)
       self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.seq_length, self.num_labels)		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		,      UpperCamelCase__		)      ->						int:

       '''simple docstring'''




       snake_case       : List[str]    =       DebertaForQuestionAnswering(config=UpperCamelCase__		)
       model.to(UpperCamelCase__		)
       model.eval()
       snake_case       : Any    =       model(
           UpperCamelCase__		,      attention_mask=UpperCamelCase__		,      token_type_ids=UpperCamelCase__		,      start_positions=UpperCamelCase__		,      end_positions=UpperCamelCase__		,      )
       self.parent.assertEqual(result.start_logits.shape		,      (self.batch_size, self.seq_length)		)
       self.parent.assertEqual(result.end_logits.shape		,      (self.batch_size, self.seq_length)		)







   def 					lowerCamelCase						(      self		)      ->						str:

       '''simple docstring'''




       snake_case       : Tuple    =       self.prepare_config_and_inputs()
       (
           (
           snake_case
       )   ,(
           snake_case
       )   ,(
           snake_case
       )   ,(
           snake_case
       )   ,(
           snake_case
       )   ,(
           snake_case
       )   ,(
           snake_case
       )   ,
       )       : str    =       config_and_inputs
       snake_case       : List[Any]    =       {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
       return config, inputs_dict







@require_torch
class _lowerCAmelCase							( snake_case_       ,				snake_case_       ,				unittest.TestCase      ):
   __UpperCAmelCase					:			List[str]				    =    (
       (
           DebertaModel,
           DebertaForMaskedLM,
           DebertaForSequenceClassification,
           DebertaForTokenClassification,
           DebertaForQuestionAnswering,
       )
       if is_torch_available()
       else ()
   )
   __UpperCAmelCase					:			List[Any]				    =    (
       {
           '''feature-extraction''': DebertaModel,
           '''fill-mask''': DebertaForMaskedLM,
           '''question-answering''': DebertaForQuestionAnswering,
           '''text-classification''': DebertaForSequenceClassification,
           '''token-classification''': DebertaForTokenClassification,
           '''zero-shot''': DebertaForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )

   __UpperCAmelCase					:			List[str]				    =    True
   __UpperCAmelCase					:			Union[str, Any]				    =    False
   __UpperCAmelCase					:			str				    =    False
   __UpperCAmelCase					:			str				    =    False
   __UpperCAmelCase					:			Any				    =    False







   def 					lowerCamelCase						(      self		)      ->						str:

       '''simple docstring'''




       snake_case       : Union[str, Any]    =       DebertaModelTester(self		)
       snake_case       : Optional[int]    =       ConfigTester(self		,      config_class=UpperCamelCase__		,      hidden_size=37		)







   def 					lowerCamelCase						(      self		)      ->						Union[str, Any]:

       '''simple docstring'''




       self.config_tester.run_common_tests()







   def 					lowerCamelCase						(      self		)      ->						str:

       '''simple docstring'''




       snake_case       : str    =       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_deberta_model(*UpperCamelCase__		)







   def 					lowerCamelCase						(      self		)      ->						Dict:

       '''simple docstring'''




       snake_case       : Optional[Any]    =       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_deberta_for_sequence_classification(*UpperCamelCase__		)







   def 					lowerCamelCase						(      self		)      ->						Dict:

       '''simple docstring'''




       snake_case       : Dict    =       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_deberta_for_masked_lm(*UpperCamelCase__		)







   def 					lowerCamelCase						(      self		)      ->						int:

       '''simple docstring'''




       snake_case       : Tuple    =       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_deberta_for_question_answering(*UpperCamelCase__		)







   def 					lowerCamelCase						(      self		)      ->						Dict:

       '''simple docstring'''




       snake_case       : List[str]    =       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_deberta_for_token_classification(*UpperCamelCase__		)







   @slow
   def 					lowerCamelCase						(      self		)      ->						Dict:

       '''simple docstring'''




       for model_name in DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           snake_case       : List[str]    =       DebertaModel.from_pretrained(UpperCamelCase__		)
           self.assertIsNotNone(UpperCamelCase__		)







@require_torch
@require_sentencepiece
@require_tokenizers
class _lowerCAmelCase							( unittest.TestCase      ):







   @unittest.skip(reason="Model not available yet"		)
   def 					lowerCamelCase						(      self		)      ->						List[str]:

       '''simple docstring'''




       pass







   @slow
   def 					lowerCamelCase						(      self		)      ->						Tuple:

       '''simple docstring'''




       snake_case       : int    =       DebertaModel.from_pretrained("microsoft/deberta-base"		)

       snake_case       : Optional[int]    =       torch.tensor([[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]]		)
       snake_case       : Tuple    =       torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]		)
       with torch.no_grad():
           snake_case       : int    =       model(UpperCamelCase__		,      attention_mask=UpperCamelCase__		)[0]
       # compare the actual values for a slice.
       snake_case       : int    =       torch.tensor(
           [[[-0.5986, -0.8055, -0.8462], [1.4484, -0.9348, -0.8059], [0.3123, 0.0032, -1.4131]]]		)
       self.assertTrue(torch.allclose(output[:, 1:4, 1:4]		,      UpperCamelCase__		,      atol=1e-4		)		,      F'{output[:, 1:4, 1:4]}'		)









"""simple docstring"""



import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
  from test_module.custom_tokenization_fast import CustomTokenizerFast







class _lowerCAmelCase							( unittest.TestCase      ):







   def 					lowerCamelCase						(      self		)      ->						str:

       '''simple docstring'''




       snake_case       : List[str]    =       mock.Mock()
       snake_case       : Any    =       500
       snake_case       : int    =       {}
       snake_case       : int    =       HTTPError
       snake_case       : Dict    =       {}

       # Download this model to make sure it's in the cache.
       snake_case       : Dict    =       BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"		)

       # Under the mock environment we get a 500 error when trying to reach the tokenizer.
       with mock.patch("requests.Session.request"		,      return_value=UpperCamelCase__		) as mock_head:
           snake_case       : str    =       BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"		)
           # This check we did call the fake head request
           mock_head.assert_called()







   @require_tokenizers
   def 					lowerCamelCase						(      self		)      ->						int:

       '''simple docstring'''




       snake_case       : Union[str, Any]    =       mock.Mock()
       snake_case       : int    =       500
       snake_case       : Optional[int]    =       {}
       snake_case       : int    =       HTTPError
       snake_case       : int    =       {}

       # Download this model to make sure it's in the cache.
       snake_case       : Any    =       GPTaTokenizerFast.from_pretrained("gpt2"		)

       # Under the mock environment we get a 500 error when trying to reach the tokenizer.
       with mock.patch("requests.Session.request"		,      return_value=UpperCamelCase__		) as mock_head:
           snake_case       : Tuple    =       GPTaTokenizerFast.from_pretrained("gpt2"		)
           # This check we did call the fake head request
           mock_head.assert_called()







   def 					lowerCamelCase						(      self		)      ->						List[str]:

       '''simple docstring'''




       try:
           snake_case       : Tuple    =       tempfile.mktemp()
           with open(UpperCamelCase__		,      "wb"		) as f:
               http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"		,      UpperCamelCase__		)

           snake_case       : List[str]    =       AlbertTokenizer.from_pretrained(UpperCamelCase__		)
       finally:
           os.remove(UpperCamelCase__		)

       # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
       # the current folder and have the right name.
       if os.path.isfile("tokenizer.json"		):
           # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
           return
       try:
           with open("tokenizer.json"		,      "wb"		) as f:
               http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"		,      UpperCamelCase__		)
           snake_case       : Dict    =       AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"		)
           # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
           self.assertEqual(tokenizer.vocab_size		,      1000		)
           # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

       finally:
           os.remove("tokenizer.json"		)







   def 					lowerCamelCase						(      self		)      ->						Optional[Any]:

       '''simple docstring'''




       snake_case       : int    =       AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"		)







@is_staging_test
class _lowerCAmelCase							( unittest.TestCase      ):
   __UpperCAmelCase					:			Dict				    =    ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']







   @classmethod
   def 					lowerCamelCase						(      cls		)      ->						Optional[int]:

       '''simple docstring'''




       snake_case       : List[str]    =       TOKEN
       HfFolder.save_token(UpperCamelCase__		)







   @classmethod
   def 					lowerCamelCase						(      cls		)      ->						Union[str, Any]:

       '''simple docstring'''




       try:
           delete_repo(token=cls._token		,      repo_id="test-tokenizer"		)
       except HTTPError:
           pass

       try:
           delete_repo(token=cls._token		,      repo_id="valid_org/test-tokenizer-org"		)
       except HTTPError:
           pass

       try:
           delete_repo(token=cls._token		,      repo_id="test-dynamic-tokenizer"		)
       except HTTPError:
           pass







   def 					lowerCamelCase						(      self		)      ->						Dict:

       '''simple docstring'''




       with tempfile.TemporaryDirectory() as tmp_dir:
           snake_case       : Any    =       os.path.join(UpperCamelCase__		,      "vocab.txt"		)
           with open(UpperCamelCase__		,      "w"		,      encoding="utf-8"		) as vocab_writer:
               vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]		)		)
           snake_case       : List[str]    =       BertTokenizer(UpperCamelCase__		)

       tokenizer.push_to_hub("test-tokenizer"		,      use_auth_token=self._token		)
       snake_case       : Dict    =       BertTokenizer.from_pretrained(F'{USER}/test-tokenizer'		)
       self.assertDictEqual(new_tokenizer.vocab		,      tokenizer.vocab		)

       # Reset repo
       delete_repo(token=self._token		,      repo_id="test-tokenizer"		)

       # Push to hub via save_pretrained
       with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(UpperCamelCase__		,      repo_id="test-tokenizer"		,      push_to_hub=UpperCamelCase__		,      use_auth_token=self._token		)

       snake_case       : Dict    =       BertTokenizer.from_pretrained(F'{USER}/test-tokenizer'		)
       self.assertDictEqual(new_tokenizer.vocab		,      tokenizer.vocab		)







   def 					lowerCamelCase						(      self		)      ->						Any:

       '''simple docstring'''




       with tempfile.TemporaryDirectory() as tmp_dir:
           snake_case       : Dict    =       os.path.join(UpperCamelCase__		,      "vocab.txt"		)
           with open(UpperCamelCase__		,      "w"		,      encoding="utf-8"		) as vocab_writer:
               vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]		)		)
           snake_case       : Optional[int]    =       BertTokenizer(UpperCamelCase__		)

       tokenizer.push_to_hub("valid_org/test-tokenizer-org"		,      use_auth_token=self._token		)
       snake_case       : Dict    =       BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"		)
       self.assertDictEqual(new_tokenizer.vocab		,      tokenizer.vocab		)

       # Reset repo
       delete_repo(token=self._token		,      repo_id="valid_org/test-tokenizer-org"		)

       # Push to hub via save_pretrained
       with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(
               UpperCamelCase__		,      repo_id="valid_org/test-tokenizer-org"		,      push_to_hub=UpperCamelCase__		,      use_auth_token=self._token		)

       snake_case       : Optional[int]    =       BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"		)
       self.assertDictEqual(new_tokenizer.vocab		,      tokenizer.vocab		)







   @require_tokenizers
   def 					lowerCamelCase						(      self		)      ->						Union[str, Any]:

       '''simple docstring'''




       CustomTokenizer.register_for_auto_class()
       with tempfile.TemporaryDirectory() as tmp_dir:
           snake_case       : Tuple    =       os.path.join(UpperCamelCase__		,      "vocab.txt"		)
           with open(UpperCamelCase__		,      "w"		,      encoding="utf-8"		) as vocab_writer:
               vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]		)		)
           snake_case       : Any    =       CustomTokenizer(UpperCamelCase__		)

       # No fast custom tokenizer
       tokenizer.push_to_hub("test-dynamic-tokenizer"		,      use_auth_token=self._token		)

       snake_case       : Union[str, Any]    =       AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'		,      trust_remote_code=UpperCamelCase__		)
       # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
       self.assertEqual(tokenizer.__class__.__name__		,      "CustomTokenizer"		)

       # Fast and slow custom tokenizer
       CustomTokenizerFast.register_for_auto_class()
       with tempfile.TemporaryDirectory() as tmp_dir:
           snake_case       : int    =       os.path.join(UpperCamelCase__		,      "vocab.txt"		)
           with open(UpperCamelCase__		,      "w"		,      encoding="utf-8"		) as vocab_writer:
               vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]		)		)

           snake_case       : Optional[Any]    =       BertTokenizerFast.from_pretrained(UpperCamelCase__		)
           bert_tokenizer.save_pretrained(UpperCamelCase__		)
           snake_case       : List[Any]    =       CustomTokenizerFast.from_pretrained(UpperCamelCase__		)

       tokenizer.push_to_hub("test-dynamic-tokenizer"		,      use_auth_token=self._token		)

       snake_case       : Union[str, Any]    =       AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'		,      trust_remote_code=UpperCamelCase__		)
       # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
       self.assertEqual(tokenizer.__class__.__name__		,      "CustomTokenizerFast"		)
       snake_case       : Optional[Any]    =       AutoTokenizer.from_pretrained(
           F'{USER}/test-dynamic-tokenizer'		,      use_fast=UpperCamelCase__		,      trust_remote_code=UpperCamelCase__		)
       # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
       self.assertEqual(tokenizer.__class__.__name__		,      "CustomTokenizer"		)







class _lowerCAmelCase							( unittest.TestCase      ):







   def 					lowerCamelCase						(      self		)      ->						str:

       '''simple docstring'''




       snake_case       : Any    =       Trie()
       trie.add("Hello 友達"		)
       self.assertEqual(trie.data		,      {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}		)
       trie.add("Hello"		)
       trie.data
       self.assertEqual(trie.data		,      {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}		)







   def 					lowerCamelCase						(      self		)      ->						Any:

       '''simple docstring'''




       snake_case       : Tuple    =       Trie()
       self.assertEqual(trie.split("[CLS] This is a extra_id_100"		)		,      ["[CLS] This is a extra_id_100"]		)
       trie.add("[CLS]"		)
       trie.add("extra_id_1"		)
       trie.add("extra_id_100"		)
       self.assertEqual(trie.split("[CLS] This is a extra_id_100"		)		,      ["[CLS]", " This is a ", "extra_id_100"]		)







   def 					lowerCamelCase						(      self		)      ->						List[str]:

       '''simple docstring'''




       snake_case       : str    =       Trie()
       trie.add("A"		)
       self.assertEqual(trie.split("ABC"		)		,      ["A", "BC"]		)
       self.assertEqual(trie.split("BCA"		)		,      ["BC", "A"]		)







   def 					lowerCamelCase						(      self		)      ->						int:

       '''simple docstring'''




       snake_case       : Union[str, Any]    =       Trie()
       trie.add("TOKEN]"		)
       trie.add("[SPECIAL_TOKEN]"		)
       self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"		)		,      ["This is something ", "[SPECIAL_TOKEN]"]		)







   def 					lowerCamelCase						(      self		)      ->						Tuple:

       '''simple docstring'''




       snake_case       : Optional[int]    =       Trie()
       trie.add("A"		)
       trie.add("P"		)
       trie.add("[SPECIAL_TOKEN]"		)
       self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"		)		,      ["This is something ", "[SPECIAL_TOKEN]"]		)







   def 					lowerCamelCase						(      self		)      ->						List[str]:

       '''simple docstring'''




       snake_case       : List[Any]    =       Trie()
       trie.add("AB"		)
       trie.add("B"		)
       trie.add("C"		)
       self.assertEqual(trie.split("ABC"		)		,      ["AB", "C"]		)







   def 					lowerCamelCase						(      self		)      ->						Dict:

       '''simple docstring'''




       snake_case       : str    =       Trie()
       trie.add("ABC"		)
       trie.add("B"		)
       trie.add("CD"		)
       self.assertEqual(trie.split("ABCD"		)		,      ["ABC", "D"]		)







   def 					lowerCamelCase						(      self		)      ->						Optional[Any]:

       '''simple docstring'''




       snake_case       : Optional[int]    =       Trie()
       snake_case       : Union[str, Any]    =       trie.cut_text("ABC"		,      [0, 0, 2, 1, 2, 3]		)
       self.assertEqual(UpperCamelCase__		,      ["AB", "C"]		)





