


import copy
import inspect
import unittest

from transformers import PretrainedConfig, SwiftFormerConfig
from transformers.testing_utils import (
    require_torch,
    require_vision,
    slow,
    torch_device,
)
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch
       from torch import nn

       from transformers import SwiftFormerForImageClassification, SwiftFormerModel
       from transformers.models.swiftformer.modeling_swiftformer import SWIFTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
       from PIL import Image

       from transformers import ViTImageProcessor

class       __magic_name__	:


     def __init__(      self							:     str							,	lowerCamelCase__							:     Union[str, Any]							,	lowerCamelCase__							:     Tuple=13							,	lowerCamelCase__							:     List[str]=3							,	lowerCamelCase__							:     str=True							,	lowerCamelCase__							:     Any=True							,	lowerCamelCase__							:     List[Any]=0.1							,	lowerCamelCase__							:     Dict=0.1							,	lowerCamelCase__							:     List[Any]=224							,	lowerCamelCase__							:     str=1000							,	lowerCamelCase__							:     Tuple=[3, 3, 6, 4]							,	lowerCamelCase__							:     int=[48, 56, 112, 220]							,	)  ->      List[Any]:





      '''simple docstring'''






      UpperCamelCase__					:    List[Any]          =   parent
      UpperCamelCase__					:    Dict          =   batch_size
      UpperCamelCase__					:    str          =   num_channels
      UpperCamelCase__					:    List[Any]          =   is_training
      UpperCamelCase__					:    str          =   use_labels
      UpperCamelCase__					:    List[str]          =   hidden_dropout_prob
      UpperCamelCase__					:    Optional[int]          =   attention_probs_dropout_prob
      UpperCamelCase__					:    Any          =   num_labels
      UpperCamelCase__					:    Any          =   image_size
      UpperCamelCase__					:    int          =   layer_depths
      UpperCamelCase__					:    Any          =   embed_dims


     def      UpperCAmelCase__	(      self							:     Tuple			)  ->      List[str]:





      '''simple docstring'''






      UpperCamelCase__					:    Union[str, Any]          =   floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]			)

      UpperCamelCase__					:    Optional[Any]          =   None
      if self.use_labels:
       UpperCamelCase__					:    Optional[int]          =   ids_tensor([self.batch_size]							,	self.num_labels			)

      UpperCamelCase__					:    Optional[int]          =   self.get_config()

      return config, pixel_values, labels


     def      UpperCAmelCase__	(      self							:     Any			)  ->      Optional[Any]:





      '''simple docstring'''






      return SwiftFormerConfig(
          depths=self.layer_depths							,	embed_dims=self.embed_dims							,	mlp_ratio=4							,	downsamples=[True, True, True, True]							,	hidden_act='''gelu'''							,	num_labels=self.num_labels							,	down_patch_size=3							,	down_stride=2							,	down_pad=1							,	drop_rate=0.0							,	drop_path_rate=0.0							,	use_layer_scale=lowerCamelCase__							,	layer_scale_init_value=1E-5							,	)


     def      UpperCAmelCase__	(      self							:     str							,	lowerCamelCase__							:     str							,	lowerCamelCase__							:     int							,	lowerCamelCase__							:     List[str]			)  ->      Dict:





      '''simple docstring'''






      UpperCamelCase__					:    Optional[int]          =   SwiftFormerModel(config=lowerCamelCase__			)
      model.to(lowerCamelCase__			)
      model.eval()
      UpperCamelCase__					:    Dict          =   model(lowerCamelCase__			)
      self.parent.assertEqual(result.last_hidden_state.shape							,	(self.batch_size, self.embed_dims[-1], 7, 7)			)


     def      UpperCAmelCase__	(      self							:     List[Any]							,	lowerCamelCase__							:     Tuple							,	lowerCamelCase__							:     List[str]							,	lowerCamelCase__							:     List[Any]			)  ->      str:





      '''simple docstring'''






      UpperCamelCase__					:    Optional[int]          =   self.num_labels
      UpperCamelCase__					:    List[Any]          =   SwiftFormerForImageClassification(lowerCamelCase__			)
      model.to(lowerCamelCase__			)
      model.eval()
      UpperCamelCase__					:    Optional[Any]          =   model(lowerCamelCase__							,	labels=lowerCamelCase__			)
      self.parent.assertEqual(result.logits.shape							,	(self.batch_size, self.num_labels)			)

      UpperCamelCase__					:    Tuple          =   SwiftFormerForImageClassification(lowerCamelCase__			)
      model.to(lowerCamelCase__			)
      model.eval()

      UpperCamelCase__					:    Tuple          =   floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]			)
      UpperCamelCase__					:    Optional[int]          =   model(lowerCamelCase__			)
      self.parent.assertEqual(result.logits.shape							,	(self.batch_size, self.num_labels)			)


     def      UpperCAmelCase__	(      self							:     Optional[Any]			)  ->      str:





      '''simple docstring'''






      ((UpperCamelCase__)       ,				(UpperCamelCase__)       ,				(UpperCamelCase__))					:    Union[str, Any]          =   self.prepare_config_and_inputs()
      UpperCamelCase__					:    Tuple          =   {'''pixel_values''': pixel_values}
      return config, inputs_dict



@require_torch
class       __magic_name__	(     __lowerCAmelCase					,				__lowerCAmelCase					,				unittest.TestCase):
     A:		Tuple			=      (SwiftFormerModel, SwiftFormerForImageClassification) if is_torch_available() else ()
     A:		Optional[int]			=      (
         {"feature-extraction": SwiftFormerModel, "image-classification": SwiftFormerForImageClassification}
         if is_torch_available()
         else {}
     )

     A:		Optional[Any]			=      False
     A:		Tuple			=      False
     A:		List[str]			=      False
     A:		Union[str, Any]			=      False
     A:		List[str]			=      False


     def      UpperCAmelCase__	(      self							:     int			)  ->      List[str]:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   SwiftFormerModelTester(self			)
      UpperCamelCase__					:    Any          =   ConfigTester(
          self							,	config_class=lowerCamelCase__							,	has_text_modality=lowerCamelCase__							,	hidden_size=37							,	num_attention_heads=12							,	num_hidden_layers=12							,	)


     def      UpperCAmelCase__	(      self							:     Optional[int]			)  ->      List[Any]:





      '''simple docstring'''






      self.config_tester.run_common_tests()


     @unittest.skip(reason='''SwiftFormer does not use inputs_embeds'''			)
     def      UpperCAmelCase__	(      self							:     Optional[int]			)  ->      List[str]:





      '''simple docstring'''






      pass


     def      UpperCAmelCase__	(      self							:     str			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__       ,				UpperCamelCase__					:    int          =   self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
       UpperCamelCase__					:    Any          =   model_class(lowerCamelCase__			)
       UpperCamelCase__					:    Optional[Any]          =   model.get_output_embeddings()
       self.assertTrue(x is None or isinstance(lowerCamelCase__							,	nn.Linear			)			)


     def      UpperCAmelCase__	(      self							:     int			)  ->      Optional[int]:





      '''simple docstring'''






      UpperCamelCase__       ,				UpperCamelCase__					:    Dict          =   self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
       UpperCamelCase__					:    Tuple          =   model_class(lowerCamelCase__			)
       UpperCamelCase__					:    Tuple          =   inspect.signature(model.forward			)
       # signature.parameters is an OrderedDict => so arg_names order is deterministic
       UpperCamelCase__					:    Union[str, Any]          =   [*signature.parameters.keys()]

       UpperCamelCase__					:    Optional[int]          =   ['''pixel_values''']
       self.assertListEqual(arg_names[:1]							,	lowerCamelCase__			)


     def      UpperCAmelCase__	(      self							:     str			)  ->      Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__					:    List[Any]          =   self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*lowerCamelCase__			)


     def      UpperCAmelCase__	(      self							:     Optional[int]			)  ->      str:





      '''simple docstring'''






      UpperCamelCase__					:    str          =   self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_image_classification(*lowerCamelCase__			)


     @slow
     def      UpperCAmelCase__	(      self							:     Union[str, Any]			)  ->      List[str]:





      '''simple docstring'''






      for model_name in SWIFTFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       UpperCamelCase__					:    Tuple          =   SwiftFormerModel.from_pretrained(lowerCamelCase__			)
       self.assertIsNotNone(lowerCamelCase__			)


     @unittest.skip(reason='''SwiftFormer does not output attentions'''			)
     def      UpperCAmelCase__	(      self							:     Dict			)  ->      str:





      '''simple docstring'''






      pass


     def      UpperCAmelCase__	(      self							:     Dict			)  ->      List[Any]:





      '''simple docstring'''






      def check_hidden_states_output(lowerCamelCase__							:     Union[str, Any]							,	lowerCamelCase__							:     List[str]							,	lowerCamelCase__							:     Optional[int]			):
       UpperCamelCase__					:    Optional[int]          =   model_class(lowerCamelCase__			)
       model.to(lowerCamelCase__			)
       model.eval()

       with torch.no_grad():
        UpperCamelCase__					:    Optional[int]          =   model(**self._prepare_for_class(lowerCamelCase__							,	lowerCamelCase__			)			)

       UpperCamelCase__					:    Tuple          =   outputs.hidden_states

       UpperCamelCase__					:    int          =   8
       self.assertEqual(len(lowerCamelCase__			)							,	lowerCamelCase__			)  # TODO

       # SwiftFormer's feature maps are of shape (batch_size, embed_dims, height, width)
       # with the width and height being successively divided by 2, after every 2 blocks
       for i in range(len(lowerCamelCase__			)			):
        self.assertEqual(
            hidden_states[i].shape							,	torch.Size(
                [
                    self.model_tester.batch_size,
                    self.model_tester.embed_dims[i // 2],
                    (self.model_tester.image_size // 4) // 2 ** (i // 2),
                    (self.model_tester.image_size // 4) // 2 ** (i // 2),
                ]			)							,	)

      UpperCamelCase__       ,				UpperCamelCase__					:    Optional[int]          =   self.model_tester.prepare_config_and_inputs_for_common()

      for model_class in self.all_model_classes:
       UpperCamelCase__					:    List[str]          =   True
       check_hidden_states_output(lowerCamelCase__							,	lowerCamelCase__							,	lowerCamelCase__			)

       # check that output_hidden_states also work using config
       del inputs_dict["output_hidden_states"]
       UpperCamelCase__					:    str          =   True

       check_hidden_states_output(lowerCamelCase__							,	lowerCamelCase__							,	lowerCamelCase__			)


     def      UpperCAmelCase__	(      self							:     List[Any]			)  ->      Optional[int]:





      '''simple docstring'''






      def _config_zero_init(lowerCamelCase__							:     Optional[int]			):
       UpperCamelCase__					:    int          =   copy.deepcopy(lowerCamelCase__			)
       for key in configs_no_init.__dict__.keys():
        if "_range" in key or "_std" in key or "initializer_factor" in key or "layer_scale" in key:
         setattr(lowerCamelCase__							,	lowerCamelCase__							,	1E-1_0			)
        if isinstance(getattr(lowerCamelCase__							,	lowerCamelCase__							,	lowerCamelCase__			)							,	lowerCamelCase__			):
         UpperCamelCase__					:    Optional[Any]          =   _config_zero_init(getattr(lowerCamelCase__							,	lowerCamelCase__			)			)
         setattr(lowerCamelCase__							,	lowerCamelCase__							,	lowerCamelCase__			)
       return configs_no_init

      UpperCamelCase__       ,				UpperCamelCase__					:    Dict          =   self.model_tester.prepare_config_and_inputs_for_common()

      UpperCamelCase__					:    int          =   _config_zero_init(lowerCamelCase__			)
      for model_class in self.all_model_classes:
       UpperCamelCase__					:    List[str]          =   model_class(config=lowerCamelCase__			)
       for name, param in model.named_parameters():
        if param.requires_grad:
         self.assertIn(
             ((param.data.mean() * 1E9) / 1E9).round().item()							,	[0.0, 1.0]							,	msg=F"Parameter {name} of model {model_class} seems not properly initialized"							,	)


     @unittest.skip('''Will be fixed soon by reducing the size of the model used for common tests.'''			)
     def      UpperCAmelCase__	(      self							:     str			)  ->      Optional[int]:





      '''simple docstring'''






      pass



def 			_a  (				):







 """simple docstring"""



 UpperCamelCase__					:    Tuple          =   Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''     )
 return image






@require_torch
@require_vision
class       __magic_name__	(     unittest.TestCase):


     @cached_property
     def      UpperCAmelCase__	(      self							:     List[str]			)  ->      Optional[int]:





      '''simple docstring'''






      return ViTImageProcessor.from_pretrained('''MBZUAI/swiftformer-xs'''			) if is_vision_available() else None


     @slow
     def      UpperCAmelCase__	(      self							:     str			)  ->      Union[str, Any]:





      '''simple docstring'''






      UpperCamelCase__					:    str          =   SwiftFormerForImageClassification.from_pretrained('''MBZUAI/swiftformer-xs'''			).to(lowerCamelCase__			)

      UpperCamelCase__					:    Any          =   self.default_image_processor
      UpperCamelCase__					:    Optional[int]          =   prepare_img()
      UpperCamelCase__					:    List[Any]          =   image_processor(images=lowerCamelCase__							,	return_tensors='''pt'''			).to(lowerCamelCase__			)

      # forward pass
      with torch.no_grad():
       UpperCamelCase__					:    Tuple          =   model(**lowerCamelCase__			)

      # verify the logits
      UpperCamelCase__					:    Union[str, Any]          =   torch.Size((1, 1000)			)
      self.assertEqual(outputs.logits.shape							,	lowerCamelCase__			)

      UpperCamelCase__					:    Tuple          =   torch.tensor([[-2.1_7_0_3E0_0, 2.1_1_0_7E0_0, -2.0_8_1_1E0_0]]			).to(lowerCamelCase__			)
      self.assertTrue(torch.allclose(outputs.logits[0, :3]							,	lowerCamelCase__							,	atol=1E-4			)			)






import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
       from test_module.custom_tokenization_fast import CustomTokenizerFast

class       __magic_name__	(     unittest.TestCase):


     def      UpperCAmelCase__	(      self							:     Union[str, Any]			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   mock.Mock()
      UpperCamelCase__					:    Tuple          =   500
      UpperCamelCase__					:    Dict          =   {}
      UpperCamelCase__					:    Dict          =   HTTPError
      UpperCamelCase__					:    Optional[Any]          =   {}

      # Download this model to make sure it's in the cache.
      UpperCamelCase__					:    List[Any]          =   BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''			)

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('''requests.Session.request'''							,	return_value=lowerCamelCase__			) as mock_head:
       UpperCamelCase__					:    Any          =   BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''			)
       # This check we did call the fake head request
       mock_head.assert_called()


     @require_tokenizers
     def      UpperCAmelCase__	(      self							:     str			)  ->      Any:





      '''simple docstring'''






      UpperCamelCase__					:    Union[str, Any]          =   mock.Mock()
      UpperCamelCase__					:    Any          =   500
      UpperCamelCase__					:    str          =   {}
      UpperCamelCase__					:    Optional[Any]          =   HTTPError
      UpperCamelCase__					:    Tuple          =   {}

      # Download this model to make sure it's in the cache.
      UpperCamelCase__					:    Optional[Any]          =   GPTaTokenizerFast.from_pretrained('''gpt2'''			)

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('''requests.Session.request'''							,	return_value=lowerCamelCase__			) as mock_head:
       UpperCamelCase__					:    str          =   GPTaTokenizerFast.from_pretrained('''gpt2'''			)
       # This check we did call the fake head request
       mock_head.assert_called()


     def      UpperCAmelCase__	(      self							:     Any			)  ->      List[str]:





      '''simple docstring'''






      try:
       UpperCamelCase__					:    Dict          =   tempfile.mktemp()
       with open(lowerCamelCase__							,	'''wb'''			) as f:
        http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''							,	lowerCamelCase__			)

       UpperCamelCase__					:    Dict          =   AlbertTokenizer.from_pretrained(lowerCamelCase__			)
      finally:
       os.remove(lowerCamelCase__			)

      # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
      # the current folder and have the right name.
      if os.path.isfile('''tokenizer.json'''			):
       # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
       return
      try:
       with open('''tokenizer.json'''							,	'''wb'''			) as f:
        http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''							,	lowerCamelCase__			)
       UpperCamelCase__					:    Union[str, Any]          =   AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''			)
       # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
       self.assertEqual(tokenizer.vocab_size							,	1000			)
       # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

      finally:
       os.remove('''tokenizer.json'''			)


     def      UpperCAmelCase__	(      self							:     str			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''			)



@is_staging_test
class       __magic_name__	(     unittest.TestCase):
     A:		List[Any]			=      ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]


     @classmethod
     def      UpperCAmelCase__	(      cls							:     List[str]			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    List[str]          =   TOKEN
      HfFolder.save_token(lowerCamelCase__			)


     @classmethod
     def      UpperCAmelCase__	(      cls							:     Any			)  ->      Optional[Any]:





      '''simple docstring'''






      try:
       delete_repo(token=cls._token							,	repo_id='''test-tokenizer'''			)
      except HTTPError:
       pass

      try:
       delete_repo(token=cls._token							,	repo_id='''valid_org/test-tokenizer-org'''			)
      except HTTPError:
       pass

      try:
       delete_repo(token=cls._token							,	repo_id='''test-dynamic-tokenizer'''			)
      except HTTPError:
       pass


     def      UpperCAmelCase__	(      self							:     Union[str, Any]			)  ->      Optional[int]:





      '''simple docstring'''






      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    int          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)
       UpperCamelCase__					:    Tuple          =   BertTokenizer(lowerCamelCase__			)

      tokenizer.push_to_hub('''test-tokenizer'''							,	use_auth_token=self._token			)
      UpperCamelCase__					:    Union[str, Any]          =   BertTokenizer.from_pretrained(F"{USER}/test-tokenizer"			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)

      # Reset repo
      delete_repo(token=self._token							,	repo_id='''test-tokenizer'''			)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(lowerCamelCase__							,	repo_id='''test-tokenizer'''							,	push_to_hub=lowerCamelCase__							,	use_auth_token=self._token			)

      UpperCamelCase__					:    str          =   BertTokenizer.from_pretrained(F"{USER}/test-tokenizer"			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)


     def      UpperCAmelCase__	(      self							:     str			)  ->      Optional[int]:





      '''simple docstring'''






      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    Dict          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)
       UpperCamelCase__					:    List[str]          =   BertTokenizer(lowerCamelCase__			)

      tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''							,	use_auth_token=self._token			)
      UpperCamelCase__					:    Optional[Any]          =   BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)

      # Reset repo
      delete_repo(token=self._token							,	repo_id='''valid_org/test-tokenizer-org'''			)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(
           lowerCamelCase__							,	repo_id='''valid_org/test-tokenizer-org'''							,	push_to_hub=lowerCamelCase__							,	use_auth_token=self._token			)

      UpperCamelCase__					:    Union[str, Any]          =   BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)


     @require_tokenizers
     def      UpperCAmelCase__	(      self							:     List[str]			)  ->      int:





      '''simple docstring'''






      CustomTokenizer.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    List[Any]          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)
       UpperCamelCase__					:    Any          =   CustomTokenizer(lowerCamelCase__			)

      # No fast custom tokenizer
      tokenizer.push_to_hub('''test-dynamic-tokenizer'''							,	use_auth_token=self._token			)

      UpperCamelCase__					:    str          =   AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"							,	trust_remote_code=lowerCamelCase__			)
      # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__							,	'''CustomTokenizer'''			)

      # Fast and slow custom tokenizer
      CustomTokenizerFast.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    List[Any]          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)

       UpperCamelCase__					:    List[Any]          =   BertTokenizerFast.from_pretrained(lowerCamelCase__			)
       bert_tokenizer.save_pretrained(lowerCamelCase__			)
       UpperCamelCase__					:    Optional[int]          =   CustomTokenizerFast.from_pretrained(lowerCamelCase__			)

      tokenizer.push_to_hub('''test-dynamic-tokenizer'''							,	use_auth_token=self._token			)

      UpperCamelCase__					:    str          =   AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"							,	trust_remote_code=lowerCamelCase__			)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__							,	'''CustomTokenizerFast'''			)
      UpperCamelCase__					:    int          =   AutoTokenizer.from_pretrained(
          F"{USER}/test-dynamic-tokenizer"							,	use_fast=lowerCamelCase__							,	trust_remote_code=lowerCamelCase__			)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__							,	'''CustomTokenizer'''			)



class       __magic_name__	(     unittest.TestCase):


     def      UpperCAmelCase__	(      self							:     str			)  ->      Dict:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   Trie()
      trie.add('''Hello 友達'''			)
      self.assertEqual(trie.data							,	{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}			)
      trie.add('''Hello'''			)
      trie.data
      self.assertEqual(trie.data							,	{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}			)


     def      UpperCAmelCase__	(      self							:     str			)  ->      str:





      '''simple docstring'''






      UpperCamelCase__					:    Dict          =   Trie()
      self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''			)							,	['''[CLS] This is a extra_id_100''']			)
      trie.add('''[CLS]'''			)
      trie.add('''extra_id_1'''			)
      trie.add('''extra_id_100'''			)
      self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''			)							,	['''[CLS]''', ''' This is a ''', '''extra_id_100''']			)


     def      UpperCAmelCase__	(      self							:     int			)  ->      Optional[int]:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   Trie()
      trie.add('''A'''			)
      self.assertEqual(trie.split('''ABC'''			)							,	['''A''', '''BC''']			)
      self.assertEqual(trie.split('''BCA'''			)							,	['''BC''', '''A''']			)


     def      UpperCAmelCase__	(      self							:     Dict			)  ->      Optional[int]:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   Trie()
      trie.add('''TOKEN]'''			)
      trie.add('''[SPECIAL_TOKEN]'''			)
      self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''			)							,	['''This is something ''', '''[SPECIAL_TOKEN]''']			)


     def      UpperCAmelCase__	(      self							:     List[str]			)  ->      Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__					:    List[Any]          =   Trie()
      trie.add('''A'''			)
      trie.add('''P'''			)
      trie.add('''[SPECIAL_TOKEN]'''			)
      self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''			)							,	['''This is something ''', '''[SPECIAL_TOKEN]''']			)


     def      UpperCAmelCase__	(      self							:     Any			)  ->      List[str]:





      '''simple docstring'''






      UpperCamelCase__					:    List[Any]          =   Trie()
      trie.add('''AB'''			)
      trie.add('''B'''			)
      trie.add('''C'''			)
      self.assertEqual(trie.split('''ABC'''			)							,	['''AB''', '''C''']			)


     def      UpperCAmelCase__	(      self							:     Optional[int]			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   Trie()
      trie.add('''ABC'''			)
      trie.add('''B'''			)
      trie.add('''CD'''			)
      self.assertEqual(trie.split('''ABCD'''			)							,	['''ABC''', '''D''']			)


     def      UpperCAmelCase__	(      self							:     Optional[int]			)  ->      Union[str, Any]:





      '''simple docstring'''






      UpperCamelCase__					:    Dict          =   Trie()
      UpperCamelCase__					:    Optional[Any]          =   trie.cut_text('''ABC'''							,	[0, 0, 2, 1, 2, 3]			)
      self.assertEqual(lowerCamelCase__							,	['''AB''', '''C''']			)


