


import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import (
    BertTokenizer,
    ViltConfig,
    ViltForImageAndTextRetrieval,
    ViltForImagesAndTextClassification,
    ViltForMaskedLM,
    ViltForQuestionAnswering,
    ViltImageProcessor,
    ViltProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
__UpperCamelCase       :       int						       =				logging.get_logger(__name__)

def 			_a  (				SCREAMING_SNAKE_CASE	:				str    , SCREAMING_SNAKE_CASE	:				Tuple=False    , SCREAMING_SNAKE_CASE	:				str=False    , SCREAMING_SNAKE_CASE	:				Optional[int]=False     ):







 """simple docstring"""



 UpperCamelCase__					:    Any          =   []
 for i in range(config.num_hidden_layers     ):
  # encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
  rename_keys.append((F"transformer.blocks.{i}.norm1.weight", F"vilt.encoder.layer.{i}.layernorm_before.weight")     )
  rename_keys.append((F"transformer.blocks.{i}.norm1.bias", F"vilt.encoder.layer.{i}.layernorm_before.bias")     )
  rename_keys.append(
      (F"transformer.blocks.{i}.attn.proj.weight", F"vilt.encoder.layer.{i}.attention.output.dense.weight")     )
  rename_keys.append(
      (F"transformer.blocks.{i}.attn.proj.bias", F"vilt.encoder.layer.{i}.attention.output.dense.bias")     )
  rename_keys.append((F"transformer.blocks.{i}.norm2.weight", F"vilt.encoder.layer.{i}.layernorm_after.weight")     )
  rename_keys.append((F"transformer.blocks.{i}.norm2.bias", F"vilt.encoder.layer.{i}.layernorm_after.bias")     )
  rename_keys.append(
      (F"transformer.blocks.{i}.mlp.fc1.weight", F"vilt.encoder.layer.{i}.intermediate.dense.weight")     )
  rename_keys.append((F"transformer.blocks.{i}.mlp.fc1.bias", F"vilt.encoder.layer.{i}.intermediate.dense.bias")     )
  rename_keys.append((F"transformer.blocks.{i}.mlp.fc2.weight", F"vilt.encoder.layer.{i}.output.dense.weight")     )
  rename_keys.append((F"transformer.blocks.{i}.mlp.fc2.bias", F"vilt.encoder.layer.{i}.output.dense.bias")     )

 # embeddings
 rename_keys.extend(
     [
         # text embeddings
         ('''text_embeddings.word_embeddings.weight''', '''vilt.embeddings.text_embeddings.word_embeddings.weight'''),
         (
             '''text_embeddings.position_embeddings.weight''',
             '''vilt.embeddings.text_embeddings.position_embeddings.weight''',
         ),
         ('''text_embeddings.position_ids''', '''vilt.embeddings.text_embeddings.position_ids'''),
         (
             '''text_embeddings.token_type_embeddings.weight''',
             '''vilt.embeddings.text_embeddings.token_type_embeddings.weight''',
         ),
         ('''text_embeddings.LayerNorm.weight''', '''vilt.embeddings.text_embeddings.LayerNorm.weight'''),
         ('''text_embeddings.LayerNorm.bias''', '''vilt.embeddings.text_embeddings.LayerNorm.bias'''),
         # patch embeddings
         ('''transformer.cls_token''', '''vilt.embeddings.cls_token'''),
         ('''transformer.patch_embed.proj.weight''', '''vilt.embeddings.patch_embeddings.projection.weight'''),
         ('''transformer.patch_embed.proj.bias''', '''vilt.embeddings.patch_embeddings.projection.bias'''),
         ('''transformer.pos_embed''', '''vilt.embeddings.position_embeddings'''),
         # token type embeddings
         ('''token_type_embeddings.weight''', '''vilt.embeddings.token_type_embeddings.weight'''),
     ]     )

 # final layernorm + pooler
 rename_keys.extend(
     [
         ('''transformer.norm.weight''', '''vilt.layernorm.weight'''),
         ('''transformer.norm.bias''', '''vilt.layernorm.bias'''),
         ('''pooler.dense.weight''', '''vilt.pooler.dense.weight'''),
         ('''pooler.dense.bias''', '''vilt.pooler.dense.bias'''),
     ]     )

 # classifier head(s)
 if vqa_model:
  # classification head
  rename_keys.extend(
      [
          ('''vqa_classifier.0.weight''', '''classifier.0.weight'''),
          ('''vqa_classifier.0.bias''', '''classifier.0.bias'''),
          ('''vqa_classifier.1.weight''', '''classifier.1.weight'''),
          ('''vqa_classifier.1.bias''', '''classifier.1.bias'''),
          ('''vqa_classifier.3.weight''', '''classifier.3.weight'''),
          ('''vqa_classifier.3.bias''', '''classifier.3.bias'''),
      ]     )
 elif nlvr_model:
  # classification head
  rename_keys.extend(
      [
          ('''nlvr2_classifier.0.weight''', '''classifier.0.weight'''),
          ('''nlvr2_classifier.0.bias''', '''classifier.0.bias'''),
          ('''nlvr2_classifier.1.weight''', '''classifier.1.weight'''),
          ('''nlvr2_classifier.1.bias''', '''classifier.1.bias'''),
          ('''nlvr2_classifier.3.weight''', '''classifier.3.weight'''),
          ('''nlvr2_classifier.3.bias''', '''classifier.3.bias'''),
      ]     )
 else:
  pass

 return rename_keys

def 			_a  (				SCREAMING_SNAKE_CASE	:				Dict    , SCREAMING_SNAKE_CASE	:				Optional[Any]     ):







 """simple docstring"""



 for i in range(config.num_hidden_layers     ):
  UpperCamelCase__					:    Optional[Any]          =   '''vilt.'''
  # read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
  UpperCamelCase__					:    Tuple          =   state_dict.pop(F"transformer.blocks.{i}.attn.qkv.weight"     )
  UpperCamelCase__					:    List[Any]          =   state_dict.pop(F"transformer.blocks.{i}.attn.qkv.bias"     )
  # next, add query, keys and values (in that order) to the state dict
  UpperCamelCase__					:    List[str]          =   in_proj_weight[
      : config.hidden_size, :
  ]
  UpperCamelCase__					:    Tuple          =   in_proj_bias[: config.hidden_size]
  UpperCamelCase__					:    List[str]          =   in_proj_weight[
      config.hidden_size : config.hidden_size * 2, :
  ]
  UpperCamelCase__					:    List[Any]          =   in_proj_bias[
      config.hidden_size : config.hidden_size * 2
  ]
  UpperCamelCase__					:    List[Any]          =   in_proj_weight[
      -config.hidden_size :, :
  ]
  UpperCamelCase__					:    Optional[Any]          =   in_proj_bias[-config.hidden_size :]

def 			_a  (				SCREAMING_SNAKE_CASE	:				str     ):







 """simple docstring"""



 UpperCamelCase__					:    Any          =   ['''head.weight''', '''head.bias''']
 for k in ignore_keys:
  state_dict.pop(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE     )

def 			_a  (				SCREAMING_SNAKE_CASE	:				Tuple    , SCREAMING_SNAKE_CASE	:				Tuple    , SCREAMING_SNAKE_CASE	:				Tuple     ):







 """simple docstring"""



 UpperCamelCase__					:    int          =   dct.pop(SCREAMING_SNAKE_CASE     )
 UpperCamelCase__					:    int          =   val

@torch.no_grad()
def 			_a  (				SCREAMING_SNAKE_CASE	:				List[Any]    , SCREAMING_SNAKE_CASE	:				List[Any]     ):







 """simple docstring"""



 UpperCamelCase__					:    List[Any]          =   ViltConfig(image_size=384    , patch_size=32    , tie_word_embeddings=SCREAMING_SNAKE_CASE     )
 UpperCamelCase__					:    List[str]          =   False
 UpperCamelCase__					:    Dict          =   False
 UpperCamelCase__					:    str          =   False
 UpperCamelCase__					:    Tuple          =   False
 if "vqa" in checkpoint_url:
  UpperCamelCase__					:    Tuple          =   True
  UpperCamelCase__					:    Any          =   3129
  UpperCamelCase__					:    str          =   '''huggingface/label-files'''
  UpperCamelCase__					:    Union[str, Any]          =   '''vqa2-id2label.json'''
  UpperCamelCase__					:    List[str]          =   json.load(open(hf_hub_download(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE    , repo_type='''dataset'''     )    , '''r'''     )     )
  UpperCamelCase__					:    int          =   {int(SCREAMING_SNAKE_CASE     ): v for k, v in idalabel.items()}
  UpperCamelCase__					:    Any          =   idalabel
  UpperCamelCase__					:    List[Any]          =   {v: k for k, v in idalabel.items()}
  UpperCamelCase__					:    Any          =   ViltForQuestionAnswering(SCREAMING_SNAKE_CASE     )
 elif "nlvr" in checkpoint_url:
  UpperCamelCase__					:    int          =   True
  UpperCamelCase__					:    str          =   2
  UpperCamelCase__					:    List[Any]          =   {0: '''False''', 1: '''True'''}
  UpperCamelCase__					:    str          =   {v: k for k, v in config.idalabel.items()}
  UpperCamelCase__					:    Optional[int]          =   3
  UpperCamelCase__					:    int          =   ViltForImagesAndTextClassification(SCREAMING_SNAKE_CASE     )
 elif "irtr" in checkpoint_url:
  UpperCamelCase__					:    Tuple          =   True
  UpperCamelCase__					:    Any          =   ViltForImageAndTextRetrieval(SCREAMING_SNAKE_CASE     )
 elif "mlm_itm" in checkpoint_url:
  UpperCamelCase__					:    List[str]          =   True
  UpperCamelCase__					:    Optional[Any]          =   ViltForMaskedLM(SCREAMING_SNAKE_CASE     )
 else:
  raise ValueError('''Unknown model type'''     )

 # load state_dict of original model, remove and rename some keys
 UpperCamelCase__					:    Optional[int]          =   torch.hub.load_state_dict_from_url(SCREAMING_SNAKE_CASE    , map_location='''cpu'''     )['''state_dict''']
 UpperCamelCase__					:    Dict          =   create_rename_keys(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE     )
 for src, dest in rename_keys:
  rename_key(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE     )
 read_in_q_k_v(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE     )
 if mlm_model or irtr_model:
  UpperCamelCase__					:    Dict          =   ['''itm_score.fc.weight''', '''itm_score.fc.bias''']
  for k in ignore_keys:
   state_dict.pop(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE     )

    # load state dict into HuggingFace model
 model.eval()
 if mlm_model:
  UpperCamelCase__       ,				UpperCamelCase__					:    Tuple          =   model.load_state_dict(SCREAMING_SNAKE_CASE    , strict=SCREAMING_SNAKE_CASE     )
  assert missing_keys == ["mlm_score.decoder.bias"]
 else:
  model.load_state_dict(SCREAMING_SNAKE_CASE     )

 # Define processor
 UpperCamelCase__					:    Any          =   ViltImageProcessor(size=384     )
 UpperCamelCase__					:    Dict          =   BertTokenizer.from_pretrained('''bert-base-uncased'''     )
 UpperCamelCase__					:    Optional[Any]          =   ViltProcessor(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE     )

 # Forward pass on example inputs (image + text)
 if nlvr_model:
  UpperCamelCase__					:    int          =   Image.open(requests.get('''https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg'''    , stream=SCREAMING_SNAKE_CASE     ).raw     )
  UpperCamelCase__					:    List[Any]          =   Image.open(requests.get('''https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg'''    , stream=SCREAMING_SNAKE_CASE     ).raw     )
  UpperCamelCase__					:    int          =   (
      '''The left image contains twice the number of dogs as the right image, and at least two dogs in total are'''
      ''' standing.'''
  )
  UpperCamelCase__					:    str          =   processor(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE    , return_tensors='''pt'''     )
  UpperCamelCase__					:    Tuple          =   processor(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE    , return_tensors='''pt'''     )
  UpperCamelCase__					:    str          =   model(
      input_ids=encoding_a.input_ids    , pixel_values=encoding_a.pixel_values    , pixel_values_a=encoding_a.pixel_values    , )
 else:
  UpperCamelCase__					:    Optional[Any]          =   Image.open(requests.get('''http://images.cocodataset.org/val2017/000000039769.jpg'''    , stream=SCREAMING_SNAKE_CASE     ).raw     )
  if mlm_model:
   UpperCamelCase__					:    List[Any]          =   '''a bunch of [MASK] laying on a [MASK].'''
  else:
   UpperCamelCase__					:    Dict          =   '''How many cats are there?'''
  UpperCamelCase__					:    Optional[Any]          =   processor(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE    , return_tensors='''pt'''     )
  UpperCamelCase__					:    Dict          =   model(**SCREAMING_SNAKE_CASE     )

 # Verify outputs
 if mlm_model:
  UpperCamelCase__					:    Optional[Any]          =   torch.Size([1, 11, 30522]     )
  UpperCamelCase__					:    int          =   torch.tensor([-12.5061, -12.5123, -12.5174]     )
  assert outputs.logits.shape == expected_shape
  assert torch.allclose(outputs.logits[0, 0, :3]    , SCREAMING_SNAKE_CASE    , atol=1E-4     )

  # verify masked token prediction equals "cats"
  UpperCamelCase__					:    int          =   outputs.logits[0, 4, :].argmax(-1     ).item()
  assert tokenizer.decode([predicted_id]     ) == "cats"
 elif vqa_model:
  UpperCamelCase__					:    List[str]          =   torch.Size([1, 3129]     )
  UpperCamelCase__					:    Optional[Any]          =   torch.tensor([-15.9495, -18.1472, -10.3041]     )
  assert torch.allclose(outputs.logits[0, :3]    , SCREAMING_SNAKE_CASE    , atol=1E-4     )
  assert outputs.logits.shape == expected_shape
  assert torch.allclose(outputs.logits[0, 0, :3]    , SCREAMING_SNAKE_CASE    , atol=1E-4     )

  # verify vqa prediction equals "2"
  UpperCamelCase__					:    int          =   outputs.logits.argmax(-1     ).item()
  assert model.config.idalabel[predicted_idx] == "2"
 elif nlvr_model:
  UpperCamelCase__					:    Optional[Any]          =   torch.Size([1, 2]     )
  UpperCamelCase__					:    str          =   torch.tensor([-2.8721, 2.1291]     )
  assert torch.allclose(outputs.logits[0, :3]    , SCREAMING_SNAKE_CASE    , atol=1E-4     )
  assert outputs.logits.shape == expected_shape

 Path(SCREAMING_SNAKE_CASE     ).mkdir(exist_ok=SCREAMING_SNAKE_CASE     )
 print(F"Saving model and processor to {pytorch_dump_folder_path}"     )
 model.save_pretrained(SCREAMING_SNAKE_CASE     )
 processor.save_pretrained(SCREAMING_SNAKE_CASE     )


if __name__ == "__main__":
       __UpperCamelCase       :       Optional[int]						       =				argparse.ArgumentParser()
       # Required parameters
       parser.add_argument(
           "--checkpoint_url",
           default="https://github.com/dandelin/ViLT/releases/download/200k/vilt_200k_mlm_itm.ckpt",
           type=str,
           help="URL of the checkpoint you\'d like to convert.",
       )
       parser.add_argument(
           "--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model directory."
       )

       __UpperCamelCase       :       Union[str, Any]						       =				parser.parse_args()
       convert_vilt_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)






import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
       from test_module.custom_tokenization_fast import CustomTokenizerFast

class       __magic_name__	(     unittest.TestCase):


     def      UpperCAmelCase__	(      self							:     Union[str, Any]			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   mock.Mock()
      UpperCamelCase__					:    Tuple          =   500
      UpperCamelCase__					:    Dict          =   {}
      UpperCamelCase__					:    Dict          =   HTTPError
      UpperCamelCase__					:    Optional[Any]          =   {}

      # Download this model to make sure it's in the cache.
      UpperCamelCase__					:    List[Any]          =   BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''			)

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('''requests.Session.request'''							,	return_value=lowerCamelCase__			) as mock_head:
       UpperCamelCase__					:    Any          =   BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''			)
       # This check we did call the fake head request
       mock_head.assert_called()


     @require_tokenizers
     def      UpperCAmelCase__	(      self							:     str			)  ->      Any:





      '''simple docstring'''






      UpperCamelCase__					:    Union[str, Any]          =   mock.Mock()
      UpperCamelCase__					:    Any          =   500
      UpperCamelCase__					:    str          =   {}
      UpperCamelCase__					:    Optional[Any]          =   HTTPError
      UpperCamelCase__					:    Tuple          =   {}

      # Download this model to make sure it's in the cache.
      UpperCamelCase__					:    Optional[Any]          =   GPTaTokenizerFast.from_pretrained('''gpt2'''			)

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('''requests.Session.request'''							,	return_value=lowerCamelCase__			) as mock_head:
       UpperCamelCase__					:    str          =   GPTaTokenizerFast.from_pretrained('''gpt2'''			)
       # This check we did call the fake head request
       mock_head.assert_called()


     def      UpperCAmelCase__	(      self							:     Any			)  ->      List[str]:





      '''simple docstring'''






      try:
       UpperCamelCase__					:    Dict          =   tempfile.mktemp()
       with open(lowerCamelCase__							,	'''wb'''			) as f:
        http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''							,	lowerCamelCase__			)

       UpperCamelCase__					:    Dict          =   AlbertTokenizer.from_pretrained(lowerCamelCase__			)
      finally:
       os.remove(lowerCamelCase__			)

      # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
      # the current folder and have the right name.
      if os.path.isfile('''tokenizer.json'''			):
       # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
       return
      try:
       with open('''tokenizer.json'''							,	'''wb'''			) as f:
        http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''							,	lowerCamelCase__			)
       UpperCamelCase__					:    Union[str, Any]          =   AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''			)
       # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
       self.assertEqual(tokenizer.vocab_size							,	1000			)
       # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

      finally:
       os.remove('''tokenizer.json'''			)


     def      UpperCAmelCase__	(      self							:     str			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''			)



@is_staging_test
class       __magic_name__	(     unittest.TestCase):
     A:		List[Any]			=      ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]


     @classmethod
     def      UpperCAmelCase__	(      cls							:     List[str]			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    List[str]          =   TOKEN
      HfFolder.save_token(lowerCamelCase__			)


     @classmethod
     def      UpperCAmelCase__	(      cls							:     Any			)  ->      Optional[Any]:





      '''simple docstring'''






      try:
       delete_repo(token=cls._token							,	repo_id='''test-tokenizer'''			)
      except HTTPError:
       pass

      try:
       delete_repo(token=cls._token							,	repo_id='''valid_org/test-tokenizer-org'''			)
      except HTTPError:
       pass

      try:
       delete_repo(token=cls._token							,	repo_id='''test-dynamic-tokenizer'''			)
      except HTTPError:
       pass


     def      UpperCAmelCase__	(      self							:     Union[str, Any]			)  ->      Optional[int]:





      '''simple docstring'''






      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    int          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)
       UpperCamelCase__					:    Tuple          =   BertTokenizer(lowerCamelCase__			)

      tokenizer.push_to_hub('''test-tokenizer'''							,	use_auth_token=self._token			)
      UpperCamelCase__					:    Union[str, Any]          =   BertTokenizer.from_pretrained(F"{USER}/test-tokenizer"			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)

      # Reset repo
      delete_repo(token=self._token							,	repo_id='''test-tokenizer'''			)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(lowerCamelCase__							,	repo_id='''test-tokenizer'''							,	push_to_hub=lowerCamelCase__							,	use_auth_token=self._token			)

      UpperCamelCase__					:    str          =   BertTokenizer.from_pretrained(F"{USER}/test-tokenizer"			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)


     def      UpperCAmelCase__	(      self							:     str			)  ->      Optional[int]:





      '''simple docstring'''






      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    Dict          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)
       UpperCamelCase__					:    List[str]          =   BertTokenizer(lowerCamelCase__			)

      tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''							,	use_auth_token=self._token			)
      UpperCamelCase__					:    Optional[Any]          =   BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)

      # Reset repo
      delete_repo(token=self._token							,	repo_id='''valid_org/test-tokenizer-org'''			)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(
           lowerCamelCase__							,	repo_id='''valid_org/test-tokenizer-org'''							,	push_to_hub=lowerCamelCase__							,	use_auth_token=self._token			)

      UpperCamelCase__					:    Union[str, Any]          =   BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)


     @require_tokenizers
     def      UpperCAmelCase__	(      self							:     List[str]			)  ->      int:





      '''simple docstring'''






      CustomTokenizer.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    List[Any]          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)
       UpperCamelCase__					:    Any          =   CustomTokenizer(lowerCamelCase__			)

      # No fast custom tokenizer
      tokenizer.push_to_hub('''test-dynamic-tokenizer'''							,	use_auth_token=self._token			)

      UpperCamelCase__					:    str          =   AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"							,	trust_remote_code=lowerCamelCase__			)
      # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__							,	'''CustomTokenizer'''			)

      # Fast and slow custom tokenizer
      CustomTokenizerFast.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    List[Any]          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)

       UpperCamelCase__					:    List[Any]          =   BertTokenizerFast.from_pretrained(lowerCamelCase__			)
       bert_tokenizer.save_pretrained(lowerCamelCase__			)
       UpperCamelCase__					:    Optional[int]          =   CustomTokenizerFast.from_pretrained(lowerCamelCase__			)

      tokenizer.push_to_hub('''test-dynamic-tokenizer'''							,	use_auth_token=self._token			)

      UpperCamelCase__					:    str          =   AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"							,	trust_remote_code=lowerCamelCase__			)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__							,	'''CustomTokenizerFast'''			)
      UpperCamelCase__					:    int          =   AutoTokenizer.from_pretrained(
          F"{USER}/test-dynamic-tokenizer"							,	use_fast=lowerCamelCase__							,	trust_remote_code=lowerCamelCase__			)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__							,	'''CustomTokenizer'''			)



class       __magic_name__	(     unittest.TestCase):


     def      UpperCAmelCase__	(      self							:     str			)  ->      Dict:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   Trie()
      trie.add('''Hello 友達'''			)
      self.assertEqual(trie.data							,	{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}			)
      trie.add('''Hello'''			)
      trie.data
      self.assertEqual(trie.data							,	{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}			)


     def      UpperCAmelCase__	(      self							:     str			)  ->      str:





      '''simple docstring'''






      UpperCamelCase__					:    Dict          =   Trie()
      self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''			)							,	['''[CLS] This is a extra_id_100''']			)
      trie.add('''[CLS]'''			)
      trie.add('''extra_id_1'''			)
      trie.add('''extra_id_100'''			)
      self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''			)							,	['''[CLS]''', ''' This is a ''', '''extra_id_100''']			)


     def      UpperCAmelCase__	(      self							:     int			)  ->      Optional[int]:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   Trie()
      trie.add('''A'''			)
      self.assertEqual(trie.split('''ABC'''			)							,	['''A''', '''BC''']			)
      self.assertEqual(trie.split('''BCA'''			)							,	['''BC''', '''A''']			)


     def      UpperCAmelCase__	(      self							:     Dict			)  ->      Optional[int]:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   Trie()
      trie.add('''TOKEN]'''			)
      trie.add('''[SPECIAL_TOKEN]'''			)
      self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''			)							,	['''This is something ''', '''[SPECIAL_TOKEN]''']			)


     def      UpperCAmelCase__	(      self							:     List[str]			)  ->      Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__					:    List[Any]          =   Trie()
      trie.add('''A'''			)
      trie.add('''P'''			)
      trie.add('''[SPECIAL_TOKEN]'''			)
      self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''			)							,	['''This is something ''', '''[SPECIAL_TOKEN]''']			)


     def      UpperCAmelCase__	(      self							:     Any			)  ->      List[str]:





      '''simple docstring'''






      UpperCamelCase__					:    List[Any]          =   Trie()
      trie.add('''AB'''			)
      trie.add('''B'''			)
      trie.add('''C'''			)
      self.assertEqual(trie.split('''ABC'''			)							,	['''AB''', '''C''']			)


     def      UpperCAmelCase__	(      self							:     Optional[int]			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   Trie()
      trie.add('''ABC'''			)
      trie.add('''B'''			)
      trie.add('''CD'''			)
      self.assertEqual(trie.split('''ABCD'''			)							,	['''ABC''', '''D''']			)


     def      UpperCAmelCase__	(      self							:     Optional[int]			)  ->      Union[str, Any]:





      '''simple docstring'''






      UpperCamelCase__					:    Dict          =   Trie()
      UpperCamelCase__					:    Optional[Any]          =   trie.cut_text('''ABC'''							,	[0, 0, 2, 1, 2, 3]			)
      self.assertEqual(lowerCamelCase__							,	['''AB''', '''C''']			)


