


import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipaConfig,
    BlipaForConditionalGeneration,
    BlipaProcessor,
    BlipaVisionConfig,
    BlipImageProcessor,
    OPTConfig,
    TaConfig,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD

def 			_a  (				):







 """simple docstring"""



 UpperCamelCase__					:    List[Any]          =   '''https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png'''
 UpperCamelCase__					:    Optional[Any]          =   Image.open(requests.get(SCREAMING_SNAKE_CASE    , stream=SCREAMING_SNAKE_CASE     ).raw     ).convert('''RGB'''     )

 return image

def 			_a  (				SCREAMING_SNAKE_CASE	:				Union[str, Any]     ):







 """simple docstring"""



 UpperCamelCase__					:    Optional[Any]          =   []
 # fmt: off

 # vision encoder
 rename_keys.append(('''visual_encoder.cls_token''', '''vision_model.embeddings.class_embedding''')     )
 rename_keys.append(('''visual_encoder.pos_embed''', '''vision_model.embeddings.position_embedding''')     )
 rename_keys.append(('''visual_encoder.patch_embed.proj.weight''', '''vision_model.embeddings.patch_embedding.weight''')     )
 rename_keys.append(('''visual_encoder.patch_embed.proj.bias''', '''vision_model.embeddings.patch_embedding.bias''')     )
 rename_keys.append(('''ln_vision.weight''', '''vision_model.post_layernorm.weight''')     )
 rename_keys.append(('''ln_vision.bias''', '''vision_model.post_layernorm.bias''')     )

 for i in range(config.vision_config.num_hidden_layers     ):
  rename_keys.append((F"visual_encoder.blocks.{i}.norm1.weight", F"vision_model.encoder.layers.{i}.layer_norm1.weight")     )
  rename_keys.append((F"visual_encoder.blocks.{i}.norm1.bias", F"vision_model.encoder.layers.{i}.layer_norm1.bias")     )
  rename_keys.append((F"visual_encoder.blocks.{i}.norm2.weight", F"vision_model.encoder.layers.{i}.layer_norm2.weight")     )
  rename_keys.append((F"visual_encoder.blocks.{i}.norm2.bias", F"vision_model.encoder.layers.{i}.layer_norm2.bias")     )
  rename_keys.append((F"visual_encoder.blocks.{i}.attn.qkv.weight", F"vision_model.encoder.layers.{i}.self_attn.qkv.weight")     )
  rename_keys.append((F"visual_encoder.blocks.{i}.attn.proj.weight", F"vision_model.encoder.layers.{i}.self_attn.projection.weight",)     )
  rename_keys.append((F"visual_encoder.blocks.{i}.attn.proj.bias", F"vision_model.encoder.layers.{i}.self_attn.projection.bias")     )
  rename_keys.append((F"visual_encoder.blocks.{i}.mlp.fc1.weight", F"vision_model.encoder.layers.{i}.mlp.fc1.weight")     )
  rename_keys.append((F"visual_encoder.blocks.{i}.mlp.fc1.bias", F"vision_model.encoder.layers.{i}.mlp.fc1.bias")     )
  rename_keys.append((F"visual_encoder.blocks.{i}.mlp.fc2.weight", F"vision_model.encoder.layers.{i}.mlp.fc2.weight")     )
  rename_keys.append((F"visual_encoder.blocks.{i}.mlp.fc2.bias", F"vision_model.encoder.layers.{i}.mlp.fc2.bias")     )

 # QFormer
 rename_keys.append(('''Qformer.bert.embeddings.LayerNorm.weight''', '''qformer.layernorm.weight''')     )
 rename_keys.append(('''Qformer.bert.embeddings.LayerNorm.bias''', '''qformer.layernorm.bias''')     )

 # fmt: on
 return rename_keys

def 			_a  (				SCREAMING_SNAKE_CASE	:				int    , SCREAMING_SNAKE_CASE	:				List[Any]    , SCREAMING_SNAKE_CASE	:				int     ):







 """simple docstring"""



 UpperCamelCase__					:    Any          =   dct.pop(SCREAMING_SNAKE_CASE     )
 UpperCamelCase__					:    List[Any]          =   val

def 			_a  (				SCREAMING_SNAKE_CASE	:				Dict    , SCREAMING_SNAKE_CASE	:				Any     ):







 """simple docstring"""



 for i in range(config.vision_config.num_hidden_layers     ):
  # read in original q and v biases
  UpperCamelCase__					:    Optional[Any]          =   state_dict.pop(F"visual_encoder.blocks.{i}.attn.q_bias"     )
  UpperCamelCase__					:    List[str]          =   state_dict.pop(F"visual_encoder.blocks.{i}.attn.v_bias"     )

  # next, set bias in the state dict
  UpperCamelCase__					:    Optional[Any]          =   torch.cat((q_bias, torch.zeros_like(SCREAMING_SNAKE_CASE    , requires_grad=SCREAMING_SNAKE_CASE     ), v_bias)     )
  UpperCamelCase__					:    Union[str, Any]          =   qkv_bias

def 			_a  (				SCREAMING_SNAKE_CASE	:				Optional[Any]    , SCREAMING_SNAKE_CASE	:				List[str]     ):







 """simple docstring"""



 UpperCamelCase__					:    Tuple          =   364 if '''coco''' in model_name else 224
 UpperCamelCase__					:    Dict          =   BlipaVisionConfig(image_size=SCREAMING_SNAKE_CASE     ).to_dict()

 # make sure the models have proper bos_token_id and eos_token_id set (important for generation)
 # seems like flan-T5 models don't have bos_token_id properly set?
 if "opt-2.7b" in model_name:
  UpperCamelCase__					:    Dict          =   OPTConfig.from_pretrained('''facebook/opt-2.7b'''    , eos_token_id=SCREAMING_SNAKE_CASE     ).to_dict()
 elif "opt-6.7b" in model_name:
  UpperCamelCase__					:    Optional[int]          =   OPTConfig.from_pretrained('''facebook/opt-6.7b'''    , eos_token_id=SCREAMING_SNAKE_CASE     ).to_dict()
 elif "t5-xl" in model_name:
  UpperCamelCase__					:    Dict          =   TaConfig.from_pretrained('''google/flan-t5-xl'''    , dense_act_fn='''gelu'''    , bos_token_id=1     ).to_dict()
 elif "t5-xxl" in model_name:
  UpperCamelCase__					:    str          =   TaConfig.from_pretrained('''google/flan-t5-xxl'''    , dense_act_fn='''gelu'''    , bos_token_id=1     ).to_dict()

 UpperCamelCase__					:    Dict          =   BlipaConfig(vision_config=SCREAMING_SNAKE_CASE    , text_config=SCREAMING_SNAKE_CASE     )

 return config, image_size

@torch.no_grad()
def 			_a  (				SCREAMING_SNAKE_CASE	:				List[str]    , SCREAMING_SNAKE_CASE	:				Union[str, Any]=None    , SCREAMING_SNAKE_CASE	:				str=False     ):







 """simple docstring"""



 UpperCamelCase__					:    str          =   (
     AutoTokenizer.from_pretrained('''facebook/opt-2.7b'''     )
     if '''opt''' in model_name
     else AutoTokenizer.from_pretrained('''google/flan-t5-xl'''     )
 )
 UpperCamelCase__					:    int          =   tokenizer('''\n'''    , add_special_tokens=SCREAMING_SNAKE_CASE     ).input_ids[0]
 UpperCamelCase__       ,				UpperCamelCase__					:    List[Any]          =   get_blipa_config(SCREAMING_SNAKE_CASE    , eos_token_id=SCREAMING_SNAKE_CASE     )

 UpperCamelCase__					:    List[Any]          =   BlipaForConditionalGeneration(SCREAMING_SNAKE_CASE     ).eval()

 UpperCamelCase__					:    Dict          =   {
     '''blip2-opt-2.7b''': ('''blip2_opt''', '''pretrain_opt2.7b'''),
     '''blip2-opt-6.7b''': ('''blip2_opt''', '''pretrain_opt6.7b'''),
     '''blip2-opt-2.7b-coco''': ('''blip2_opt''', '''caption_coco_opt2.7b'''),
     '''blip2-opt-6.7b-coco''': ('''blip2_opt''', '''caption_coco_opt6.7b'''),
     '''blip2-flan-t5-xl''': ('''blip2_t5''', '''pretrain_flant5xl'''),
     '''blip2-flan-t5-xl-coco''': ('''blip2_t5''', '''caption_coco_flant5xl'''),
     '''blip2-flan-t5-xxl''': ('''blip2_t5''', '''pretrain_flant5xxl'''),
 }

 UpperCamelCase__       ,				UpperCamelCase__					:    int          =   model_name_to_original[model_name]

 # load original model
 print('''Loading original model...'''     )
 UpperCamelCase__					:    List[str]          =   '''cuda''' if torch.cuda.is_available() else '''cpu'''
 UpperCamelCase__       ,				UpperCamelCase__       ,				UpperCamelCase__					:    List[Any]          =   load_model_and_preprocess(
     name=SCREAMING_SNAKE_CASE    , model_type=SCREAMING_SNAKE_CASE    , is_eval=SCREAMING_SNAKE_CASE    , device=SCREAMING_SNAKE_CASE     )
 original_model.eval()
 print('''Done!'''     )

 # update state dict keys
 UpperCamelCase__					:    List[Any]          =   original_model.state_dict()
 UpperCamelCase__					:    List[str]          =   create_rename_keys(SCREAMING_SNAKE_CASE     )
 for src, dest in rename_keys:
  rename_key(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE     )

 # some keys can be renamed efficiently
 for key, val in state_dict.copy().items():
  UpperCamelCase__					:    Optional[Any]          =   state_dict.pop(SCREAMING_SNAKE_CASE     )
  if key.startswith('''Qformer.bert'''     ):
   UpperCamelCase__					:    Union[str, Any]          =   key.replace('''Qformer.bert'''    , '''qformer'''     )
  if "attention.self" in key:
   UpperCamelCase__					:    str          =   key.replace('''self'''    , '''attention'''     )
  if "opt_proj" in key:
   UpperCamelCase__					:    Union[str, Any]          =   key.replace('''opt_proj'''    , '''language_projection'''     )
  if "t5_proj" in key:
   UpperCamelCase__					:    Dict          =   key.replace('''t5_proj'''    , '''language_projection'''     )
  if key.startswith('''opt'''     ):
   UpperCamelCase__					:    str          =   key.replace('''opt'''    , '''language'''     )
  if key.startswith('''t5'''     ):
   UpperCamelCase__					:    Optional[Any]          =   key.replace('''t5'''    , '''language'''     )
  UpperCamelCase__					:    List[Any]          =   val

 # read in qv biases
 read_in_q_v_bias(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE     )

 UpperCamelCase__       ,				UpperCamelCase__					:    Any          =   hf_model.load_state_dict(SCREAMING_SNAKE_CASE    , strict=SCREAMING_SNAKE_CASE     )
 assert len(SCREAMING_SNAKE_CASE     ) == 0
 assert unexpected_keys == ["qformer.embeddings.position_ids"]

 UpperCamelCase__					:    str          =   load_demo_image()
 UpperCamelCase__					:    Any          =   vis_processors['''eval'''](SCREAMING_SNAKE_CASE     ).unsqueeze(0     ).to(SCREAMING_SNAKE_CASE     )
 UpperCamelCase__					:    Tuple          =   tokenizer(['''\n''']    , return_tensors='''pt'''     ).input_ids.to(SCREAMING_SNAKE_CASE     )

 # create processor
 UpperCamelCase__					:    Dict          =   BlipImageProcessor(
     size={'''height''': image_size, '''width''': image_size}    , image_mean=SCREAMING_SNAKE_CASE    , image_std=SCREAMING_SNAKE_CASE     )
 UpperCamelCase__					:    str          =   BlipaProcessor(image_processor=SCREAMING_SNAKE_CASE    , tokenizer=SCREAMING_SNAKE_CASE     )
 UpperCamelCase__					:    List[str]          =   processor(images=SCREAMING_SNAKE_CASE    , return_tensors='''pt'''     ).pixel_values.to(SCREAMING_SNAKE_CASE     )

 # make sure processor creates exact same pixel values
 assert torch.allclose(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE     )

 original_model.to(SCREAMING_SNAKE_CASE     )
 hf_model.to(SCREAMING_SNAKE_CASE     )
 with torch.no_grad():
  if "opt" in model_name:
   UpperCamelCase__					:    Dict          =   original_model({'''image''': original_pixel_values, '''text_input''': ['''''']}     ).logits
   UpperCamelCase__					:    Any          =   hf_model(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE     ).logits
  else:
   UpperCamelCase__					:    List[Any]          =   original_model(
       {'''image''': original_pixel_values, '''text_input''': ['''\n'''], '''text_output''': ['''\n''']}     ).logits
   UpperCamelCase__					:    str          =   input_ids.masked_fill(input_ids == tokenizer.pad_token_id    , -100     )
   UpperCamelCase__					:    List[Any]          =   hf_model(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE    , labels=SCREAMING_SNAKE_CASE     ).logits

 assert original_logits.shape == logits.shape
 print('''First values of original logits:'''    , original_logits[0, :3, :3]     )
 print('''First values of HF logits:'''    , logits[0, :3, :3]     )

 # assert values
 if model_name == "blip2-flan-t5-xl":
  UpperCamelCase__					:    List[str]          =   torch.tensor(
      [[-41.5850, -4.4440, -8.9922], [-47.4322, -5.9143, -1.7340]]    , device=SCREAMING_SNAKE_CASE     )
  assert torch.allclose(logits[0, :3, :3]    , SCREAMING_SNAKE_CASE    , atol=1E-4     )
 elif model_name == "blip2-flan-t5-xl-coco":
  UpperCamelCase__					:    Union[str, Any]          =   torch.tensor(
      [[-57.0109, -9.8967, -12.6280], [-68.6578, -12.7191, -10.5065]]    , device=SCREAMING_SNAKE_CASE     )
 else:
  # cast to same type
  UpperCamelCase__					:    List[str]          =   logits.dtype
  assert torch.allclose(original_logits.to(SCREAMING_SNAKE_CASE     )    , SCREAMING_SNAKE_CASE    , atol=1E-2     )
 print('''Looks ok!'''     )

 print('''Generating a caption...'''     )
 UpperCamelCase__					:    List[str]          =   ''''''
 UpperCamelCase__					:    Union[str, Any]          =   tokenizer(SCREAMING_SNAKE_CASE    , return_tensors='''pt'''     ).input_ids.to(SCREAMING_SNAKE_CASE     )

 UpperCamelCase__					:    Optional[int]          =   original_model.generate({'''image''': original_pixel_values}     )
 UpperCamelCase__					:    Dict          =   hf_model.generate(
     SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE    , do_sample=SCREAMING_SNAKE_CASE    , num_beams=5    , max_length=30    , min_length=1    , top_p=0.9    , repetition_penalty=1.0    , length_penalty=1.0    , temperature=1    , )
 print('''Original generation:'''    , SCREAMING_SNAKE_CASE     )
 UpperCamelCase__					:    Optional[Any]          =   input_ids.shape[1]
 UpperCamelCase__					:    int          =   processor.batch_decode(outputs[:, prompt_length:]    , skip_special_tokens=SCREAMING_SNAKE_CASE     )
 UpperCamelCase__					:    Optional[Any]          =   [text.strip() for text in output_text]
 print('''HF generation:'''    , SCREAMING_SNAKE_CASE     )

 if pytorch_dump_folder_path is not None:
  processor.save_pretrained(SCREAMING_SNAKE_CASE     )
  hf_model.save_pretrained(SCREAMING_SNAKE_CASE     )

 if push_to_hub:
  processor.push_to_hub(F"nielsr/{model_name}"     )
  hf_model.push_to_hub(F"nielsr/{model_name}"     )


if __name__ == "__main__":
       __UpperCamelCase       :       str						       =				argparse.ArgumentParser()
       __UpperCamelCase       :       List[str]						       =				[
           "blip2-opt-2.7b",
           "blip2-opt-6.7b",
           "blip2-opt-2.7b-coco",
           "blip2-opt-6.7b-coco",
           "blip2-flan-t5-xl",
           "blip2-flan-t5-xl-coco",
           "blip2-flan-t5-xxl",
       ]
       parser.add_argument(
           "--model_name",
           default="blip2-opt-2.7b",
           choices=choices,
           type=str,
           help="Path to hf config.json of model to convert",
       )
       parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.")
       parser.add_argument(
           "--push_to_hub",
           action="store_true",
           help="Whether to push the model and processor to the hub after converting",
       )

       __UpperCamelCase       :       Tuple						       =				parser.parse_args()

       convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)






import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
       from test_module.custom_tokenization_fast import CustomTokenizerFast

class       __magic_name__	(     unittest.TestCase):


     def      UpperCAmelCase__	(      self							:     Union[str, Any]			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   mock.Mock()
      UpperCamelCase__					:    Tuple          =   500
      UpperCamelCase__					:    Dict          =   {}
      UpperCamelCase__					:    Dict          =   HTTPError
      UpperCamelCase__					:    Optional[Any]          =   {}

      # Download this model to make sure it's in the cache.
      UpperCamelCase__					:    List[Any]          =   BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''			)

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('''requests.Session.request'''							,	return_value=lowerCamelCase__			) as mock_head:
       UpperCamelCase__					:    Any          =   BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''			)
       # This check we did call the fake head request
       mock_head.assert_called()


     @require_tokenizers
     def      UpperCAmelCase__	(      self							:     str			)  ->      Any:





      '''simple docstring'''






      UpperCamelCase__					:    Union[str, Any]          =   mock.Mock()
      UpperCamelCase__					:    Any          =   500
      UpperCamelCase__					:    str          =   {}
      UpperCamelCase__					:    Optional[Any]          =   HTTPError
      UpperCamelCase__					:    Tuple          =   {}

      # Download this model to make sure it's in the cache.
      UpperCamelCase__					:    Optional[Any]          =   GPTaTokenizerFast.from_pretrained('''gpt2'''			)

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('''requests.Session.request'''							,	return_value=lowerCamelCase__			) as mock_head:
       UpperCamelCase__					:    str          =   GPTaTokenizerFast.from_pretrained('''gpt2'''			)
       # This check we did call the fake head request
       mock_head.assert_called()


     def      UpperCAmelCase__	(      self							:     Any			)  ->      List[str]:





      '''simple docstring'''






      try:
       UpperCamelCase__					:    Dict          =   tempfile.mktemp()
       with open(lowerCamelCase__							,	'''wb'''			) as f:
        http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''							,	lowerCamelCase__			)

       UpperCamelCase__					:    Dict          =   AlbertTokenizer.from_pretrained(lowerCamelCase__			)
      finally:
       os.remove(lowerCamelCase__			)

      # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
      # the current folder and have the right name.
      if os.path.isfile('''tokenizer.json'''			):
       # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
       return
      try:
       with open('''tokenizer.json'''							,	'''wb'''			) as f:
        http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''							,	lowerCamelCase__			)
       UpperCamelCase__					:    Union[str, Any]          =   AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''			)
       # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
       self.assertEqual(tokenizer.vocab_size							,	1000			)
       # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

      finally:
       os.remove('''tokenizer.json'''			)


     def      UpperCAmelCase__	(      self							:     str			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''			)



@is_staging_test
class       __magic_name__	(     unittest.TestCase):
     A:		List[Any]			=      ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]


     @classmethod
     def      UpperCAmelCase__	(      cls							:     List[str]			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    List[str]          =   TOKEN
      HfFolder.save_token(lowerCamelCase__			)


     @classmethod
     def      UpperCAmelCase__	(      cls							:     Any			)  ->      Optional[Any]:





      '''simple docstring'''






      try:
       delete_repo(token=cls._token							,	repo_id='''test-tokenizer'''			)
      except HTTPError:
       pass

      try:
       delete_repo(token=cls._token							,	repo_id='''valid_org/test-tokenizer-org'''			)
      except HTTPError:
       pass

      try:
       delete_repo(token=cls._token							,	repo_id='''test-dynamic-tokenizer'''			)
      except HTTPError:
       pass


     def      UpperCAmelCase__	(      self							:     Union[str, Any]			)  ->      Optional[int]:





      '''simple docstring'''






      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    int          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)
       UpperCamelCase__					:    Tuple          =   BertTokenizer(lowerCamelCase__			)

      tokenizer.push_to_hub('''test-tokenizer'''							,	use_auth_token=self._token			)
      UpperCamelCase__					:    Union[str, Any]          =   BertTokenizer.from_pretrained(F"{USER}/test-tokenizer"			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)

      # Reset repo
      delete_repo(token=self._token							,	repo_id='''test-tokenizer'''			)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(lowerCamelCase__							,	repo_id='''test-tokenizer'''							,	push_to_hub=lowerCamelCase__							,	use_auth_token=self._token			)

      UpperCamelCase__					:    str          =   BertTokenizer.from_pretrained(F"{USER}/test-tokenizer"			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)


     def      UpperCAmelCase__	(      self							:     str			)  ->      Optional[int]:





      '''simple docstring'''






      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    Dict          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)
       UpperCamelCase__					:    List[str]          =   BertTokenizer(lowerCamelCase__			)

      tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''							,	use_auth_token=self._token			)
      UpperCamelCase__					:    Optional[Any]          =   BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)

      # Reset repo
      delete_repo(token=self._token							,	repo_id='''valid_org/test-tokenizer-org'''			)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(
           lowerCamelCase__							,	repo_id='''valid_org/test-tokenizer-org'''							,	push_to_hub=lowerCamelCase__							,	use_auth_token=self._token			)

      UpperCamelCase__					:    Union[str, Any]          =   BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)


     @require_tokenizers
     def      UpperCAmelCase__	(      self							:     List[str]			)  ->      int:





      '''simple docstring'''






      CustomTokenizer.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    List[Any]          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)
       UpperCamelCase__					:    Any          =   CustomTokenizer(lowerCamelCase__			)

      # No fast custom tokenizer
      tokenizer.push_to_hub('''test-dynamic-tokenizer'''							,	use_auth_token=self._token			)

      UpperCamelCase__					:    str          =   AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"							,	trust_remote_code=lowerCamelCase__			)
      # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__							,	'''CustomTokenizer'''			)

      # Fast and slow custom tokenizer
      CustomTokenizerFast.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    List[Any]          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)

       UpperCamelCase__					:    List[Any]          =   BertTokenizerFast.from_pretrained(lowerCamelCase__			)
       bert_tokenizer.save_pretrained(lowerCamelCase__			)
       UpperCamelCase__					:    Optional[int]          =   CustomTokenizerFast.from_pretrained(lowerCamelCase__			)

      tokenizer.push_to_hub('''test-dynamic-tokenizer'''							,	use_auth_token=self._token			)

      UpperCamelCase__					:    str          =   AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"							,	trust_remote_code=lowerCamelCase__			)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__							,	'''CustomTokenizerFast'''			)
      UpperCamelCase__					:    int          =   AutoTokenizer.from_pretrained(
          F"{USER}/test-dynamic-tokenizer"							,	use_fast=lowerCamelCase__							,	trust_remote_code=lowerCamelCase__			)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__							,	'''CustomTokenizer'''			)



class       __magic_name__	(     unittest.TestCase):


     def      UpperCAmelCase__	(      self							:     str			)  ->      Dict:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   Trie()
      trie.add('''Hello 友達'''			)
      self.assertEqual(trie.data							,	{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}			)
      trie.add('''Hello'''			)
      trie.data
      self.assertEqual(trie.data							,	{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}			)


     def      UpperCAmelCase__	(      self							:     str			)  ->      str:





      '''simple docstring'''






      UpperCamelCase__					:    Dict          =   Trie()
      self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''			)							,	['''[CLS] This is a extra_id_100''']			)
      trie.add('''[CLS]'''			)
      trie.add('''extra_id_1'''			)
      trie.add('''extra_id_100'''			)
      self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''			)							,	['''[CLS]''', ''' This is a ''', '''extra_id_100''']			)


     def      UpperCAmelCase__	(      self							:     int			)  ->      Optional[int]:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   Trie()
      trie.add('''A'''			)
      self.assertEqual(trie.split('''ABC'''			)							,	['''A''', '''BC''']			)
      self.assertEqual(trie.split('''BCA'''			)							,	['''BC''', '''A''']			)


     def      UpperCAmelCase__	(      self							:     Dict			)  ->      Optional[int]:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   Trie()
      trie.add('''TOKEN]'''			)
      trie.add('''[SPECIAL_TOKEN]'''			)
      self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''			)							,	['''This is something ''', '''[SPECIAL_TOKEN]''']			)


     def      UpperCAmelCase__	(      self							:     List[str]			)  ->      Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__					:    List[Any]          =   Trie()
      trie.add('''A'''			)
      trie.add('''P'''			)
      trie.add('''[SPECIAL_TOKEN]'''			)
      self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''			)							,	['''This is something ''', '''[SPECIAL_TOKEN]''']			)


     def      UpperCAmelCase__	(      self							:     Any			)  ->      List[str]:





      '''simple docstring'''






      UpperCamelCase__					:    List[Any]          =   Trie()
      trie.add('''AB'''			)
      trie.add('''B'''			)
      trie.add('''C'''			)
      self.assertEqual(trie.split('''ABC'''			)							,	['''AB''', '''C''']			)


     def      UpperCAmelCase__	(      self							:     Optional[int]			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   Trie()
      trie.add('''ABC'''			)
      trie.add('''B'''			)
      trie.add('''CD'''			)
      self.assertEqual(trie.split('''ABCD'''			)							,	['''ABC''', '''D''']			)


     def      UpperCAmelCase__	(      self							:     Optional[int]			)  ->      Union[str, Any]:





      '''simple docstring'''






      UpperCamelCase__					:    Dict          =   Trie()
      UpperCamelCase__					:    Optional[Any]          =   trie.cut_text('''ABC'''							,	[0, 0, 2, 1, 2, 3]			)
      self.assertEqual(lowerCamelCase__							,	['''AB''', '''C''']			)


