


import argparse
import json
import pickle
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import MaskFormerConfig, MaskFormerForInstanceSegmentation, MaskFormerImageProcessor, SwinConfig
from transformers.utils import logging


logging.set_verbosity_info()
__UpperCamelCase       :       List[str]						       =				logging.get_logger(__name__)

def 			_a  (				SCREAMING_SNAKE_CASE	:				str     ):







 """simple docstring"""



 UpperCamelCase__					:    Union[str, Any]          =   SwinConfig.from_pretrained(
     '''microsoft/swin-tiny-patch4-window7-224'''    , out_features=['''stage1''', '''stage2''', '''stage3''', '''stage4''']     )
 UpperCamelCase__					:    str          =   MaskFormerConfig(backbone_config=SCREAMING_SNAKE_CASE     )

 UpperCamelCase__					:    Any          =   '''huggingface/label-files'''
 if "ade20k-full" in model_name:
  # this should be ok
  UpperCamelCase__					:    List[str]          =   847
  UpperCamelCase__					:    str          =   '''maskformer-ade20k-full-id2label.json'''
 elif "ade" in model_name:
  # this should be ok
  UpperCamelCase__					:    Any          =   150
  UpperCamelCase__					:    Union[str, Any]          =   '''ade20k-id2label.json'''
 elif "coco-stuff" in model_name:
  # this should be ok
  UpperCamelCase__					:    List[str]          =   171
  UpperCamelCase__					:    str          =   '''maskformer-coco-stuff-id2label.json'''
 elif "coco" in model_name:
  # TODO
  UpperCamelCase__					:    Tuple          =   133
  UpperCamelCase__					:    List[str]          =   '''coco-panoptic-id2label.json'''
 elif "cityscapes" in model_name:
  # this should be ok
  UpperCamelCase__					:    Dict          =   19
  UpperCamelCase__					:    Tuple          =   '''cityscapes-id2label.json'''
 elif "vistas" in model_name:
  # this should be ok
  UpperCamelCase__					:    List[str]          =   65
  UpperCamelCase__					:    str          =   '''mapillary-vistas-id2label.json'''

 UpperCamelCase__					:    Union[str, Any]          =   json.load(open(hf_hub_download(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE    , repo_type='''dataset'''     )    , '''r'''     )     )
 UpperCamelCase__					:    str          =   {int(SCREAMING_SNAKE_CASE     ): v for k, v in idalabel.items()}

 return config

def 			_a  (				SCREAMING_SNAKE_CASE	:				int     ):







 """simple docstring"""



 UpperCamelCase__					:    str          =   []
 # stem
 # fmt: off
 rename_keys.append(('''backbone.patch_embed.proj.weight''', '''model.pixel_level_module.encoder.model.embeddings.patch_embeddings.projection.weight''')     )
 rename_keys.append(('''backbone.patch_embed.proj.bias''', '''model.pixel_level_module.encoder.model.embeddings.patch_embeddings.projection.bias''')     )
 rename_keys.append(('''backbone.patch_embed.norm.weight''', '''model.pixel_level_module.encoder.model.embeddings.norm.weight''')     )
 rename_keys.append(('''backbone.patch_embed.norm.bias''', '''model.pixel_level_module.encoder.model.embeddings.norm.bias''')     )
 # stages
 for i in range(len(config.backbone_config.depths     )     ):
  for j in range(config.backbone_config.depths[i]     ):
   rename_keys.append((F"backbone.layers.{i}.blocks.{j}.norm1.weight", F"model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.layernorm_before.weight")     )
   rename_keys.append((F"backbone.layers.{i}.blocks.{j}.norm1.bias", F"model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.layernorm_before.bias")     )
   rename_keys.append((F"backbone.layers.{i}.blocks.{j}.attn.relative_position_bias_table", F"model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.attention.self.relative_position_bias_table")     )
   rename_keys.append((F"backbone.layers.{i}.blocks.{j}.attn.relative_position_index", F"model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.attention.self.relative_position_index")     )
   rename_keys.append((F"backbone.layers.{i}.blocks.{j}.attn.proj.weight", F"model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.attention.output.dense.weight")     )
   rename_keys.append((F"backbone.layers.{i}.blocks.{j}.attn.proj.bias", F"model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.attention.output.dense.bias")     )
   rename_keys.append((F"backbone.layers.{i}.blocks.{j}.norm2.weight", F"model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.layernorm_after.weight")     )
   rename_keys.append((F"backbone.layers.{i}.blocks.{j}.norm2.bias", F"model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.layernorm_after.bias")     )
   rename_keys.append((F"backbone.layers.{i}.blocks.{j}.mlp.fc1.weight", F"model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.intermediate.dense.weight")     )
   rename_keys.append((F"backbone.layers.{i}.blocks.{j}.mlp.fc1.bias", F"model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.intermediate.dense.bias")     )
   rename_keys.append((F"backbone.layers.{i}.blocks.{j}.mlp.fc2.weight", F"model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.output.dense.weight")     )
   rename_keys.append((F"backbone.layers.{i}.blocks.{j}.mlp.fc2.bias", F"model.pixel_level_module.encoder.model.encoder.layers.{i}.blocks.{j}.output.dense.bias")     )

  if i < 3:
   rename_keys.append((F"backbone.layers.{i}.downsample.reduction.weight", F"model.pixel_level_module.encoder.model.encoder.layers.{i}.downsample.reduction.weight")     )
   rename_keys.append((F"backbone.layers.{i}.downsample.norm.weight", F"model.pixel_level_module.encoder.model.encoder.layers.{i}.downsample.norm.weight")     )
   rename_keys.append((F"backbone.layers.{i}.downsample.norm.bias", F"model.pixel_level_module.encoder.model.encoder.layers.{i}.downsample.norm.bias")     )
  rename_keys.append((F"backbone.norm{i}.weight", F"model.pixel_level_module.encoder.hidden_states_norms.{i}.weight")     )
  rename_keys.append((F"backbone.norm{i}.bias", F"model.pixel_level_module.encoder.hidden_states_norms.{i}.bias")     )

 # FPN
 rename_keys.append(('''sem_seg_head.layer_4.weight''', '''model.pixel_level_module.decoder.fpn.stem.0.weight''')     )
 rename_keys.append(('''sem_seg_head.layer_4.norm.weight''', '''model.pixel_level_module.decoder.fpn.stem.1.weight''')     )
 rename_keys.append(('''sem_seg_head.layer_4.norm.bias''', '''model.pixel_level_module.decoder.fpn.stem.1.bias''')     )
 for source_index, target_index in zip(range(3    , 0    , -1     )    , range(0    , 3     )     ):
  rename_keys.append((F"sem_seg_head.adapter_{source_index}.weight", F"model.pixel_level_module.decoder.fpn.layers.{target_index}.proj.0.weight")     )
  rename_keys.append((F"sem_seg_head.adapter_{source_index}.norm.weight", F"model.pixel_level_module.decoder.fpn.layers.{target_index}.proj.1.weight")     )
  rename_keys.append((F"sem_seg_head.adapter_{source_index}.norm.bias", F"model.pixel_level_module.decoder.fpn.layers.{target_index}.proj.1.bias")     )
  rename_keys.append((F"sem_seg_head.layer_{source_index}.weight", F"model.pixel_level_module.decoder.fpn.layers.{target_index}.block.0.weight")     )
  rename_keys.append((F"sem_seg_head.layer_{source_index}.norm.weight", F"model.pixel_level_module.decoder.fpn.layers.{target_index}.block.1.weight")     )
  rename_keys.append((F"sem_seg_head.layer_{source_index}.norm.bias", F"model.pixel_level_module.decoder.fpn.layers.{target_index}.block.1.bias")     )
 rename_keys.append(('''sem_seg_head.mask_features.weight''', '''model.pixel_level_module.decoder.mask_projection.weight''')     )
 rename_keys.append(('''sem_seg_head.mask_features.bias''', '''model.pixel_level_module.decoder.mask_projection.bias''')     )

 # Transformer decoder
 for idx in range(config.decoder_config.decoder_layers     ):
  # self-attention out projection
  rename_keys.append((F"sem_seg_head.predictor.transformer.decoder.layers.{idx}.self_attn.out_proj.weight", F"model.transformer_module.decoder.layers.{idx}.self_attn.out_proj.weight")     )
  rename_keys.append((F"sem_seg_head.predictor.transformer.decoder.layers.{idx}.self_attn.out_proj.bias", F"model.transformer_module.decoder.layers.{idx}.self_attn.out_proj.bias")     )
  # cross-attention out projection
  rename_keys.append((F"sem_seg_head.predictor.transformer.decoder.layers.{idx}.multihead_attn.out_proj.weight", F"model.transformer_module.decoder.layers.{idx}.encoder_attn.out_proj.weight")     )
  rename_keys.append((F"sem_seg_head.predictor.transformer.decoder.layers.{idx}.multihead_attn.out_proj.bias", F"model.transformer_module.decoder.layers.{idx}.encoder_attn.out_proj.bias")     )
  # MLP 1
  rename_keys.append((F"sem_seg_head.predictor.transformer.decoder.layers.{idx}.linear1.weight", F"model.transformer_module.decoder.layers.{idx}.fc1.weight")     )
  rename_keys.append((F"sem_seg_head.predictor.transformer.decoder.layers.{idx}.linear1.bias", F"model.transformer_module.decoder.layers.{idx}.fc1.bias")     )
  # MLP 2
  rename_keys.append((F"sem_seg_head.predictor.transformer.decoder.layers.{idx}.linear2.weight", F"model.transformer_module.decoder.layers.{idx}.fc2.weight")     )
  rename_keys.append((F"sem_seg_head.predictor.transformer.decoder.layers.{idx}.linear2.bias", F"model.transformer_module.decoder.layers.{idx}.fc2.bias")     )
  # layernorm 1 (self-attention layernorm)
  rename_keys.append((F"sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm1.weight", F"model.transformer_module.decoder.layers.{idx}.self_attn_layer_norm.weight")     )
  rename_keys.append((F"sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm1.bias", F"model.transformer_module.decoder.layers.{idx}.self_attn_layer_norm.bias")     )
  # layernorm 2 (cross-attention layernorm)
  rename_keys.append((F"sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm2.weight", F"model.transformer_module.decoder.layers.{idx}.encoder_attn_layer_norm.weight")     )
  rename_keys.append((F"sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm2.bias", F"model.transformer_module.decoder.layers.{idx}.encoder_attn_layer_norm.bias")     )
  # layernorm 3 (final layernorm)
  rename_keys.append((F"sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm3.weight", F"model.transformer_module.decoder.layers.{idx}.final_layer_norm.weight")     )
  rename_keys.append((F"sem_seg_head.predictor.transformer.decoder.layers.{idx}.norm3.bias", F"model.transformer_module.decoder.layers.{idx}.final_layer_norm.bias")     )

 rename_keys.append(('''sem_seg_head.predictor.transformer.decoder.norm.weight''', '''model.transformer_module.decoder.layernorm.weight''')     )
 rename_keys.append(('''sem_seg_head.predictor.transformer.decoder.norm.bias''', '''model.transformer_module.decoder.layernorm.bias''')     )

 # heads on top
 rename_keys.append(('''sem_seg_head.predictor.query_embed.weight''', '''model.transformer_module.queries_embedder.weight''')     )

 rename_keys.append(('''sem_seg_head.predictor.input_proj.weight''', '''model.transformer_module.input_projection.weight''')     )
 rename_keys.append(('''sem_seg_head.predictor.input_proj.bias''', '''model.transformer_module.input_projection.bias''')     )

 rename_keys.append(('''sem_seg_head.predictor.class_embed.weight''', '''class_predictor.weight''')     )
 rename_keys.append(('''sem_seg_head.predictor.class_embed.bias''', '''class_predictor.bias''')     )

 for i in range(3     ):
  rename_keys.append((F"sem_seg_head.predictor.mask_embed.layers.{i}.weight", F"mask_embedder.{i}.0.weight")     )
  rename_keys.append((F"sem_seg_head.predictor.mask_embed.layers.{i}.bias", F"mask_embedder.{i}.0.bias")     )
 # fmt: on

 return rename_keys

def 			_a  (				SCREAMING_SNAKE_CASE	:				Optional[int]    , SCREAMING_SNAKE_CASE	:				Optional[int]    , SCREAMING_SNAKE_CASE	:				List[Any]     ):







 """simple docstring"""



 UpperCamelCase__					:    str          =   dct.pop(SCREAMING_SNAKE_CASE     )
 UpperCamelCase__					:    Tuple          =   val

def 			_a  (				SCREAMING_SNAKE_CASE	:				Dict    , SCREAMING_SNAKE_CASE	:				Dict     ):







 """simple docstring"""



 UpperCamelCase__					:    int          =   [int(backbone_config.embed_dim * 2**i     ) for i in range(len(backbone_config.depths     )     )]
 for i in range(len(backbone_config.depths     )     ):
  UpperCamelCase__					:    List[str]          =   num_features[i]
  for j in range(backbone_config.depths[i]     ):
   # fmt: off
   # read in weights + bias of input projection layer (in original implementation, this is a single matrix + bias)
   UpperCamelCase__					:    int          =   state_dict.pop(F"backbone.layers.{i}.blocks.{j}.attn.qkv.weight"     )
   UpperCamelCase__					:    List[Any]          =   state_dict.pop(F"backbone.layers.{i}.blocks.{j}.attn.qkv.bias"     )
   # next, add query, keys and values (in that order) to the state dict
   UpperCamelCase__					:    List[Any]          =   in_proj_weight[:dim, :]
   UpperCamelCase__					:    Optional[Any]          =   in_proj_bias[: dim]
   UpperCamelCase__					:    Union[str, Any]          =   in_proj_weight[
       dim : dim * 2, :
   ]
   UpperCamelCase__					:    int          =   in_proj_bias[
       dim : dim * 2
   ]
   UpperCamelCase__					:    Any          =   in_proj_weight[
       -dim :, :
   ]
   UpperCamelCase__					:    str          =   in_proj_bias[-dim :]
   # fmt: on

def 			_a  (				SCREAMING_SNAKE_CASE	:				int    , SCREAMING_SNAKE_CASE	:				Optional[int]     ):







 """simple docstring"""



 UpperCamelCase__					:    Union[str, Any]          =   config.decoder_config.hidden_size
 for idx in range(config.decoder_config.decoder_layers     ):
  # read in weights + bias of self-attention input projection layer (in the original implementation, this is a single matrix + bias)
  UpperCamelCase__					:    Tuple          =   state_dict.pop(F"sem_seg_head.predictor.transformer.decoder.layers.{idx}.self_attn.in_proj_weight"     )
  UpperCamelCase__					:    Optional[Any]          =   state_dict.pop(F"sem_seg_head.predictor.transformer.decoder.layers.{idx}.self_attn.in_proj_bias"     )
  # next, add query, keys and values (in that order) to the state dict
  UpperCamelCase__					:    int          =   in_proj_weight[: hidden_size, :]
  UpperCamelCase__					:    Any          =   in_proj_bias[:config.hidden_size]
  UpperCamelCase__					:    Any          =   in_proj_weight[hidden_size : hidden_size * 2, :]
  UpperCamelCase__					:    Any          =   in_proj_bias[hidden_size : hidden_size * 2]
  UpperCamelCase__					:    int          =   in_proj_weight[-hidden_size :, :]
  UpperCamelCase__					:    str          =   in_proj_bias[-hidden_size :]
  # read in weights + bias of cross-attention input projection layer (in the original implementation, this is a single matrix + bias)
  UpperCamelCase__					:    Dict          =   state_dict.pop(F"sem_seg_head.predictor.transformer.decoder.layers.{idx}.multihead_attn.in_proj_weight"     )
  UpperCamelCase__					:    Optional[Any]          =   state_dict.pop(F"sem_seg_head.predictor.transformer.decoder.layers.{idx}.multihead_attn.in_proj_bias"     )
  # next, add query, keys and values (in that order) to the state dict
  UpperCamelCase__					:    Dict          =   in_proj_weight[: hidden_size, :]
  UpperCamelCase__					:    Tuple          =   in_proj_bias[:config.hidden_size]
  UpperCamelCase__					:    Optional[Any]          =   in_proj_weight[hidden_size : hidden_size * 2, :]
  UpperCamelCase__					:    Tuple          =   in_proj_bias[hidden_size : hidden_size * 2]
  UpperCamelCase__					:    Union[str, Any]          =   in_proj_weight[-hidden_size :, :]
  UpperCamelCase__					:    int          =   in_proj_bias[-hidden_size :]
 # fmt: on

def 			_a  (				):







 """simple docstring"""



 UpperCamelCase__					:    Optional[Any]          =   '''http://images.cocodataset.org/val2017/000000039769.jpg'''
 UpperCamelCase__					:    Optional[Any]          =   Image.open(requests.get(SCREAMING_SNAKE_CASE    , stream=SCREAMING_SNAKE_CASE     ).raw     )
 return im

@torch.no_grad()
def 			_a  (				SCREAMING_SNAKE_CASE	:				str    , SCREAMING_SNAKE_CASE	:				str    , SCREAMING_SNAKE_CASE	:				str    , SCREAMING_SNAKE_CASE	:				bool = False     ):







 """simple docstring"""



 UpperCamelCase__					:    Union[str, Any]          =   get_maskformer_config(SCREAMING_SNAKE_CASE     )

 # load original state_dict
 with open(SCREAMING_SNAKE_CASE    , '''rb'''     ) as f:
  UpperCamelCase__					:    Dict          =   pickle.load(SCREAMING_SNAKE_CASE     )
 UpperCamelCase__					:    List[Any]          =   data['''model''']

 # for name, param in state_dict.items():
 #     print(name, param.shape)

 # rename keys
 UpperCamelCase__					:    Optional[int]          =   create_rename_keys(SCREAMING_SNAKE_CASE     )
 for src, dest in rename_keys:
  rename_key(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE     )
 read_in_swin_q_k_v(SCREAMING_SNAKE_CASE    , config.backbone_config     )
 read_in_decoder_q_k_v(SCREAMING_SNAKE_CASE    , SCREAMING_SNAKE_CASE     )

 # update to torch tensors
 for key, value in state_dict.items():
  UpperCamelCase__					:    int          =   torch.from_numpy(SCREAMING_SNAKE_CASE     )

 # load 🤗 model
 UpperCamelCase__					:    Dict          =   MaskFormerForInstanceSegmentation(SCREAMING_SNAKE_CASE     )
 model.eval()

 for name, param in model.named_parameters():
  print(SCREAMING_SNAKE_CASE    , param.shape     )

 UpperCamelCase__					:    Any          =   model.load_state_dict(SCREAMING_SNAKE_CASE    , strict=SCREAMING_SNAKE_CASE     )
 assert missing_keys == [
     "model.pixel_level_module.encoder.model.layernorm.weight",
     "model.pixel_level_module.encoder.model.layernorm.bias",
 ]
 assert len(SCREAMING_SNAKE_CASE     ) == 0, F"Unexpected keys: {unexpected_keys}"

 # verify results
 UpperCamelCase__					:    List[Any]          =   prepare_img()
 if "vistas" in model_name:
  UpperCamelCase__					:    Optional[Any]          =   65
 elif "cityscapes" in model_name:
  UpperCamelCase__					:    int          =   65535
 else:
  UpperCamelCase__					:    str          =   255
 UpperCamelCase__					:    Optional[Any]          =   True if '''ade''' in model_name else False
 UpperCamelCase__					:    Tuple          =   MaskFormerImageProcessor(ignore_index=SCREAMING_SNAKE_CASE    , reduce_labels=SCREAMING_SNAKE_CASE     )

 UpperCamelCase__					:    Tuple          =   image_processor(SCREAMING_SNAKE_CASE    , return_tensors='''pt'''     )

 UpperCamelCase__					:    Any          =   model(**SCREAMING_SNAKE_CASE     )

 print('''Logits:'''    , outputs.class_queries_logits[0, :3, :3]     )

 if model_name == "maskformer-swin-tiny-ade":
  UpperCamelCase__					:    Dict          =   torch.tensor(
      [[3.6353, -4.4770, -2.6065], [0.5081, -4.2394, -3.5343], [2.1909, -5.0353, -1.9323]]     )
 assert torch.allclose(outputs.class_queries_logits[0, :3, :3]    , SCREAMING_SNAKE_CASE    , atol=1E-4     )
 print('''Looks ok!'''     )

 if pytorch_dump_folder_path is not None:
  print(F"Saving model and image processor to {pytorch_dump_folder_path}"     )
  Path(SCREAMING_SNAKE_CASE     ).mkdir(exist_ok=SCREAMING_SNAKE_CASE     )
  model.save_pretrained(SCREAMING_SNAKE_CASE     )
  image_processor.save_pretrained(SCREAMING_SNAKE_CASE     )

 if push_to_hub:
  print('''Pushing model and image processor to the hub...'''     )
  model.push_to_hub(F"nielsr/{model_name}"     )
  image_processor.push_to_hub(F"nielsr/{model_name}"     )


if __name__ == "__main__":
       __UpperCamelCase       :       Tuple						       =				argparse.ArgumentParser()
       # Required parameters
       parser.add_argument(
           "--model_name",
           default="maskformer-swin-tiny-ade",
           type=str,
           help=("Name of the MaskFormer model you'd like to convert",),
       )
       parser.add_argument(
           "--checkpoint_path",
           default="/Users/nielsrogge/Documents/MaskFormer_checkpoints/MaskFormer-Swin-tiny-ADE20k/model.pkl",
           type=str,
           help="Path to the original state dict (.pth file).",
       )
       parser.add_argument(
           "--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model directory."
       )
       parser.add_argument(
           "--push_to_hub", action="store_true", help="Whether or not to push the converted model to the 🤗 hub."
       )

       __UpperCamelCase       :       Optional[Any]						       =				parser.parse_args()
       convert_maskformer_checkpoint(
           args.model_name, args.checkpoint_path, args.pytorch_dump_folder_path, args.push_to_hub
       )






import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
       from test_module.custom_tokenization_fast import CustomTokenizerFast

class       __magic_name__	(     unittest.TestCase):


     def      UpperCAmelCase__	(      self							:     Union[str, Any]			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   mock.Mock()
      UpperCamelCase__					:    Tuple          =   500
      UpperCamelCase__					:    Dict          =   {}
      UpperCamelCase__					:    Dict          =   HTTPError
      UpperCamelCase__					:    Optional[Any]          =   {}

      # Download this model to make sure it's in the cache.
      UpperCamelCase__					:    List[Any]          =   BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''			)

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('''requests.Session.request'''							,	return_value=lowerCamelCase__			) as mock_head:
       UpperCamelCase__					:    Any          =   BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''			)
       # This check we did call the fake head request
       mock_head.assert_called()


     @require_tokenizers
     def      UpperCAmelCase__	(      self							:     str			)  ->      Any:





      '''simple docstring'''






      UpperCamelCase__					:    Union[str, Any]          =   mock.Mock()
      UpperCamelCase__					:    Any          =   500
      UpperCamelCase__					:    str          =   {}
      UpperCamelCase__					:    Optional[Any]          =   HTTPError
      UpperCamelCase__					:    Tuple          =   {}

      # Download this model to make sure it's in the cache.
      UpperCamelCase__					:    Optional[Any]          =   GPTaTokenizerFast.from_pretrained('''gpt2'''			)

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('''requests.Session.request'''							,	return_value=lowerCamelCase__			) as mock_head:
       UpperCamelCase__					:    str          =   GPTaTokenizerFast.from_pretrained('''gpt2'''			)
       # This check we did call the fake head request
       mock_head.assert_called()


     def      UpperCAmelCase__	(      self							:     Any			)  ->      List[str]:





      '''simple docstring'''






      try:
       UpperCamelCase__					:    Dict          =   tempfile.mktemp()
       with open(lowerCamelCase__							,	'''wb'''			) as f:
        http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''							,	lowerCamelCase__			)

       UpperCamelCase__					:    Dict          =   AlbertTokenizer.from_pretrained(lowerCamelCase__			)
      finally:
       os.remove(lowerCamelCase__			)

      # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
      # the current folder and have the right name.
      if os.path.isfile('''tokenizer.json'''			):
       # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
       return
      try:
       with open('''tokenizer.json'''							,	'''wb'''			) as f:
        http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''							,	lowerCamelCase__			)
       UpperCamelCase__					:    Union[str, Any]          =   AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''			)
       # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
       self.assertEqual(tokenizer.vocab_size							,	1000			)
       # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

      finally:
       os.remove('''tokenizer.json'''			)


     def      UpperCAmelCase__	(      self							:     str			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''			)



@is_staging_test
class       __magic_name__	(     unittest.TestCase):
     A:		List[Any]			=      ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]


     @classmethod
     def      UpperCAmelCase__	(      cls							:     List[str]			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    List[str]          =   TOKEN
      HfFolder.save_token(lowerCamelCase__			)


     @classmethod
     def      UpperCAmelCase__	(      cls							:     Any			)  ->      Optional[Any]:





      '''simple docstring'''






      try:
       delete_repo(token=cls._token							,	repo_id='''test-tokenizer'''			)
      except HTTPError:
       pass

      try:
       delete_repo(token=cls._token							,	repo_id='''valid_org/test-tokenizer-org'''			)
      except HTTPError:
       pass

      try:
       delete_repo(token=cls._token							,	repo_id='''test-dynamic-tokenizer'''			)
      except HTTPError:
       pass


     def      UpperCAmelCase__	(      self							:     Union[str, Any]			)  ->      Optional[int]:





      '''simple docstring'''






      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    int          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)
       UpperCamelCase__					:    Tuple          =   BertTokenizer(lowerCamelCase__			)

      tokenizer.push_to_hub('''test-tokenizer'''							,	use_auth_token=self._token			)
      UpperCamelCase__					:    Union[str, Any]          =   BertTokenizer.from_pretrained(F"{USER}/test-tokenizer"			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)

      # Reset repo
      delete_repo(token=self._token							,	repo_id='''test-tokenizer'''			)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(lowerCamelCase__							,	repo_id='''test-tokenizer'''							,	push_to_hub=lowerCamelCase__							,	use_auth_token=self._token			)

      UpperCamelCase__					:    str          =   BertTokenizer.from_pretrained(F"{USER}/test-tokenizer"			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)


     def      UpperCAmelCase__	(      self							:     str			)  ->      Optional[int]:





      '''simple docstring'''






      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    Dict          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)
       UpperCamelCase__					:    List[str]          =   BertTokenizer(lowerCamelCase__			)

      tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''							,	use_auth_token=self._token			)
      UpperCamelCase__					:    Optional[Any]          =   BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)

      # Reset repo
      delete_repo(token=self._token							,	repo_id='''valid_org/test-tokenizer-org'''			)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(
           lowerCamelCase__							,	repo_id='''valid_org/test-tokenizer-org'''							,	push_to_hub=lowerCamelCase__							,	use_auth_token=self._token			)

      UpperCamelCase__					:    Union[str, Any]          =   BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''			)
      self.assertDictEqual(new_tokenizer.vocab							,	tokenizer.vocab			)


     @require_tokenizers
     def      UpperCAmelCase__	(      self							:     List[str]			)  ->      int:





      '''simple docstring'''






      CustomTokenizer.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    List[Any]          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)
       UpperCamelCase__					:    Any          =   CustomTokenizer(lowerCamelCase__			)

      # No fast custom tokenizer
      tokenizer.push_to_hub('''test-dynamic-tokenizer'''							,	use_auth_token=self._token			)

      UpperCamelCase__					:    str          =   AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"							,	trust_remote_code=lowerCamelCase__			)
      # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__							,	'''CustomTokenizer'''			)

      # Fast and slow custom tokenizer
      CustomTokenizerFast.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
       UpperCamelCase__					:    List[Any]          =   os.path.join(lowerCamelCase__							,	'''vocab.txt'''			)
       with open(lowerCamelCase__							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]			)			)

       UpperCamelCase__					:    List[Any]          =   BertTokenizerFast.from_pretrained(lowerCamelCase__			)
       bert_tokenizer.save_pretrained(lowerCamelCase__			)
       UpperCamelCase__					:    Optional[int]          =   CustomTokenizerFast.from_pretrained(lowerCamelCase__			)

      tokenizer.push_to_hub('''test-dynamic-tokenizer'''							,	use_auth_token=self._token			)

      UpperCamelCase__					:    str          =   AutoTokenizer.from_pretrained(F"{USER}/test-dynamic-tokenizer"							,	trust_remote_code=lowerCamelCase__			)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__							,	'''CustomTokenizerFast'''			)
      UpperCamelCase__					:    int          =   AutoTokenizer.from_pretrained(
          F"{USER}/test-dynamic-tokenizer"							,	use_fast=lowerCamelCase__							,	trust_remote_code=lowerCamelCase__			)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__							,	'''CustomTokenizer'''			)



class       __magic_name__	(     unittest.TestCase):


     def      UpperCAmelCase__	(      self							:     str			)  ->      Dict:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   Trie()
      trie.add('''Hello 友達'''			)
      self.assertEqual(trie.data							,	{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}			)
      trie.add('''Hello'''			)
      trie.data
      self.assertEqual(trie.data							,	{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}			)


     def      UpperCAmelCase__	(      self							:     str			)  ->      str:





      '''simple docstring'''






      UpperCamelCase__					:    Dict          =   Trie()
      self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''			)							,	['''[CLS] This is a extra_id_100''']			)
      trie.add('''[CLS]'''			)
      trie.add('''extra_id_1'''			)
      trie.add('''extra_id_100'''			)
      self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''			)							,	['''[CLS]''', ''' This is a ''', '''extra_id_100''']			)


     def      UpperCAmelCase__	(      self							:     int			)  ->      Optional[int]:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   Trie()
      trie.add('''A'''			)
      self.assertEqual(trie.split('''ABC'''			)							,	['''A''', '''BC''']			)
      self.assertEqual(trie.split('''BCA'''			)							,	['''BC''', '''A''']			)


     def      UpperCAmelCase__	(      self							:     Dict			)  ->      Optional[int]:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   Trie()
      trie.add('''TOKEN]'''			)
      trie.add('''[SPECIAL_TOKEN]'''			)
      self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''			)							,	['''This is something ''', '''[SPECIAL_TOKEN]''']			)


     def      UpperCAmelCase__	(      self							:     List[str]			)  ->      Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__					:    List[Any]          =   Trie()
      trie.add('''A'''			)
      trie.add('''P'''			)
      trie.add('''[SPECIAL_TOKEN]'''			)
      self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''			)							,	['''This is something ''', '''[SPECIAL_TOKEN]''']			)


     def      UpperCAmelCase__	(      self							:     Any			)  ->      List[str]:





      '''simple docstring'''






      UpperCamelCase__					:    List[Any]          =   Trie()
      trie.add('''AB'''			)
      trie.add('''B'''			)
      trie.add('''C'''			)
      self.assertEqual(trie.split('''ABC'''			)							,	['''AB''', '''C''']			)


     def      UpperCAmelCase__	(      self							:     Optional[int]			)  ->      int:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   Trie()
      trie.add('''ABC'''			)
      trie.add('''B'''			)
      trie.add('''CD'''			)
      self.assertEqual(trie.split('''ABCD'''			)							,	['''ABC''', '''D''']			)


     def      UpperCAmelCase__	(      self							:     Optional[int]			)  ->      Union[str, Any]:





      '''simple docstring'''






      UpperCamelCase__					:    Dict          =   Trie()
      UpperCamelCase__					:    Optional[Any]          =   trie.cut_text('''ABC'''							,	[0, 0, 2, 1, 2, 3]			)
      self.assertEqual(lowerCamelCase__							,	['''AB''', '''C''']			)


