





import argparse
import json

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import SegformerImageProcessor, SwinConfig, UperNetConfig, UperNetForSemanticSegmentation

def      UpperCAmelCase_    (    _UpperCAmelCase							:Optional[int]				)	->				str:
     '''simple docstring'''


     A_          =       384
     A_          =       7
     if "tiny" in model_name:
          A_          =       96
          A_          =       (2, 2, 6, 2)
          A_          =       (3, 6, 12, 24)
     elif "small" in model_name:
          A_          =       96
          A_          =       (2, 2, 18, 2)
          A_          =       (3, 6, 12, 24)
     elif "base" in model_name:
          A_          =       128
          A_          =       (2, 2, 18, 2)
          A_          =       (4, 8, 16, 32)
          A_          =       12
          A_          =       512
     elif "large" in model_name:
          A_          =       192
          A_          =       (2, 2, 18, 2)
          A_          =       (6, 12, 24, 48)
          A_          =       12
          A_          =       768

     # set label information
     A_          =       150
     A_          =       'huggingface/label-files'
     A_          =       'ade20k-id2label.json'
     A_          =       json.load(open(hf_hub_download(_UpperCAmelCase  ,						_UpperCAmelCase  ,						repo_type='''dataset'''				)  ,						'''r'''				)				)
     A_          =       {int(_UpperCAmelCase				): v for k, v in idalabel.items()}
     A_          =       {v: k for k, v in idalabel.items()}

     A_          =       SwinConfig(
         embed_dim=_UpperCAmelCase  ,						depths=_UpperCAmelCase  ,						num_heads=_UpperCAmelCase  ,						window_size=_UpperCAmelCase  ,						out_features=['''stage1''', '''stage2''', '''stage3''', '''stage4''']  ,						)
     A_          =       UperNetConfig(
         backbone_config=_UpperCAmelCase  ,						auxiliary_in_channels=_UpperCAmelCase  ,						num_labels=_UpperCAmelCase  ,						idalabel=_UpperCAmelCase  ,						labelaid=_UpperCAmelCase  ,						)

     return config

def      UpperCAmelCase_    (    _UpperCAmelCase							:Tuple				)	->				List[str]:
     '''simple docstring'''


     A_          =       []

     # fmt: off
     # stem
     rename_keys.append(('''backbone.patch_embed.projection.weight''', '''backbone.embeddings.patch_embeddings.projection.weight''')				)
     rename_keys.append(('''backbone.patch_embed.projection.bias''', '''backbone.embeddings.patch_embeddings.projection.bias''')				)
     rename_keys.append(('''backbone.patch_embed.norm.weight''', '''backbone.embeddings.norm.weight''')				)
     rename_keys.append(('''backbone.patch_embed.norm.bias''', '''backbone.embeddings.norm.bias''')				)
     # stages
     for i in range(len(config.backbone_config.depths				)				):
          for j in range(config.backbone_config.depths[i]				):
               rename_keys.append((f'backbone.stages.{i}.blocks.{j}.norm1.weight', f'backbone.encoder.layers.{i}.blocks.{j}.layernorm_before.weight')				)
               rename_keys.append((f'backbone.stages.{i}.blocks.{j}.norm1.bias', f'backbone.encoder.layers.{i}.blocks.{j}.layernorm_before.bias')				)
               rename_keys.append((f'backbone.stages.{i}.blocks.{j}.attn.w_msa.relative_position_bias_table', f'backbone.encoder.layers.{i}.blocks.{j}.attention.self.relative_position_bias_table')				)
               rename_keys.append((f'backbone.stages.{i}.blocks.{j}.attn.w_msa.relative_position_index', f'backbone.encoder.layers.{i}.blocks.{j}.attention.self.relative_position_index')				)
               rename_keys.append((f'backbone.stages.{i}.blocks.{j}.attn.w_msa.proj.weight', f'backbone.encoder.layers.{i}.blocks.{j}.attention.output.dense.weight')				)
               rename_keys.append((f'backbone.stages.{i}.blocks.{j}.attn.w_msa.proj.bias', f'backbone.encoder.layers.{i}.blocks.{j}.attention.output.dense.bias')				)
               rename_keys.append((f'backbone.stages.{i}.blocks.{j}.norm2.weight', f'backbone.encoder.layers.{i}.blocks.{j}.layernorm_after.weight')				)
               rename_keys.append((f'backbone.stages.{i}.blocks.{j}.norm2.bias', f'backbone.encoder.layers.{i}.blocks.{j}.layernorm_after.bias')				)
               rename_keys.append((f'backbone.stages.{i}.blocks.{j}.ffn.layers.0.0.weight', f'backbone.encoder.layers.{i}.blocks.{j}.intermediate.dense.weight')				)
               rename_keys.append((f'backbone.stages.{i}.blocks.{j}.ffn.layers.0.0.bias', f'backbone.encoder.layers.{i}.blocks.{j}.intermediate.dense.bias')				)
               rename_keys.append((f'backbone.stages.{i}.blocks.{j}.ffn.layers.1.weight', f'backbone.encoder.layers.{i}.blocks.{j}.output.dense.weight')				)
               rename_keys.append((f'backbone.stages.{i}.blocks.{j}.ffn.layers.1.bias', f'backbone.encoder.layers.{i}.blocks.{j}.output.dense.bias')				)

          if i < 3:
               rename_keys.append((f'backbone.stages.{i}.downsample.reduction.weight', f'backbone.encoder.layers.{i}.downsample.reduction.weight')				)
               rename_keys.append((f'backbone.stages.{i}.downsample.norm.weight', f'backbone.encoder.layers.{i}.downsample.norm.weight')				)
               rename_keys.append((f'backbone.stages.{i}.downsample.norm.bias', f'backbone.encoder.layers.{i}.downsample.norm.bias')				)
          rename_keys.append((f'backbone.norm{i}.weight', f'backbone.hidden_states_norms.stage{i+1}.weight')				)
          rename_keys.append((f'backbone.norm{i}.bias', f'backbone.hidden_states_norms.stage{i+1}.bias')				)

     # decode head
     rename_keys.extend(
         [
             ('''decode_head.conv_seg.weight''', '''decode_head.classifier.weight'''),
             ('''decode_head.conv_seg.bias''', '''decode_head.classifier.bias'''),
             ('''auxiliary_head.conv_seg.weight''', '''auxiliary_head.classifier.weight'''),
             ('''auxiliary_head.conv_seg.bias''', '''auxiliary_head.classifier.bias'''),
         ]				)
     # fmt: on

     return rename_keys

def      UpperCAmelCase_    (    _UpperCAmelCase							:Optional[Any]  ,						_UpperCAmelCase							:Union[str, Any]  ,						_UpperCAmelCase							:str				)	->				Dict:
     '''simple docstring'''


     A_          =       dct.pop(_UpperCAmelCase				)
     A_          =       val

def      UpperCAmelCase_    (    _UpperCAmelCase							:List[Any]  ,						_UpperCAmelCase							:str				)	->				Any:
     '''simple docstring'''


     A_          =       [int(backbone_config.embed_dim * 2**i				) for i in range(len(backbone_config.depths				)				)]
     for i in range(len(backbone_config.depths				)				):
          A_          =       num_features[i]
          for j in range(backbone_config.depths[i]				):
               # fmt: off
               # read in weights + bias of input projection layer (in original implementation, this is a single matrix + bias)
               A_          =       state_dict.pop(f'backbone.stages.{i}.blocks.{j}.attn.w_msa.qkv.weight'				)
               A_          =       state_dict.pop(f'backbone.stages.{i}.blocks.{j}.attn.w_msa.qkv.bias'				)
               # next, add query, keys and values (in that order) to the state dict
               A_          =       in_proj_weight[:dim, :]
               A_          =       in_proj_bias[: dim]
               A_          =       in_proj_weight[
                   dim : dim * 2, :
               ]
               A_          =       in_proj_bias[
                   dim : dim * 2
               ]
               A_          =       in_proj_weight[
                   -dim :, :
               ]
               A_          =       in_proj_bias[-dim :]
               # fmt: on

def      UpperCAmelCase_    (    _UpperCAmelCase							:Tuple				)	->				Union[str, Any]:
     '''simple docstring'''


     A_          =       x.shape
     A_          =       x.reshape(_UpperCAmelCase  ,						4  ,						in_channel // 4				)
     A_          =       x[:, [0, 2, 1, 3], :].transpose(1  ,						2				).reshape(_UpperCAmelCase  ,						_UpperCAmelCase				)
     return x

def      UpperCAmelCase_    (    _UpperCAmelCase							:int				)	->				Any:
     '''simple docstring'''


     A_          =       x.shape
     A_          =       x.reshape(_UpperCAmelCase  ,						in_channel // 4  ,						4				)
     A_          =       x[:, :, [0, 2, 1, 3]].transpose(1  ,						2				).reshape(_UpperCAmelCase  ,						_UpperCAmelCase				)

     return x

def      UpperCAmelCase_    (    _UpperCAmelCase							:Optional[Any]				)	->				Optional[int]:
     '''simple docstring'''


     A_          =       x.shape[0]
     A_          =       x.reshape(4  ,						in_channel // 4				)
     A_          =       x[[0, 2, 1, 3], :].transpose(0  ,						1				).reshape(_UpperCAmelCase				)
     return x

def      UpperCAmelCase_    (    _UpperCAmelCase							:int				)	->				Tuple:
     '''simple docstring'''


     A_          =       x.shape[0]
     A_          =       x.reshape(in_channel // 4  ,						4				)
     A_          =       x[:, [0, 2, 1, 3]].transpose(0  ,						1				).reshape(_UpperCAmelCase				)
     return x

def      UpperCAmelCase_    (    _UpperCAmelCase							:Union[str, Any]  ,						_UpperCAmelCase							:Optional[int]  ,						_UpperCAmelCase							:List[Any]				)	->				Any:
     '''simple docstring'''


     A_          =       {
         'upernet-swin-tiny': 'https://download.openmmlab.com/mmsegmentation/v0.5/swin/upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K/upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K_20210531_112542-e380ad3e.pth',
         'upernet-swin-small': 'https://download.openmmlab.com/mmsegmentation/v0.5/swin/upernet_swin_small_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K/upernet_swin_small_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K_20210526_192015-ee2fff1c.pth',
         'upernet-swin-base': 'https://download.openmmlab.com/mmsegmentation/v0.5/swin/upernet_swin_base_patch4_window12_512x512_160k_ade20k_pretrain_384x384_22K/upernet_swin_base_patch4_window12_512x512_160k_ade20k_pretrain_384x384_22K_20210531_125459-429057bf.pth',
         'upernet-swin-large': 'https://download.openmmlab.com/mmsegmentation/v0.5/swin/upernet_swin_large_patch4_window12_512x512_pretrain_384x384_22K_160k_ade20k/upernet_swin_large_patch4_window12_512x512_pretrain_384x384_22K_160k_ade20k_20220318_091743-9ba68901.pth',
     }
     A_          =       model_name_to_url[model_name]
     A_          =       torch.hub.load_state_dict_from_url(_UpperCAmelCase  ,						map_location='''cpu'''  ,						file_name=_UpperCAmelCase				)[
         'state_dict'
     ]

     for name, param in state_dict.items():
          print(_UpperCAmelCase  ,						param.shape				)

     A_          =       get_upernet_config(_UpperCAmelCase				)
     A_          =       UperNetForSemanticSegmentation(_UpperCAmelCase				)
     model.eval()

     # replace "bn" => "batch_norm"
     for key in state_dict.copy().keys():
          A_          =       state_dict.pop(_UpperCAmelCase				)
          if "bn" in key:
               A_          =       key.replace('''bn'''  ,						'''batch_norm'''				)
          A_          =       val

     # rename keys
     A_          =       create_rename_keys(_UpperCAmelCase				)
     for src, dest in rename_keys:
          rename_key(_UpperCAmelCase  ,						_UpperCAmelCase  ,						_UpperCAmelCase				)
     read_in_q_k_v(_UpperCAmelCase  ,						config.backbone_config				)

     # fix downsample parameters
     for key, value in state_dict.items():
          if "downsample" in key:
               if "reduction" in key:
                    A_          =       reverse_correct_unfold_reduction_order(_UpperCAmelCase				)
               if "norm" in key:
                    A_          =       reverse_correct_unfold_norm_order(_UpperCAmelCase				)

     model.load_state_dict(_UpperCAmelCase				)

     # verify on image
     A_          =       'https://huggingface.co/datasets/hf-internal-testing/fixtures_ade20k/resolve/main/ADE_val_00000001.jpg'
     A_          =       Image.open(requests.get(_UpperCAmelCase  ,						stream=_UpperCAmelCase				).raw				).convert('''RGB'''				)

     A_          =       SegformerImageProcessor()
     A_          =       processor(_UpperCAmelCase  ,						return_tensors='''pt'''				).pixel_values

     with torch.no_grad():
          A_          =       model(_UpperCAmelCase				)
          A_          =       outputs.logits

     print(logits.shape				)
     print('''First values of logits:'''  ,						logits[0, 0, :3, :3]				)
     # assert values
     if model_name == "upernet-swin-tiny":
          A_          =       torch.tensor(
              [[-7.5958, -7.5958, -7.4302], [-7.5958, -7.5958, -7.4302], [-7.4797, -7.4797, -7.3068]]				)
     elif model_name == "upernet-swin-small":
          A_          =       torch.tensor(
              [[-7.1921, -7.1921, -6.9532], [-7.1921, -7.1921, -6.9532], [-7.0908, -7.0908, -6.8534]]				)
     elif model_name == "upernet-swin-base":
          A_          =       torch.tensor(
              [[-6.5851, -6.5851, -6.4330], [-6.5851, -6.5851, -6.4330], [-6.4763, -6.4763, -6.3254]]				)
     elif model_name == "upernet-swin-large":
          A_          =       torch.tensor(
              [[-7.5297, -7.5297, -7.3802], [-7.5297, -7.5297, -7.3802], [-7.4044, -7.4044, -7.2586]]				)
     print('''Logits:'''  ,						outputs.logits[0, 0, :3, :3]				)
     assert torch.allclose(outputs.logits[0, 0, :3, :3]  ,						_UpperCAmelCase  ,						atol=1e-4				)
     print('''Looks ok!'''				)

     if pytorch_dump_folder_path is not None:
          print(f'Saving model {model_name} to {pytorch_dump_folder_path}'				)
          model.save_pretrained(_UpperCAmelCase				)
          print(f'Saving processor to {pytorch_dump_folder_path}'				)
          processor.save_pretrained(_UpperCAmelCase				)

     if push_to_hub:
          print(f'Pushing model and processor for {model_name} to hub'				)
          model.push_to_hub(f'openmmlab/{model_name}'				)
          processor.push_to_hub(f'openmmlab/{model_name}'				)


if __name__ == "__main__":
      a__      :     str         =		argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          '--model_name',
          default='upernet-swin-tiny',
          type=str,
          choices=[f'''upernet-swin-{size}''' for size in ['tiny', 'small', 'base', 'large']],
          help='Name of the Swin + UperNet model you\'d like to convert.',
      )
      parser.add_argument(
          '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
      )
      parser.add_argument(
          '--push_to_hub', action='store_true', help='Whether or not to push the converted model to the 🤗 hub.'
      )

      a__      :     Union[str, Any]         =		parser.parse_args()
      convert_upernet_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)








'''simple docstring'''





import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class  snake_case__						(    SCREAMING_SNAKE_CASE_  ,     unittest.TestCase				):
 A__				= BertTokenizer
 A__				= BertTokenizerFast
 A__				= True
 A__				= True
 A__				= filter_non_english
 def A_   (   self						:							Optional[Any]  )      ->   Dict:







       '''simple docstring'''
       super().setUp()

       __snake_case :   List[Any]													=    [
           '[UNK]',
           '[CLS]',
           '[SEP]',
           '[PAD]',
           '[MASK]',
           'want',
           '##want',
           '##ed',
           'wa',
           'un',
           'runn',
           '##ing',
           ',',
           'low',
           'lowest',
       ]
       __snake_case :   Optional[int]													=    os.path.join(self.tmpdirname      ,     VOCAB_FILES_NAMES['vocab_file']  )
       with open(self.vocab_file      ,     'w'      ,     encoding='utf-8'  ) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )
 def A_   (   self						:							Dict      ,     __a						:							Optional[int]  )      ->   int:







       '''simple docstring'''
       __snake_case :   List[Any]													=    'UNwant\u00E9d,running'
       __snake_case :   Tuple													=    'unwanted, running'
       return input_text, output_text
 def A_   (   self						:							Tuple  )      ->   Dict:







       '''simple docstring'''
       __snake_case :   Any													=    self.tokenizer_class(self.vocab_file  )

       __snake_case :   Tuple													=    tokenizer.tokenize('UNwant\u00E9d,running'  )
       self.assertListEqual(__a      ,     ['un', '##want', '##ed', ',', 'runn', '##ing']  )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__a  )      ,     [9, 6, 7, 12, 10, 11]  )
 def A_   (   self						:							Optional[Any]  )      ->   Tuple:







       '''simple docstring'''
       if not self.test_rust_tokenizer:
             return

       __snake_case :   int													=    self.get_tokenizer()
       __snake_case :   Optional[Any]													=    self.get_rust_tokenizer()

       __snake_case :   int													=    'UNwant\u00E9d,running'

       __snake_case :   Any													=    tokenizer.tokenize(__a  )
       __snake_case :   Dict													=    rust_tokenizer.tokenize(__a  )
       self.assertListEqual(__a      ,     __a  )

       __snake_case :   Optional[int]													=    tokenizer.encode(__a      ,     add_special_tokens=__a  )
       __snake_case :   Optional[int]													=    rust_tokenizer.encode(__a      ,     add_special_tokens=__a  )
       self.assertListEqual(__a      ,     __a  )

       __snake_case :   str													=    self.get_rust_tokenizer()
       __snake_case :   Tuple													=    tokenizer.encode(__a  )
       __snake_case :   Tuple													=    rust_tokenizer.encode(__a  )
       self.assertListEqual(__a      ,     __a  )

       # With lower casing
       __snake_case :   Union[str, Any]													=    self.get_tokenizer(do_lower_case=__a  )
       __snake_case :   Any													=    self.get_rust_tokenizer(do_lower_case=__a  )

       __snake_case :   Optional[int]													=    'UNwant\u00E9d,running'

       __snake_case :   List[Any]													=    tokenizer.tokenize(__a  )
       __snake_case :   List[str]													=    rust_tokenizer.tokenize(__a  )
       self.assertListEqual(__a      ,     __a  )

       __snake_case :   List[str]													=    tokenizer.encode(__a      ,     add_special_tokens=__a  )
       __snake_case :   Union[str, Any]													=    rust_tokenizer.encode(__a      ,     add_special_tokens=__a  )
       self.assertListEqual(__a      ,     __a  )

       __snake_case :   Union[str, Any]													=    self.get_rust_tokenizer()
       __snake_case :   Dict													=    tokenizer.encode(__a  )
       __snake_case :   str													=    rust_tokenizer.encode(__a  )
       self.assertListEqual(__a      ,     __a  )
 def A_   (   self						:							Tuple  )      ->   Optional[int]:







       '''simple docstring'''
       __snake_case :   Optional[Any]													=    BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'  )      ,     ['ah', '\u535A', '\u63A8', 'zz']  )
 def A_   (   self						:							List[str]  )      ->   Any:







       '''simple docstring'''
       __snake_case :   Optional[Any]													=    BasicTokenizer(do_lower_case=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )      ,     ['hello', '!', 'how', 'are', 'you', '?']  )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )      ,     ['hello']  )
 def A_   (   self						:							str  )      ->   int:







       '''simple docstring'''
       __snake_case :   Dict													=    BasicTokenizer(do_lower_case=__a      ,     strip_accents=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,     ['hällo', '!', 'how', 'are', 'you', '?']  )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )      ,     ['h\u00E9llo']  )
 def A_   (   self						:							List[str]  )      ->   Tuple:







       '''simple docstring'''
       __snake_case :   str													=    BasicTokenizer(do_lower_case=__a      ,     strip_accents=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,     ['hallo', '!', 'how', 'are', 'you', '?']  )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )      ,     ['hello']  )
 def A_   (   self						:							str  )      ->   str:







       '''simple docstring'''
       __snake_case :   Tuple													=    BasicTokenizer(do_lower_case=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,     ['hallo', '!', 'how', 'are', 'you', '?']  )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )      ,     ['hello']  )
 def A_   (   self						:							str  )      ->   Optional[Any]:







       '''simple docstring'''
       __snake_case :   int													=    BasicTokenizer(do_lower_case=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )      ,     ['HeLLo', '!', 'how', 'Are', 'yoU', '?']  )
 def A_   (   self						:							Dict  )      ->   List[str]:







       '''simple docstring'''
       __snake_case :   Any													=    BasicTokenizer(do_lower_case=__a      ,     strip_accents=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,     ['HäLLo', '!', 'how', 'Are', 'yoU', '?']  )
 def A_   (   self						:							Dict  )      ->   Union[str, Any]:







       '''simple docstring'''
       __snake_case :   List[Any]													=    BasicTokenizer(do_lower_case=__a      ,     strip_accents=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,     ['HaLLo', '!', 'how', 'Are', 'yoU', '?']  )
 def A_   (   self						:							Optional[Any]  )      ->   List[Any]:







       '''simple docstring'''
       __snake_case :   List[str]													=    BasicTokenizer(do_lower_case=__a      ,     never_split=['[UNK]']  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'  )      ,     ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']  )
 def A_   (   self						:							Optional[Any]  )      ->   Optional[int]:







       '''simple docstring'''
       __snake_case :   int													=    BasicTokenizer()
       __snake_case :   List[str]													=    'a\n\'ll !!to?\'d of, can\'t.'
       __snake_case :   str													=    ['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
       self.assertListEqual(tokenizer.tokenize(__a  )      ,     __a  )
 def A_   (   self						:							List[Any]  )      ->   Tuple:







       '''simple docstring'''
       __snake_case :   str													=    ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

       __snake_case :   Dict													=    {}
       for i, token in enumerate(__a  ):
             __snake_case :   str													=    i
       __snake_case :   List[str]													=    WordpieceTokenizer(vocab=__a      ,     unk_token='[UNK]'  )

       self.assertListEqual(tokenizer.tokenize(''  )      ,     []  )

       self.assertListEqual(tokenizer.tokenize('unwanted running'  )      ,     ['un', '##want', '##ed', 'runn', '##ing']  )

       self.assertListEqual(tokenizer.tokenize('unwantedX running'  )      ,     ['[UNK]', 'runn', '##ing']  )
 def A_   (   self						:							Dict  )      ->   List[str]:







       '''simple docstring'''
       self.assertTrue(_is_whitespace(' '  )  )
       self.assertTrue(_is_whitespace('\t'  )  )
       self.assertTrue(_is_whitespace('\r'  )  )
       self.assertTrue(_is_whitespace('\n'  )  )
       self.assertTrue(_is_whitespace('\u00A0'  )  )

       self.assertFalse(_is_whitespace('A'  )  )
       self.assertFalse(_is_whitespace('-'  )  )
 def A_   (   self						:							List[str]  )      ->   List[Any]:







       '''simple docstring'''
       self.assertTrue(_is_control('\u0005'  )  )

       self.assertFalse(_is_control('A'  )  )
       self.assertFalse(_is_control(' '  )  )
       self.assertFalse(_is_control('\t'  )  )
       self.assertFalse(_is_control('\r'  )  )
 def A_   (   self						:							Any  )      ->   Union[str, Any]:







       '''simple docstring'''
       self.assertTrue(_is_punctuation('-'  )  )
       self.assertTrue(_is_punctuation('$'  )  )
       self.assertTrue(_is_punctuation('`'  )  )
       self.assertTrue(_is_punctuation('.'  )  )

       self.assertFalse(_is_punctuation('A'  )  )
       self.assertFalse(_is_punctuation(' '  )  )
 def A_   (   self						:							List[Any]  )      ->   Tuple:







       '''simple docstring'''
       __snake_case :   str													=    self.get_tokenizer()
       __snake_case :   int													=    self.get_rust_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(__a  ) for t in ['Test', '\xad', 'test']]      ,     [['[UNK]'], [], ['[UNK]']]  )

       self.assertListEqual(
           [rust_tokenizer.tokenize(__a  ) for t in ['Test', '\xad', 'test']]      ,     [['[UNK]'], [], ['[UNK]']]  )
 @slow
 def A_   (   self						:							List[str]  )      ->   List[Any]:







       '''simple docstring'''
       __snake_case :   Dict													=    self.tokenizer_class.from_pretrained('bert-base-uncased'  )

       __snake_case :   Dict													=    tokenizer.encode('sequence builders'      ,     add_special_tokens=__a  )
       __snake_case :   int													=    tokenizer.encode('multi-sequence build'      ,     add_special_tokens=__a  )

       __snake_case :   int													=    tokenizer.build_inputs_with_special_tokens(__a  )
       __snake_case :   Union[str, Any]													=    tokenizer.build_inputs_with_special_tokens(__a      ,     __a  )

       assert encoded_sentence == [101] + text + [102]
       assert encoded_pair == [101] + text + [102] + text_a + [102]
 def A_   (   self						:							Optional[int]  )      ->   str:







       '''simple docstring'''
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                   __snake_case :   Optional[int]													=    self.rust_tokenizer_class.from_pretrained(__a      ,     **__a  )

                   __snake_case :   Optional[Any]													=    f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                   __snake_case :   Tuple													=    tokenizer_r.encode_plus(
                       __a      ,     return_attention_mask=__a      ,     return_token_type_ids=__a      ,     return_offsets_mapping=__a      ,     add_special_tokens=__a      ,     )

                   __snake_case :   Optional[Any]													=    tokenizer_r.do_lower_case if hasattr(__a      ,     'do_lower_case'  ) else False
                   __snake_case :   Any													=    (
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), 'A'),
                           ((1, 2), ','),
                           ((3, 5), 'na'),
                           ((5, 6), '##ï'),
                           ((6, 8), '##ve'),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), 'Allen'),
                           ((21, 23), '##NL'),
                           ((23, 24), '##P'),
                           ((25, 33), 'sentence'),
                           ((33, 34), '.'),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), 'a'),
                           ((1, 2), ','),
                           ((3, 8), 'naive'),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), 'allen'),
                           ((21, 23), '##nl'),
                           ((23, 24), '##p'),
                           ((25, 33), 'sentence'),
                           ((33, 34), '.'),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results]      ,     tokenizer_r.convert_ids_to_tokens(tokens['input_ids']  )  )
                   self.assertEqual([e[0] for e in expected_results]      ,     tokens['offset_mapping']  )





 def A_   (   self						:							Any  )      ->   List[Any]:







       '''simple docstring'''
       __snake_case :   Any													=    ['的', '人', '有']
       __snake_case :   List[Any]													=    ''.join(__a  )
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                   __snake_case :   List[Any]													=    True
                   __snake_case :   int													=    self.tokenizer_class.from_pretrained(__a      ,     **__a  )
                   __snake_case :   Tuple													=    self.rust_tokenizer_class.from_pretrained(__a      ,     **__a  )

                   __snake_case :   Union[str, Any]													=    tokenizer_p.encode(__a      ,     add_special_tokens=__a  )
                   __snake_case :   Optional[Any]													=    tokenizer_r.encode(__a      ,     add_special_tokens=__a  )

                   __snake_case :   Dict													=    tokenizer_r.convert_ids_to_tokens(__a  )
                   __snake_case :   str													=    tokenizer_p.convert_ids_to_tokens(__a  )

                   # it is expected that each Chinese character is not preceded by "##"
                   self.assertListEqual(__a      ,     __a  )
                   self.assertListEqual(__a      ,     __a  )

                   __snake_case :   Any													=    False
                   __snake_case :   Tuple													=    self.rust_tokenizer_class.from_pretrained(__a      ,     **__a  )
                   __snake_case :   int													=    self.tokenizer_class.from_pretrained(__a      ,     **__a  )

                   __snake_case :   int													=    tokenizer_r.encode(__a      ,     add_special_tokens=__a  )
                   __snake_case :   str													=    tokenizer_p.encode(__a      ,     add_special_tokens=__a  )

                   __snake_case :   Optional[Any]													=    tokenizer_r.convert_ids_to_tokens(__a  )
                   __snake_case :   Optional[Any]													=    tokenizer_p.convert_ids_to_tokens(__a  )

                   # it is expected that only the first Chinese character is not preceded by "##".
                   __snake_case :   Dict													=    [
                       f'''##{token}''' if idx != 0 else token for idx, token in enumerate(__a  )
                   ]
                   self.assertListEqual(__a      ,     __a  )
                   self.assertListEqual(__a      ,     __a  )



