





import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import (
    BertTokenizer,
    ViltConfig,
    ViltForImageAndTextRetrieval,
    ViltForImagesAndTextClassification,
    ViltForMaskedLM,
    ViltForQuestionAnswering,
    ViltImageProcessor,
    ViltProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
a__      :     Optional[int]         =		logging.get_logger(__name__)

def      UpperCAmelCase_    (    _UpperCAmelCase							:List[str]  ,						_UpperCAmelCase							:Union[str, Any]=False  ,						_UpperCAmelCase							:Optional[int]=False  ,						_UpperCAmelCase							:int=False				)	->				Dict:
     '''simple docstring'''


     A_          =       []
     for i in range(config.num_hidden_layers				):
          # encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
          rename_keys.append((f'transformer.blocks.{i}.norm1.weight', f'vilt.encoder.layer.{i}.layernorm_before.weight')				)
          rename_keys.append((f'transformer.blocks.{i}.norm1.bias', f'vilt.encoder.layer.{i}.layernorm_before.bias')				)
          rename_keys.append(
              (f'transformer.blocks.{i}.attn.proj.weight', f'vilt.encoder.layer.{i}.attention.output.dense.weight')				)
          rename_keys.append(
              (f'transformer.blocks.{i}.attn.proj.bias', f'vilt.encoder.layer.{i}.attention.output.dense.bias')				)
          rename_keys.append((f'transformer.blocks.{i}.norm2.weight', f'vilt.encoder.layer.{i}.layernorm_after.weight')				)
          rename_keys.append((f'transformer.blocks.{i}.norm2.bias', f'vilt.encoder.layer.{i}.layernorm_after.bias')				)
          rename_keys.append(
              (f'transformer.blocks.{i}.mlp.fc1.weight', f'vilt.encoder.layer.{i}.intermediate.dense.weight')				)
          rename_keys.append((f'transformer.blocks.{i}.mlp.fc1.bias', f'vilt.encoder.layer.{i}.intermediate.dense.bias')				)
          rename_keys.append((f'transformer.blocks.{i}.mlp.fc2.weight', f'vilt.encoder.layer.{i}.output.dense.weight')				)
          rename_keys.append((f'transformer.blocks.{i}.mlp.fc2.bias', f'vilt.encoder.layer.{i}.output.dense.bias')				)

     # embeddings
     rename_keys.extend(
         [
             # text embeddings
             ('''text_embeddings.word_embeddings.weight''', '''vilt.embeddings.text_embeddings.word_embeddings.weight'''),
             (
                 '''text_embeddings.position_embeddings.weight''',
                 '''vilt.embeddings.text_embeddings.position_embeddings.weight''',
             ),
             ('''text_embeddings.position_ids''', '''vilt.embeddings.text_embeddings.position_ids'''),
             (
                 '''text_embeddings.token_type_embeddings.weight''',
                 '''vilt.embeddings.text_embeddings.token_type_embeddings.weight''',
             ),
             ('''text_embeddings.LayerNorm.weight''', '''vilt.embeddings.text_embeddings.LayerNorm.weight'''),
             ('''text_embeddings.LayerNorm.bias''', '''vilt.embeddings.text_embeddings.LayerNorm.bias'''),
             # patch embeddings
             ('''transformer.cls_token''', '''vilt.embeddings.cls_token'''),
             ('''transformer.patch_embed.proj.weight''', '''vilt.embeddings.patch_embeddings.projection.weight'''),
             ('''transformer.patch_embed.proj.bias''', '''vilt.embeddings.patch_embeddings.projection.bias'''),
             ('''transformer.pos_embed''', '''vilt.embeddings.position_embeddings'''),
             # token type embeddings
             ('''token_type_embeddings.weight''', '''vilt.embeddings.token_type_embeddings.weight'''),
         ]				)

     # final layernorm + pooler
     rename_keys.extend(
         [
             ('''transformer.norm.weight''', '''vilt.layernorm.weight'''),
             ('''transformer.norm.bias''', '''vilt.layernorm.bias'''),
             ('''pooler.dense.weight''', '''vilt.pooler.dense.weight'''),
             ('''pooler.dense.bias''', '''vilt.pooler.dense.bias'''),
         ]				)

     # classifier head(s)
     if vqa_model:
          # classification head
          rename_keys.extend(
              [
                  ('''vqa_classifier.0.weight''', '''classifier.0.weight'''),
                  ('''vqa_classifier.0.bias''', '''classifier.0.bias'''),
                  ('''vqa_classifier.1.weight''', '''classifier.1.weight'''),
                  ('''vqa_classifier.1.bias''', '''classifier.1.bias'''),
                  ('''vqa_classifier.3.weight''', '''classifier.3.weight'''),
                  ('''vqa_classifier.3.bias''', '''classifier.3.bias'''),
              ]				)
     elif nlvr_model:
          # classification head
          rename_keys.extend(
              [
                  ('''nlvr2_classifier.0.weight''', '''classifier.0.weight'''),
                  ('''nlvr2_classifier.0.bias''', '''classifier.0.bias'''),
                  ('''nlvr2_classifier.1.weight''', '''classifier.1.weight'''),
                  ('''nlvr2_classifier.1.bias''', '''classifier.1.bias'''),
                  ('''nlvr2_classifier.3.weight''', '''classifier.3.weight'''),
                  ('''nlvr2_classifier.3.bias''', '''classifier.3.bias'''),
              ]				)
     else:
          pass

     return rename_keys

def      UpperCAmelCase_    (    _UpperCAmelCase							:Optional[int]  ,						_UpperCAmelCase							:Any				)	->				Any:
     '''simple docstring'''


     for i in range(config.num_hidden_layers				):
          A_          =       'vilt.'
          # read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
          A_          =       state_dict.pop(f'transformer.blocks.{i}.attn.qkv.weight'				)
          A_          =       state_dict.pop(f'transformer.blocks.{i}.attn.qkv.bias'				)
          # next, add query, keys and values (in that order) to the state dict
          A_          =       in_proj_weight[
              : config.hidden_size, :
          ]
          A_          =       in_proj_bias[: config.hidden_size]
          A_          =       in_proj_weight[
              config.hidden_size : config.hidden_size * 2, :
          ]
          A_          =       in_proj_bias[
              config.hidden_size : config.hidden_size * 2
          ]
          A_          =       in_proj_weight[
              -config.hidden_size :, :
          ]
          A_          =       in_proj_bias[-config.hidden_size :]

def      UpperCAmelCase_    (    _UpperCAmelCase							:Tuple				)	->				Tuple:
     '''simple docstring'''


     A_          =       ['head.weight', 'head.bias']
     for k in ignore_keys:
          state_dict.pop(_UpperCAmelCase  ,						_UpperCAmelCase				)

def      UpperCAmelCase_    (    _UpperCAmelCase							:int  ,						_UpperCAmelCase							:List[str]  ,						_UpperCAmelCase							:int				)	->				Any:
     '''simple docstring'''


     A_          =       dct.pop(_UpperCAmelCase				)
     A_          =       val

@torch.no_grad()
def      UpperCAmelCase_    (    _UpperCAmelCase							:Any  ,						_UpperCAmelCase							:Dict				)	->				Union[str, Any]:
     '''simple docstring'''


     A_          =       ViltConfig(image_size=384  ,						patch_size=32  ,						tie_word_embeddings=_UpperCAmelCase				)
     A_          =       False
     A_          =       False
     A_          =       False
     A_          =       False
     if "vqa" in checkpoint_url:
          A_          =       True
          A_          =       3129
          A_          =       'huggingface/label-files'
          A_          =       'vqa2-id2label.json'
          A_          =       json.load(open(hf_hub_download(_UpperCAmelCase  ,						_UpperCAmelCase  ,						repo_type='''dataset'''				)  ,						'''r'''				)				)
          A_          =       {int(_UpperCAmelCase				): v for k, v in idalabel.items()}
          A_          =       idalabel
          A_          =       {v: k for k, v in idalabel.items()}
          A_          =       ViltForQuestionAnswering(_UpperCAmelCase				)
     elif "nlvr" in checkpoint_url:
          A_          =       True
          A_          =       2
          A_          =       {0: 'False', 1: 'True'}
          A_          =       {v: k for k, v in config.idalabel.items()}
          A_          =       3
          A_          =       ViltForImagesAndTextClassification(_UpperCAmelCase				)
     elif "irtr" in checkpoint_url:
          A_          =       True
          A_          =       ViltForImageAndTextRetrieval(_UpperCAmelCase				)
     elif "mlm_itm" in checkpoint_url:
          A_          =       True
          A_          =       ViltForMaskedLM(_UpperCAmelCase				)
     else:
          raise ValueError('''Unknown model type'''				)

     # load state_dict of original model, remove and rename some keys
     A_          =       torch.hub.load_state_dict_from_url(_UpperCAmelCase  ,						map_location='''cpu'''				)['state_dict']
     A_          =       create_rename_keys(_UpperCAmelCase  ,						_UpperCAmelCase  ,						_UpperCAmelCase  ,						_UpperCAmelCase				)
     for src, dest in rename_keys:
          rename_key(_UpperCAmelCase  ,						_UpperCAmelCase  ,						_UpperCAmelCase				)
     read_in_q_k_v(_UpperCAmelCase  ,						_UpperCAmelCase				)
     if mlm_model or irtr_model:
          A_          =       ['itm_score.fc.weight', 'itm_score.fc.bias']
          for k in ignore_keys:
               state_dict.pop(_UpperCAmelCase  ,						_UpperCAmelCase				)

    # load state dict into HuggingFace model
     model.eval()
     if mlm_model:
          A_          =       model.load_state_dict(_UpperCAmelCase  ,						strict=_UpperCAmelCase				)
          assert missing_keys == ["mlm_score.decoder.bias"]
     else:
          model.load_state_dict(_UpperCAmelCase				)

     # Define processor
     A_          =       ViltImageProcessor(size=384				)
     A_          =       BertTokenizer.from_pretrained('''bert-base-uncased'''				)
     A_          =       ViltProcessor(_UpperCAmelCase  ,						_UpperCAmelCase				)

     # Forward pass on example inputs (image + text)
     if nlvr_model:
          A_          =       Image.open(requests.get('''https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg'''  ,						stream=_UpperCAmelCase				).raw				)
          A_          =       Image.open(requests.get('''https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg'''  ,						stream=_UpperCAmelCase				).raw				)
          A_          =       (
              'The left image contains twice the number of dogs as the right image, and at least two dogs in total are'
              ' standing.'
          )
          A_          =       processor(_UpperCAmelCase  ,						_UpperCAmelCase  ,						return_tensors='''pt'''				)
          A_          =       processor(_UpperCAmelCase  ,						_UpperCAmelCase  ,						return_tensors='''pt'''				)
          A_          =       model(
              input_ids=encoding_a.input_ids  ,						pixel_values=encoding_a.pixel_values  ,						pixel_values_a=encoding_a.pixel_values  ,						)
     else:
          A_          =       Image.open(requests.get('''http://images.cocodataset.org/val2017/000000039769.jpg'''  ,						stream=_UpperCAmelCase				).raw				)
          if mlm_model:
               A_          =       'a bunch of [MASK] laying on a [MASK].'
          else:
               A_          =       'How many cats are there?'
          A_          =       processor(_UpperCAmelCase  ,						_UpperCAmelCase  ,						return_tensors='''pt'''				)
          A_          =       model(**_UpperCAmelCase				)

     # Verify outputs
     if mlm_model:
          A_          =       torch.Size([1, 11, 30522]				)
          A_          =       torch.tensor([-12.5061, -12.5123, -12.5174]				)
          assert outputs.logits.shape == expected_shape
          assert torch.allclose(outputs.logits[0, 0, :3]  ,						_UpperCAmelCase  ,						atol=1e-4				)

          # verify masked token prediction equals "cats"
          A_          =       outputs.logits[0, 4, :].argmax(-1				).item()
          assert tokenizer.decode([predicted_id]				) == "cats"
     elif vqa_model:
          A_          =       torch.Size([1, 3129]				)
          A_          =       torch.tensor([-15.9495, -18.1472, -10.3041]				)
          assert torch.allclose(outputs.logits[0, :3]  ,						_UpperCAmelCase  ,						atol=1e-4				)
          assert outputs.logits.shape == expected_shape
          assert torch.allclose(outputs.logits[0, 0, :3]  ,						_UpperCAmelCase  ,						atol=1e-4				)

          # verify vqa prediction equals "2"
          A_          =       outputs.logits.argmax(-1				).item()
          assert model.config.idalabel[predicted_idx] == "2"
     elif nlvr_model:
          A_          =       torch.Size([1, 2]				)
          A_          =       torch.tensor([-2.8721, 2.1291]				)
          assert torch.allclose(outputs.logits[0, :3]  ,						_UpperCAmelCase  ,						atol=1e-4				)
          assert outputs.logits.shape == expected_shape

     Path(_UpperCAmelCase				).mkdir(exist_ok=_UpperCAmelCase				)
     print(f'Saving model and processor to {pytorch_dump_folder_path}'				)
     model.save_pretrained(_UpperCAmelCase				)
     processor.save_pretrained(_UpperCAmelCase				)


if __name__ == "__main__":
      a__      :     List[Any]         =		argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          '--checkpoint_url',
          default='https://github.com/dandelin/ViLT/releases/download/200k/vilt_200k_mlm_itm.ckpt',
          type=str,
          help='URL of the checkpoint you\'d like to convert.',
      )
      parser.add_argument(
          '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
      )

      a__      :     List[str]         =		parser.parse_args()
      convert_vilt_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)








'''simple docstring'''





import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class  snake_case__						(    SCREAMING_SNAKE_CASE_  ,     unittest.TestCase				):
 A__				= BertTokenizer
 A__				= BertTokenizerFast
 A__				= True
 A__				= True
 A__				= filter_non_english
 def A_   (   self						:							Optional[Any]  )      ->   Dict:







       '''simple docstring'''
       super().setUp()

       __snake_case :   List[Any]													=    [
           '[UNK]',
           '[CLS]',
           '[SEP]',
           '[PAD]',
           '[MASK]',
           'want',
           '##want',
           '##ed',
           'wa',
           'un',
           'runn',
           '##ing',
           ',',
           'low',
           'lowest',
       ]
       __snake_case :   Optional[int]													=    os.path.join(self.tmpdirname      ,     VOCAB_FILES_NAMES['vocab_file']  )
       with open(self.vocab_file      ,     'w'      ,     encoding='utf-8'  ) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )
 def A_   (   self						:							Dict      ,     __a						:							Optional[int]  )      ->   int:







       '''simple docstring'''
       __snake_case :   List[Any]													=    'UNwant\u00E9d,running'
       __snake_case :   Tuple													=    'unwanted, running'
       return input_text, output_text
 def A_   (   self						:							Tuple  )      ->   Dict:







       '''simple docstring'''
       __snake_case :   Any													=    self.tokenizer_class(self.vocab_file  )

       __snake_case :   Tuple													=    tokenizer.tokenize('UNwant\u00E9d,running'  )
       self.assertListEqual(__a      ,     ['un', '##want', '##ed', ',', 'runn', '##ing']  )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__a  )      ,     [9, 6, 7, 12, 10, 11]  )
 def A_   (   self						:							Optional[Any]  )      ->   Tuple:







       '''simple docstring'''
       if not self.test_rust_tokenizer:
             return

       __snake_case :   int													=    self.get_tokenizer()
       __snake_case :   Optional[Any]													=    self.get_rust_tokenizer()

       __snake_case :   int													=    'UNwant\u00E9d,running'

       __snake_case :   Any													=    tokenizer.tokenize(__a  )
       __snake_case :   Dict													=    rust_tokenizer.tokenize(__a  )
       self.assertListEqual(__a      ,     __a  )

       __snake_case :   Optional[int]													=    tokenizer.encode(__a      ,     add_special_tokens=__a  )
       __snake_case :   Optional[int]													=    rust_tokenizer.encode(__a      ,     add_special_tokens=__a  )
       self.assertListEqual(__a      ,     __a  )

       __snake_case :   str													=    self.get_rust_tokenizer()
       __snake_case :   Tuple													=    tokenizer.encode(__a  )
       __snake_case :   Tuple													=    rust_tokenizer.encode(__a  )
       self.assertListEqual(__a      ,     __a  )

       # With lower casing
       __snake_case :   Union[str, Any]													=    self.get_tokenizer(do_lower_case=__a  )
       __snake_case :   Any													=    self.get_rust_tokenizer(do_lower_case=__a  )

       __snake_case :   Optional[int]													=    'UNwant\u00E9d,running'

       __snake_case :   List[Any]													=    tokenizer.tokenize(__a  )
       __snake_case :   List[str]													=    rust_tokenizer.tokenize(__a  )
       self.assertListEqual(__a      ,     __a  )

       __snake_case :   List[str]													=    tokenizer.encode(__a      ,     add_special_tokens=__a  )
       __snake_case :   Union[str, Any]													=    rust_tokenizer.encode(__a      ,     add_special_tokens=__a  )
       self.assertListEqual(__a      ,     __a  )

       __snake_case :   Union[str, Any]													=    self.get_rust_tokenizer()
       __snake_case :   Dict													=    tokenizer.encode(__a  )
       __snake_case :   str													=    rust_tokenizer.encode(__a  )
       self.assertListEqual(__a      ,     __a  )
 def A_   (   self						:							Tuple  )      ->   Optional[int]:







       '''simple docstring'''
       __snake_case :   Optional[Any]													=    BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'  )      ,     ['ah', '\u535A', '\u63A8', 'zz']  )
 def A_   (   self						:							List[str]  )      ->   Any:







       '''simple docstring'''
       __snake_case :   Optional[Any]													=    BasicTokenizer(do_lower_case=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )      ,     ['hello', '!', 'how', 'are', 'you', '?']  )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )      ,     ['hello']  )
 def A_   (   self						:							str  )      ->   int:







       '''simple docstring'''
       __snake_case :   Dict													=    BasicTokenizer(do_lower_case=__a      ,     strip_accents=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,     ['hällo', '!', 'how', 'are', 'you', '?']  )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )      ,     ['h\u00E9llo']  )
 def A_   (   self						:							List[str]  )      ->   Tuple:







       '''simple docstring'''
       __snake_case :   str													=    BasicTokenizer(do_lower_case=__a      ,     strip_accents=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,     ['hallo', '!', 'how', 'are', 'you', '?']  )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )      ,     ['hello']  )
 def A_   (   self						:							str  )      ->   str:







       '''simple docstring'''
       __snake_case :   Tuple													=    BasicTokenizer(do_lower_case=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,     ['hallo', '!', 'how', 'are', 'you', '?']  )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )      ,     ['hello']  )
 def A_   (   self						:							str  )      ->   Optional[Any]:







       '''simple docstring'''
       __snake_case :   int													=    BasicTokenizer(do_lower_case=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )      ,     ['HeLLo', '!', 'how', 'Are', 'yoU', '?']  )
 def A_   (   self						:							Dict  )      ->   List[str]:







       '''simple docstring'''
       __snake_case :   Any													=    BasicTokenizer(do_lower_case=__a      ,     strip_accents=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,     ['HäLLo', '!', 'how', 'Are', 'yoU', '?']  )
 def A_   (   self						:							Dict  )      ->   Union[str, Any]:







       '''simple docstring'''
       __snake_case :   List[Any]													=    BasicTokenizer(do_lower_case=__a      ,     strip_accents=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,     ['HaLLo', '!', 'how', 'Are', 'yoU', '?']  )
 def A_   (   self						:							Optional[Any]  )      ->   List[Any]:







       '''simple docstring'''
       __snake_case :   List[str]													=    BasicTokenizer(do_lower_case=__a      ,     never_split=['[UNK]']  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'  )      ,     ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']  )
 def A_   (   self						:							Optional[Any]  )      ->   Optional[int]:







       '''simple docstring'''
       __snake_case :   int													=    BasicTokenizer()
       __snake_case :   List[str]													=    'a\n\'ll !!to?\'d of, can\'t.'
       __snake_case :   str													=    ['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
       self.assertListEqual(tokenizer.tokenize(__a  )      ,     __a  )
 def A_   (   self						:							List[Any]  )      ->   Tuple:







       '''simple docstring'''
       __snake_case :   str													=    ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

       __snake_case :   Dict													=    {}
       for i, token in enumerate(__a  ):
             __snake_case :   str													=    i
       __snake_case :   List[str]													=    WordpieceTokenizer(vocab=__a      ,     unk_token='[UNK]'  )

       self.assertListEqual(tokenizer.tokenize(''  )      ,     []  )

       self.assertListEqual(tokenizer.tokenize('unwanted running'  )      ,     ['un', '##want', '##ed', 'runn', '##ing']  )

       self.assertListEqual(tokenizer.tokenize('unwantedX running'  )      ,     ['[UNK]', 'runn', '##ing']  )
 def A_   (   self						:							Dict  )      ->   List[str]:







       '''simple docstring'''
       self.assertTrue(_is_whitespace(' '  )  )
       self.assertTrue(_is_whitespace('\t'  )  )
       self.assertTrue(_is_whitespace('\r'  )  )
       self.assertTrue(_is_whitespace('\n'  )  )
       self.assertTrue(_is_whitespace('\u00A0'  )  )

       self.assertFalse(_is_whitespace('A'  )  )
       self.assertFalse(_is_whitespace('-'  )  )
 def A_   (   self						:							List[str]  )      ->   List[Any]:







       '''simple docstring'''
       self.assertTrue(_is_control('\u0005'  )  )

       self.assertFalse(_is_control('A'  )  )
       self.assertFalse(_is_control(' '  )  )
       self.assertFalse(_is_control('\t'  )  )
       self.assertFalse(_is_control('\r'  )  )
 def A_   (   self						:							Any  )      ->   Union[str, Any]:







       '''simple docstring'''
       self.assertTrue(_is_punctuation('-'  )  )
       self.assertTrue(_is_punctuation('$'  )  )
       self.assertTrue(_is_punctuation('`'  )  )
       self.assertTrue(_is_punctuation('.'  )  )

       self.assertFalse(_is_punctuation('A'  )  )
       self.assertFalse(_is_punctuation(' '  )  )
 def A_   (   self						:							List[Any]  )      ->   Tuple:







       '''simple docstring'''
       __snake_case :   str													=    self.get_tokenizer()
       __snake_case :   int													=    self.get_rust_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(__a  ) for t in ['Test', '\xad', 'test']]      ,     [['[UNK]'], [], ['[UNK]']]  )

       self.assertListEqual(
           [rust_tokenizer.tokenize(__a  ) for t in ['Test', '\xad', 'test']]      ,     [['[UNK]'], [], ['[UNK]']]  )
 @slow
 def A_   (   self						:							List[str]  )      ->   List[Any]:







       '''simple docstring'''
       __snake_case :   Dict													=    self.tokenizer_class.from_pretrained('bert-base-uncased'  )

       __snake_case :   Dict													=    tokenizer.encode('sequence builders'      ,     add_special_tokens=__a  )
       __snake_case :   int													=    tokenizer.encode('multi-sequence build'      ,     add_special_tokens=__a  )

       __snake_case :   int													=    tokenizer.build_inputs_with_special_tokens(__a  )
       __snake_case :   Union[str, Any]													=    tokenizer.build_inputs_with_special_tokens(__a      ,     __a  )

       assert encoded_sentence == [101] + text + [102]
       assert encoded_pair == [101] + text + [102] + text_a + [102]
 def A_   (   self						:							Optional[int]  )      ->   str:







       '''simple docstring'''
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                   __snake_case :   Optional[int]													=    self.rust_tokenizer_class.from_pretrained(__a      ,     **__a  )

                   __snake_case :   Optional[Any]													=    f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                   __snake_case :   Tuple													=    tokenizer_r.encode_plus(
                       __a      ,     return_attention_mask=__a      ,     return_token_type_ids=__a      ,     return_offsets_mapping=__a      ,     add_special_tokens=__a      ,     )

                   __snake_case :   Optional[Any]													=    tokenizer_r.do_lower_case if hasattr(__a      ,     'do_lower_case'  ) else False
                   __snake_case :   Any													=    (
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), 'A'),
                           ((1, 2), ','),
                           ((3, 5), 'na'),
                           ((5, 6), '##ï'),
                           ((6, 8), '##ve'),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), 'Allen'),
                           ((21, 23), '##NL'),
                           ((23, 24), '##P'),
                           ((25, 33), 'sentence'),
                           ((33, 34), '.'),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), 'a'),
                           ((1, 2), ','),
                           ((3, 8), 'naive'),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), 'allen'),
                           ((21, 23), '##nl'),
                           ((23, 24), '##p'),
                           ((25, 33), 'sentence'),
                           ((33, 34), '.'),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results]      ,     tokenizer_r.convert_ids_to_tokens(tokens['input_ids']  )  )
                   self.assertEqual([e[0] for e in expected_results]      ,     tokens['offset_mapping']  )





 def A_   (   self						:							Any  )      ->   List[Any]:







       '''simple docstring'''
       __snake_case :   Any													=    ['的', '人', '有']
       __snake_case :   List[Any]													=    ''.join(__a  )
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                   __snake_case :   List[Any]													=    True
                   __snake_case :   int													=    self.tokenizer_class.from_pretrained(__a      ,     **__a  )
                   __snake_case :   Tuple													=    self.rust_tokenizer_class.from_pretrained(__a      ,     **__a  )

                   __snake_case :   Union[str, Any]													=    tokenizer_p.encode(__a      ,     add_special_tokens=__a  )
                   __snake_case :   Optional[Any]													=    tokenizer_r.encode(__a      ,     add_special_tokens=__a  )

                   __snake_case :   Dict													=    tokenizer_r.convert_ids_to_tokens(__a  )
                   __snake_case :   str													=    tokenizer_p.convert_ids_to_tokens(__a  )

                   # it is expected that each Chinese character is not preceded by "##"
                   self.assertListEqual(__a      ,     __a  )
                   self.assertListEqual(__a      ,     __a  )

                   __snake_case :   Any													=    False
                   __snake_case :   Tuple													=    self.rust_tokenizer_class.from_pretrained(__a      ,     **__a  )
                   __snake_case :   int													=    self.tokenizer_class.from_pretrained(__a      ,     **__a  )

                   __snake_case :   int													=    tokenizer_r.encode(__a      ,     add_special_tokens=__a  )
                   __snake_case :   str													=    tokenizer_p.encode(__a      ,     add_special_tokens=__a  )

                   __snake_case :   Optional[Any]													=    tokenizer_r.convert_ids_to_tokens(__a  )
                   __snake_case :   Optional[Any]													=    tokenizer_p.convert_ids_to_tokens(__a  )

                   # it is expected that only the first Chinese character is not preceded by "##".
                   __snake_case :   Dict													=    [
                       f'''##{token}''' if idx != 0 else token for idx, token in enumerate(__a  )
                   ]
                   self.assertListEqual(__a      ,     __a  )
                   self.assertListEqual(__a      ,     __a  )



