

"""simple docstring"""


import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import cached_download, hf_hub_download, hf_hub_url
from PIL import Image

from transformers import DetaConfig, DetaForObjectDetection, DetaImageProcessor, SwinConfig
from transformers.utils import logging


logging.set_verbosity_info()
UpperCAmelCase													=logging.get_logger(__name__)



def   _A					(	_a	:						Optional[Any]				):



      """simple docstring"""






      A				  =		SwinConfig(
          embed_dim=1_9_2     ,   depths=(2, 2, 1_8, 2)     ,   num_heads=(6, 1_2, 2_4, 4_8)     ,   window_size=1_2     ,   out_features=["""stage2""", """stage3""", """stage4"""]     ,   )

      A				  =		DetaConfig(
          backbone_config=__lowerCAmelCase     ,   num_queries=9_0_0     ,   encoder_ffn_dim=2_0_4_8     ,   decoder_ffn_dim=2_0_4_8     ,   num_feature_levels=5     ,   assign_first_stage=__lowerCAmelCase     ,   with_box_refine=__lowerCAmelCase     ,   two_stage=__lowerCAmelCase     ,   )

      # set labels
      A				  =		"""huggingface/label-files"""
      if "o365" in model_name:
            A				  =		3_6_6
            A				  =		"""object365-id2label.json"""
      else:
            A				  =		9_1
            A				  =		"""coco-detection-id2label.json"""

      A				  =		num_labels
      A				  =		json.load(open(cached_download(hf_hub_url(__lowerCAmelCase     ,   __lowerCAmelCase     ,   repo_type="""dataset"""				)				)     ,   """r"""				)				)
      A				  =		{int(__lowerCAmelCase				): v for k, v in idalabel.items()}
      A				  =		idalabel
      A				  =		{v: k for k, v in idalabel.items()}

      return config



def   _A					(	_a	:						int				):



      """simple docstring"""






      A				  =		[]

      # stem
      # fmt: off
      rename_keys.append(("""backbone.0.body.patch_embed.proj.weight""", """model.backbone.model.embeddings.patch_embeddings.projection.weight""")				)
      rename_keys.append(("""backbone.0.body.patch_embed.proj.bias""", """model.backbone.model.embeddings.patch_embeddings.projection.bias""")				)
      rename_keys.append(("""backbone.0.body.patch_embed.norm.weight""", """model.backbone.model.embeddings.norm.weight""")				)
      rename_keys.append(("""backbone.0.body.patch_embed.norm.bias""", """model.backbone.model.embeddings.norm.bias""")				)
      # stages
      for i in range(len(config.backbone_config.depths				)				):
            for j in range(config.backbone_config.depths[i]				):
                  rename_keys.append((f'backbone.0.body.layers.{i}.blocks.{j}.norm1.weight', f'model.backbone.model.encoder.layers.{i}.blocks.{j}.layernorm_before.weight')				)
                  rename_keys.append((f'backbone.0.body.layers.{i}.blocks.{j}.norm1.bias', f'model.backbone.model.encoder.layers.{i}.blocks.{j}.layernorm_before.bias')				)
                  rename_keys.append((f'backbone.0.body.layers.{i}.blocks.{j}.attn.relative_position_bias_table', f'model.backbone.model.encoder.layers.{i}.blocks.{j}.attention.self.relative_position_bias_table')				)
                  rename_keys.append((f'backbone.0.body.layers.{i}.blocks.{j}.attn.relative_position_index', f'model.backbone.model.encoder.layers.{i}.blocks.{j}.attention.self.relative_position_index')				)
                  rename_keys.append((f'backbone.0.body.layers.{i}.blocks.{j}.attn.proj.weight', f'model.backbone.model.encoder.layers.{i}.blocks.{j}.attention.output.dense.weight')				)
                  rename_keys.append((f'backbone.0.body.layers.{i}.blocks.{j}.attn.proj.bias', f'model.backbone.model.encoder.layers.{i}.blocks.{j}.attention.output.dense.bias')				)
                  rename_keys.append((f'backbone.0.body.layers.{i}.blocks.{j}.norm2.weight', f'model.backbone.model.encoder.layers.{i}.blocks.{j}.layernorm_after.weight')				)
                  rename_keys.append((f'backbone.0.body.layers.{i}.blocks.{j}.norm2.bias', f'model.backbone.model.encoder.layers.{i}.blocks.{j}.layernorm_after.bias')				)
                  rename_keys.append((f'backbone.0.body.layers.{i}.blocks.{j}.mlp.fc1.weight', f'model.backbone.model.encoder.layers.{i}.blocks.{j}.intermediate.dense.weight')				)
                  rename_keys.append((f'backbone.0.body.layers.{i}.blocks.{j}.mlp.fc1.bias', f'model.backbone.model.encoder.layers.{i}.blocks.{j}.intermediate.dense.bias')				)
                  rename_keys.append((f'backbone.0.body.layers.{i}.blocks.{j}.mlp.fc2.weight', f'model.backbone.model.encoder.layers.{i}.blocks.{j}.output.dense.weight')				)
                  rename_keys.append((f'backbone.0.body.layers.{i}.blocks.{j}.mlp.fc2.bias', f'model.backbone.model.encoder.layers.{i}.blocks.{j}.output.dense.bias')				)

            if i < 3:
                  rename_keys.append((f'backbone.0.body.layers.{i}.downsample.reduction.weight', f'model.backbone.model.encoder.layers.{i}.downsample.reduction.weight')				)
                  rename_keys.append((f'backbone.0.body.layers.{i}.downsample.norm.weight', f'model.backbone.model.encoder.layers.{i}.downsample.norm.weight')				)
                  rename_keys.append((f'backbone.0.body.layers.{i}.downsample.norm.bias', f'model.backbone.model.encoder.layers.{i}.downsample.norm.bias')				)

      rename_keys.append(("""backbone.0.body.norm1.weight""", """model.backbone.model.hidden_states_norms.stage2.weight""")				)
      rename_keys.append(("""backbone.0.body.norm1.bias""", """model.backbone.model.hidden_states_norms.stage2.bias""")				)
      rename_keys.append(("""backbone.0.body.norm2.weight""", """model.backbone.model.hidden_states_norms.stage3.weight""")				)
      rename_keys.append(("""backbone.0.body.norm2.bias""", """model.backbone.model.hidden_states_norms.stage3.bias""")				)
      rename_keys.append(("""backbone.0.body.norm3.weight""", """model.backbone.model.hidden_states_norms.stage4.weight""")				)
      rename_keys.append(("""backbone.0.body.norm3.bias""", """model.backbone.model.hidden_states_norms.stage4.bias""")				)

      # transformer encoder
      for i in range(config.encoder_layers				):
            rename_keys.append((f'transformer.encoder.layers.{i}.self_attn.sampling_offsets.weight', f'model.encoder.layers.{i}.self_attn.sampling_offsets.weight')				)
            rename_keys.append((f'transformer.encoder.layers.{i}.self_attn.sampling_offsets.bias', f'model.encoder.layers.{i}.self_attn.sampling_offsets.bias')				)
            rename_keys.append((f'transformer.encoder.layers.{i}.self_attn.attention_weights.weight', f'model.encoder.layers.{i}.self_attn.attention_weights.weight')				)
            rename_keys.append((f'transformer.encoder.layers.{i}.self_attn.attention_weights.bias', f'model.encoder.layers.{i}.self_attn.attention_weights.bias')				)
            rename_keys.append((f'transformer.encoder.layers.{i}.self_attn.value_proj.weight', f'model.encoder.layers.{i}.self_attn.value_proj.weight')				)
            rename_keys.append((f'transformer.encoder.layers.{i}.self_attn.value_proj.bias', f'model.encoder.layers.{i}.self_attn.value_proj.bias')				)
            rename_keys.append((f'transformer.encoder.layers.{i}.self_attn.output_proj.weight', f'model.encoder.layers.{i}.self_attn.output_proj.weight')				)
            rename_keys.append((f'transformer.encoder.layers.{i}.self_attn.output_proj.bias', f'model.encoder.layers.{i}.self_attn.output_proj.bias')				)
            rename_keys.append((f'transformer.encoder.layers.{i}.norm1.weight', f'model.encoder.layers.{i}.self_attn_layer_norm.weight')				)
            rename_keys.append((f'transformer.encoder.layers.{i}.norm1.bias', f'model.encoder.layers.{i}.self_attn_layer_norm.bias')				)
            rename_keys.append((f'transformer.encoder.layers.{i}.linear1.weight', f'model.encoder.layers.{i}.fc1.weight')				)
            rename_keys.append((f'transformer.encoder.layers.{i}.linear1.bias', f'model.encoder.layers.{i}.fc1.bias')				)
            rename_keys.append((f'transformer.encoder.layers.{i}.linear2.weight', f'model.encoder.layers.{i}.fc2.weight')				)
            rename_keys.append((f'transformer.encoder.layers.{i}.linear2.bias', f'model.encoder.layers.{i}.fc2.bias')				)
            rename_keys.append((f'transformer.encoder.layers.{i}.norm2.weight', f'model.encoder.layers.{i}.final_layer_norm.weight')				)
            rename_keys.append((f'transformer.encoder.layers.{i}.norm2.bias', f'model.encoder.layers.{i}.final_layer_norm.bias')				)

      # transformer decoder
      for i in range(config.decoder_layers				):
            rename_keys.append((f'transformer.decoder.layers.{i}.cross_attn.sampling_offsets.weight', f'model.decoder.layers.{i}.encoder_attn.sampling_offsets.weight')				)
            rename_keys.append((f'transformer.decoder.layers.{i}.cross_attn.sampling_offsets.bias', f'model.decoder.layers.{i}.encoder_attn.sampling_offsets.bias')				)
            rename_keys.append((f'transformer.decoder.layers.{i}.cross_attn.attention_weights.weight', f'model.decoder.layers.{i}.encoder_attn.attention_weights.weight')				)
            rename_keys.append((f'transformer.decoder.layers.{i}.cross_attn.attention_weights.bias', f'model.decoder.layers.{i}.encoder_attn.attention_weights.bias')				)
            rename_keys.append((f'transformer.decoder.layers.{i}.cross_attn.value_proj.weight', f'model.decoder.layers.{i}.encoder_attn.value_proj.weight')				)
            rename_keys.append((f'transformer.decoder.layers.{i}.cross_attn.value_proj.bias', f'model.decoder.layers.{i}.encoder_attn.value_proj.bias')				)
            rename_keys.append((f'transformer.decoder.layers.{i}.cross_attn.output_proj.weight', f'model.decoder.layers.{i}.encoder_attn.output_proj.weight')				)
            rename_keys.append((f'transformer.decoder.layers.{i}.cross_attn.output_proj.bias', f'model.decoder.layers.{i}.encoder_attn.output_proj.bias')				)
            rename_keys.append((f'transformer.decoder.layers.{i}.norm1.weight', f'model.decoder.layers.{i}.encoder_attn_layer_norm.weight')				)
            rename_keys.append((f'transformer.decoder.layers.{i}.norm1.bias', f'model.decoder.layers.{i}.encoder_attn_layer_norm.bias')				)
            rename_keys.append((f'transformer.decoder.layers.{i}.self_attn.out_proj.weight', f'model.decoder.layers.{i}.self_attn.out_proj.weight')				)
            rename_keys.append((f'transformer.decoder.layers.{i}.self_attn.out_proj.bias', f'model.decoder.layers.{i}.self_attn.out_proj.bias')				)
            rename_keys.append((f'transformer.decoder.layers.{i}.norm2.weight', f'model.decoder.layers.{i}.self_attn_layer_norm.weight')				)
            rename_keys.append((f'transformer.decoder.layers.{i}.norm2.bias', f'model.decoder.layers.{i}.self_attn_layer_norm.bias')				)
            rename_keys.append((f'transformer.decoder.layers.{i}.linear1.weight', f'model.decoder.layers.{i}.fc1.weight')				)
            rename_keys.append((f'transformer.decoder.layers.{i}.linear1.bias', f'model.decoder.layers.{i}.fc1.bias')				)
            rename_keys.append((f'transformer.decoder.layers.{i}.linear2.weight', f'model.decoder.layers.{i}.fc2.weight')				)
            rename_keys.append((f'transformer.decoder.layers.{i}.linear2.bias', f'model.decoder.layers.{i}.fc2.bias')				)
            rename_keys.append((f'transformer.decoder.layers.{i}.norm3.weight', f'model.decoder.layers.{i}.final_layer_norm.weight')				)
            rename_keys.append((f'transformer.decoder.layers.{i}.norm3.bias', f'model.decoder.layers.{i}.final_layer_norm.bias')				)

      # fmt: on

      return rename_keys



def   _A					(	_a	:						Dict     ,   _a	:						List[Any]     ,   _a	:						Any				):



      """simple docstring"""






      A				  =		dct.pop(__lowerCAmelCase				)
      A				  =		val



def   _A					(	_a	:						str     ,   _a	:						Optional[int]				):



      """simple docstring"""






      A				  =		[int(backbone_config.embed_dim * 2**i				) for i in range(len(backbone_config.depths				)				)]
      for i in range(len(backbone_config.depths				)				):
            A				  =		num_features[i]
            for j in range(backbone_config.depths[i]				):
                  # fmt: off
                  # read in weights + bias of input projection layer (in original implementation, this is a single matrix + bias)
                  A				  =		state_dict.pop(f'backbone.0.body.layers.{i}.blocks.{j}.attn.qkv.weight'				)
                  A				  =		state_dict.pop(f'backbone.0.body.layers.{i}.blocks.{j}.attn.qkv.bias'				)
                  # next, add query, keys and values (in that order) to the state dict
                  A				  =		in_proj_weight[:dim, :]
                  A				  =		in_proj_bias[: dim]
                  A				  =		in_proj_weight[
                      dim : dim * 2, :
                  ]
                  A				  =		in_proj_bias[
                      dim : dim * 2
                  ]
                  A				  =		in_proj_weight[
                      -dim :, :
                  ]
                  A				  =		in_proj_bias[-dim :]
                  # fmt: on



def   _A					(	_a	:						Tuple     ,   _a	:						Tuple				):



      """simple docstring"""






      A				  =		config.d_model
      for i in range(config.decoder_layers				):
            # read in weights + bias of input projection layer of self-attention
            A				  =		state_dict.pop(f'transformer.decoder.layers.{i}.self_attn.in_proj_weight'				)
            A				  =		state_dict.pop(f'transformer.decoder.layers.{i}.self_attn.in_proj_bias'				)
            # next, add query, keys and values (in that order) to the state dict
            A				  =		in_proj_weight[:hidden_size, :]
            A				  =		in_proj_bias[:hidden_size]
            A				  =		in_proj_weight[
                hidden_size : hidden_size * 2, :
            ]
            A				  =		in_proj_bias[hidden_size : hidden_size * 2]
            A				  =		in_proj_weight[-hidden_size:, :]
            A				  =		in_proj_bias[-hidden_size:]



def   _A					(	):



      """simple docstring"""






      A				  =		"""http://images.cocodataset.org/val2017/000000039769.jpg"""
      A				  =		Image.open(requests.get(__lowerCAmelCase     ,   stream=__lowerCAmelCase				).raw				)

      return im



@torch.no_grad()
def   _A					(	_a	:						str     ,   _a	:						str     ,   _a	:						int				):



      """simple docstring"""






      A				  =		get_deta_config(__lowerCAmelCase				)

      # load original state dict
      if model_name == "deta-swin-large":
            A				  =		hf_hub_download(repo_id="""nielsr/deta-checkpoints"""     ,   filename="""adet_swin_ft.pth"""				)
      elif model_name == "deta-swin-large-o365":
            A				  =		hf_hub_download(repo_id="""jozhang97/deta-swin-l-o365"""     ,   filename="""deta_swin_pt_o365.pth"""				)
      else:
            raise ValueError(f'Model name {model_name} not supported'				)

      A				  =		torch.load(__lowerCAmelCase     ,   map_location="""cpu"""				)["""model"""]

      # original state dict
      for name, param in state_dict.items():
            print(__lowerCAmelCase     ,   param.shape				)

      # rename keys
      A				  =		create_rename_keys(__lowerCAmelCase				)
      for src, dest in rename_keys:
            rename_key(__lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase				)
      read_in_swin_q_k_v(__lowerCAmelCase     ,   config.backbone_config				)
      read_in_decoder_q_k_v(__lowerCAmelCase     ,   __lowerCAmelCase				)

      # fix some prefixes
      for key in state_dict.copy().keys():
            if "transformer.decoder.class_embed" in key or "transformer.decoder.bbox_embed" in key:
                  A				  =		state_dict.pop(__lowerCAmelCase				)
                  A				  =		val
            if "input_proj" in key:
                  A				  =		state_dict.pop(__lowerCAmelCase				)
                  A				  =		val
            if "level_embed" in key or "pos_trans" in key or "pix_trans" in key or "enc_output" in key:
                  A				  =		state_dict.pop(__lowerCAmelCase				)
                  A				  =		val

    # finally, create HuggingFace model and load state dict
      A				  =		DetaForObjectDetection(__lowerCAmelCase				)
      model.load_state_dict(__lowerCAmelCase				)
      model.eval()

      A				  =		"""cuda""" if torch.cuda.is_available() else """cpu"""
      model.to(__lowerCAmelCase				)

      # load image processor
      A				  =		DetaImageProcessor(format="""coco_detection"""				)

      # verify our conversion on image
      A				  =		prepare_img()
      A				  =		processor(images=__lowerCAmelCase     ,   return_tensors="""pt"""				)
      A				  =		encoding["""pixel_values"""]
      A				  =		model(pixel_values.to(__lowerCAmelCase				)				)

      # verify logits
      print("""Logits:"""     ,   outputs.logits[0, :3, :3]				)
      print("""Boxes:"""     ,   outputs.pred_boxes[0, :3, :3]				)
      if model_name == "deta-swin-large":
            A				  =		torch.tensor(
                [[-7.63_08, -2.84_85, -5.37_37], [-7.20_37, -4.55_05, -4.80_27], [-7.29_43, -4.26_11, -4.66_17]]				)
            A				  =		torch.tensor([[0.49_87, 0.49_69, 0.99_99], [0.25_49, 0.54_98, 0.48_05], [0.54_98, 0.27_57, 0.05_69]]				)
      elif model_name == "deta-swin-large-o365":
            A				  =		torch.tensor(
                [[-8.01_22, -3.57_20, -4.97_17], [-8.15_47, -3.68_86, -4.63_89], [-7.66_10, -3.61_94, -5.01_34]]				)
            A				  =		torch.tensor([[0.25_23, 0.55_49, 0.48_81], [0.77_15, 0.41_49, 0.46_01], [0.55_03, 0.27_53, 0.05_75]]				)
      assert torch.allclose(outputs.logits[0, :3, :3]     ,   expected_logits.to(__lowerCAmelCase				)     ,   atol=1E-4				)
      assert torch.allclose(outputs.pred_boxes[0, :3, :3]     ,   expected_boxes.to(__lowerCAmelCase				)     ,   atol=1E-4				)
      print("""Everything ok!"""				)

      if pytorch_dump_folder_path:
            # Save model and processor
            logger.info(f'Saving PyTorch model and processor to {pytorch_dump_folder_path}...'				)
            Path(__lowerCAmelCase				).mkdir(exist_ok=__lowerCAmelCase				)
            model.save_pretrained(__lowerCAmelCase				)
            processor.save_pretrained(__lowerCAmelCase				)

      # Push to hub
      if push_to_hub:
            print("""Pushing model and processor to hub..."""				)
            model.push_to_hub(f'jozhang97/{model_name}'				)
            processor.push_to_hub(f'jozhang97/{model_name}'				)


if __name__ == "__main__":
  UpperCAmelCase													=argparse.ArgumentParser()

  parser.add_argument(
      "--model_name",
      type=str,
      default="deta-swin-large",
      choices=["deta-swin-large", "deta-swin-large-o365"],
      help="Name of the model you'd like to convert.",
  )
  parser.add_argument(
      "--pytorch_dump_folder_path",
      default=None,
      type=str,
      help="Path to the folder to output PyTorch model.",
  )
  parser.add_argument(
      "--push_to_hub", action="store_true", help="Whether or not to push the converted model to the 🤗 hub."
  )
  UpperCAmelCase													=parser.parse_args()
  convert_deta_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)











'''simple docstring'''




import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_SCREAMING_SNAKE_CASE							     =			get_tests_dir("fixtures/test_sentencepiece.model")
@require_sentencepiece
@require_tokenizers
class 			_lowerCAmelCase		(			A__							,     unittest.TestCase	):






     """simple docstring"""






     snake_case_						      =   XLNetTokenizer
     snake_case_						      =   XLNetTokenizerFast
     snake_case_						      =   True
     snake_case_						      =   True



     def      lowerCAmelCase			(   self							:			List[Any]						)->	Union[str, Any]:
        super().setUp()

        # We have a SentencePiece fixture for testing
        snake_case 						=   XLNetTokenizer(__snake_case   ,       keep_accents=__snake_case						)
        tokenizer.sanitize_special_tokens()
        tokenizer.save_pretrained(self.tmpdirname						)



     def      lowerCAmelCase			(   self							:			List[Any]						)->	str:
        snake_case 						=   """<s>"""
        snake_case 						=   1

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(__snake_case						)   ,       __snake_case						)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(__snake_case						)   ,       __snake_case						)



     def      lowerCAmelCase			(   self							:			Tuple						)->	str:
        snake_case 						=   list(self.get_tokenizer().get_vocab().keys()						)

        self.assertEqual(vocab_keys[0]   ,       """<unk>"""						)
        self.assertEqual(vocab_keys[1]   ,       """<s>"""						)
        self.assertEqual(vocab_keys[-1]   ,       """<eod>"""						)
        self.assertEqual(len(__snake_case						)   ,       10_06						)



     def      lowerCAmelCase			(   self							:			Optional[Any]						)->	Tuple:
        self.assertEqual(self.get_tokenizer().vocab_size   ,       10_00						)



     def      lowerCAmelCase			(   self							:			List[str]						)->	int:
        snake_case 						=   XLNetTokenizer(__snake_case   ,       keep_accents=__snake_case						)

        snake_case 						=   tokenizer.tokenize("""This is a test"""						)
        self.assertListEqual(__snake_case   ,       ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]						)

        self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case						)   ,       [2_85, 46, 10, 1_70, 3_82]						)

        snake_case 						=   tokenizer.tokenize("""I was born in 92000, and this is falsé."""						)
        self.assertListEqual(
            __snake_case   ,       [
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """9""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """s""",
                """é""",
                """.""",
            ]   ,       )
        snake_case 						=   tokenizer.convert_tokens_to_ids(__snake_case						)
        self.assertListEqual(__snake_case   ,       [8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]						)

        snake_case 						=   tokenizer.convert_ids_to_tokens(__snake_case						)
        self.assertListEqual(
            __snake_case   ,       [
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """<unk>""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """s""",
                """<unk>""",
                """.""",
            ]   ,       )



     def      lowerCAmelCase			(   self							:			str						)->	str:
        snake_case 						=   XLNetTokenizer(__snake_case   ,       do_lower_case=__snake_case						)
        snake_case 						=   tokenizer.tokenize("""I was born in 92000, and this is falsé."""						)
        self.assertListEqual(
            __snake_case   ,       [
                SPIECE_UNDERLINE + """""",
                """i""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """9""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """se""",
                """.""",
            ]   ,       )
        self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""						)   ,       ["""▁he""", """ll""", """o"""]						)



     def      lowerCAmelCase			(   self							:			List[str]						)->	str:
        snake_case 						=   XLNetTokenizer(__snake_case   ,       do_lower_case=__snake_case						)
        snake_case 						=   tokenizer.tokenize("""I was born in 92000, and this is falsé."""						)
        self.assertListEqual(
            __snake_case   ,       [
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """9""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """se""",
                """.""",
            ]   ,       )



     @slow
     def      lowerCAmelCase			(   self							:			Optional[Any]						)->	Dict:
        snake_case 						=   XLNetTokenizer.from_pretrained("""xlnet-base-cased"""						)

        snake_case 						=   tokenizer.encode("""sequence builders"""   ,       add_special_tokens=__snake_case						)
        snake_case 						=   tokenizer.encode("""multi-sequence build"""   ,       add_special_tokens=__snake_case						)

        snake_case 						=   tokenizer.build_inputs_with_special_tokens(__snake_case						)
        snake_case 						=   tokenizer.build_inputs_with_special_tokens(__snake_case   ,       __snake_case						)

        assert encoded_sentence == text + [4, 3]
        assert encoded_pair == text + [4] + text_a + [4, 3]



     @slow
     def      lowerCAmelCase			(   self							:			Optional[int]						)->	Union[str, Any]:
        # fmt: off
        snake_case 						=   {"""input_ids""": [[17, 2_14_42, 2_70, 17, 10, 1_46_45, 3_18, 34, 17, 45_46, 31_45, 7_87, 13, 77_52, 2_20_18, 23, 21, 17, 45_46, 31_45, 7_87, 13, 33_52, 1_44_31, 13, 55_00, 11, 11_76, 5_80, 13, 1_68_19, 47_97, 23, 17, 10, 1_71_35, 6_58, 19, 4_57, 79_32, 13, 1_84, 19, 31_54, 1_71_35, 64_68, 19, 14_04, 1_22_69, 19, 42_29, 53_56, 1_62_64, 46, 19, 17, 2_05_45, 1_03_95, 9, 9, 9, 11, 28, 64_21, 95_31, 2_07_29, 17, 10, 3_53, 1_70_22, 11, 21, 64_21, 95_31, 1_69_49, 17, 10, 1_15_09, 7_53, 11, 33, 95, 24_21, 73_85, 9_56, 1_44_31, 26_26, 25, 8_42, 73_85, 48_36, 21, 14_29, 22_72, 98_55, 31_20, 1_61, 2_47_38, 19, 1_32_03, 6_58, 2_18, 7_87, 21, 4_30, 1_84_82, 8_47, 26_37, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_22, 2_21_78, 27, 10_64, 22, 9_56, 13, 1_11_01, 14_29, 58_54, 2_43_13, 1_89_53, 40, 4_22, 2_43_66, 68, 17_58, 37, 1_04_83, 1_42_57, 31, 2_07, 2_63, 21, 2_03, 37_73, 25, 71, 97_35, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 20_49, 34_42, 17, 1_38_94, 33_80, 23, 95, 18, 1_76_34, 22_88, 9, 4, 3]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=__snake_case   ,       model_name="""xlnet-base-cased"""   ,       revision="""c841166438c31ec7ca9a106dee7bb312b73ae511"""   ,       )

