
import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCamelCase				=			get_tests_dir("""fixtures/test_sentencepiece.model""")






@require_sentencepiece
@require_tokenizers
class     UpperCAmelCase						(		A_			,unittest.TestCase					):
   A__   :						List[str]			=	ReformerTokenizer
   A__   :						List[str]			=	ReformerTokenizerFast
   A__   :						Tuple			=	True
   A__   :						List[Any]			=	False
   A__   :						List[Any]			=	True



   def 							_SCREAMING_SNAKE_CASE  (self						:   List[Any]					)						->      List[Any]:


          '''simple docstring'''


          super().setUp()

          snake_case			:  Dict     							= ReformerTokenizer(snake_case__       ,		keep_accents=snake_case__					)
          tokenizer.save_pretrained(self.tmpdirname					)



   def 							_SCREAMING_SNAKE_CASE  (self						:   List[str]					)						->      List[str]:


          '''simple docstring'''


          snake_case			:  Tuple     							= "<s>"
          snake_case			:  int     							= 1

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case__					)       ,		snake_case__					)
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case__					)       ,		snake_case__					)



   def 							_SCREAMING_SNAKE_CASE  (self						:   Optional[int]					)						->      Dict:


          '''simple docstring'''


          snake_case			:  Optional[int]     							= list(self.get_tokenizer().get_vocab().keys()					)

          self.assertEqual(vocab_keys[0]       ,		"<unk>"					)
          self.assertEqual(vocab_keys[1]       ,		"<s>"					)
          self.assertEqual(vocab_keys[-1]       ,		"j"					)
          self.assertEqual(len(snake_case__					)       ,		10_00					)



   def 							_SCREAMING_SNAKE_CASE  (self						:   Dict					)						->      List[Any]:


          '''simple docstring'''


          self.assertEqual(self.get_tokenizer().vocab_size       ,		10_00					)



   def 							_SCREAMING_SNAKE_CASE  (self						:   int					)						->      Optional[Any]:


          '''simple docstring'''


          if not self.test_rust_tokenizer:
                 return

          snake_case			:  List[Any]     							= self.get_tokenizer()
          snake_case			:  Dict     							= self.get_rust_tokenizer()

          snake_case			:  List[Any]     							= "I was born in 92000, and this is falsé."

          snake_case			:  int     							= tokenizer.tokenize(snake_case__					)
          snake_case			:  Dict     							= rust_tokenizer.tokenize(snake_case__					)
          self.assertListEqual(snake_case__       ,		snake_case__					)

          snake_case			:  str     							= tokenizer.encode(snake_case__       ,		add_special_tokens=snake_case__					)
          snake_case			:  int     							= rust_tokenizer.encode(snake_case__       ,		add_special_tokens=snake_case__					)
          self.assertListEqual(snake_case__       ,		snake_case__					)

          snake_case			:  Optional[int]     							= self.get_rust_tokenizer()
          snake_case			:  Optional[Any]     							= tokenizer.encode(snake_case__					)
          snake_case			:  List[Any]     							= rust_tokenizer.encode(snake_case__					)
          self.assertListEqual(snake_case__       ,		snake_case__					)



   def 							_SCREAMING_SNAKE_CASE  (self						:   str       ,		snake_case__						:   Optional[int]=15					)						->      Optional[int]:


          '''simple docstring'''


          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""					):
                        snake_case			:  Any     							= self.rust_tokenizer_class.from_pretrained(snake_case__       ,		**snake_case__					)

                        # Simple input
                        snake_case			:  Tuple     							= "This is a simple input"
                        snake_case			:  Optional[int]     							= ["This is a simple input 1", "This is a simple input 2"]
                        snake_case			:  str     							= ("This is a simple input", "This is a pair")
                        snake_case			:  Union[str, Any]     							= [
                            ("This is a simple input 1", "This is a simple input 2"),
                            ("This is a simple pair 1", "This is a simple pair 2"),
                        ]

                        # Simple input tests
                        self.assertRaises(snake_case__       ,		tokenizer_r.encode       ,		snake_case__       ,		max_length=snake_case__       ,		padding="max_length"					)

                        # Simple input
                        self.assertRaises(snake_case__       ,		tokenizer_r.encode_plus       ,		snake_case__       ,		max_length=snake_case__       ,		padding="max_length"					)

                        # Simple input
                        self.assertRaises(
                            snake_case__       ,		tokenizer_r.batch_encode_plus       ,		snake_case__       ,		max_length=snake_case__       ,		padding="max_length"       ,		)

                        # Pair input
                        self.assertRaises(snake_case__       ,		tokenizer_r.encode       ,		snake_case__       ,		max_length=snake_case__       ,		padding="max_length"					)

                        # Pair input
                        self.assertRaises(snake_case__       ,		tokenizer_r.encode_plus       ,		snake_case__       ,		max_length=snake_case__       ,		padding="max_length"					)

                        # Pair input
                        self.assertRaises(
                            snake_case__       ,		tokenizer_r.batch_encode_plus       ,		snake_case__       ,		max_length=snake_case__       ,		padding="max_length"       ,		)



   def 							_SCREAMING_SNAKE_CASE  (self						:   List[Any]					)						->      str:


          '''simple docstring'''


          pass



   def 							_SCREAMING_SNAKE_CASE  (self						:   Optional[int]					)						->      Optional[Any]:


          '''simple docstring'''


          snake_case			:  List[str]     							= ReformerTokenizer(snake_case__       ,		keep_accents=snake_case__					)

          snake_case			:  List[str]     							= tokenizer.tokenize("This is a test"					)
          self.assertListEqual(snake_case__       ,		["▁This", "▁is", "▁a", "▁t", "est"]					)

          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(snake_case__					)       ,		[2_85, 46, 10, 1_70, 3_82]       ,		)

          snake_case			:  str     							= tokenizer.tokenize("I was born in 92000, and this is falsé."					)
          self.assertListEqual(
              snake_case__       ,		[
                  SPIECE_UNDERLINE + "I",
                  SPIECE_UNDERLINE + "was",
                  SPIECE_UNDERLINE + "b",
                  "or",
                  "n",
                  SPIECE_UNDERLINE + "in",
                  SPIECE_UNDERLINE + "",
                  "9",
                  "2",
                  "0",
                  "0",
                  "0",
                  ",",
                  SPIECE_UNDERLINE + "and",
                  SPIECE_UNDERLINE + "this",
                  SPIECE_UNDERLINE + "is",
                  SPIECE_UNDERLINE + "f",
                  "al",
                  "s",
                  "é",
                  ".",
              ]       ,		)
          snake_case			:  List[Any]     							= tokenizer.convert_tokens_to_ids(snake_case__					)
          self.assertListEqual(
              snake_case__       ,		[8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]       ,		)

          snake_case			:  int     							= tokenizer.convert_ids_to_tokens(snake_case__					)
          self.assertListEqual(
              snake_case__       ,		[
                  SPIECE_UNDERLINE + "I",
                  SPIECE_UNDERLINE + "was",
                  SPIECE_UNDERLINE + "b",
                  "or",
                  "n",
                  SPIECE_UNDERLINE + "in",
                  SPIECE_UNDERLINE + "",
                  "<unk>",
                  "2",
                  "0",
                  "0",
                  "0",
                  ",",
                  SPIECE_UNDERLINE + "and",
                  SPIECE_UNDERLINE + "this",
                  SPIECE_UNDERLINE + "is",
                  SPIECE_UNDERLINE + "f",
                  "al",
                  "s",
                  "<unk>",
                  ".",
              ]       ,		)



   @cached_property
   def 							_SCREAMING_SNAKE_CASE  (self						:   Any					)						->      Dict:


          '''simple docstring'''


          return ReformerTokenizer.from_pretrained("google/reformer-crime-and-punishment"					)



   @slow
   def 							_SCREAMING_SNAKE_CASE  (self						:   Any					)						->      Union[str, Any]:


          '''simple docstring'''


          snake_case			:  Any     							= "Hello World!"
          snake_case			:  int     							= [1_26, 32, 2_62, 1_52, 38, 72, 2_87]

          self.assertListEqual(snake_case__       ,		self.big_tokenizer.encode(snake_case__					)					)



   @slow
   def 							_SCREAMING_SNAKE_CASE  (self						:   Union[str, Any]					)						->      Optional[int]:


          '''simple docstring'''


          snake_case			:  str     							= (
              "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
              " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
          )
          snake_case			:  Optional[int]     							= [
              1_08,
              2_65,
              24,
              1_11,
              4,
              2_58,
              1_56,
              35,
              28,
              2_75,
              3,
              2_59,
              2_97,
              2_60,
              84,
              4,
              35,
              1_10,
              44,
              8,
              2_59,
              91,
              2_68,
              21,
              11,
              2_09,
              2_74,
              1_09,
              2_66,
              2_77,
              1_17,
              86,
              93,
              3_15,
              2_58,
              2_78,
              2_58,
              2_77,
              2_58,
              0,
              2_58,
              2_88,
              2_58,
              3_19,
              2_58,
              0,
              2_58,
              0,
              2_58,
              0,
              2_58,
              0,
              2_58,
              2_87,
              2_58,
              3_15,
              2_58,
              2_89,
              2_58,
              2_78,
              99,
              2_69,
              2_66,
              2_62,
              8,
              2_59,
              2_41,
              4,
              2_17,
              2_30,
              2_68,
              2_66,
              55,
              1_68,
              1_06,
              75,
              1_93,
              2_66,
              2_23,
              27,
              49,
              26,
              2_82,
              25,
              2_64,
              2_99,
              19,
              26,
              0,
              2_58,
              2_77,
              1_17,
              86,
              93,
              1_76,
              1_83,
              2_70,
              11,
              2_62,
              42,
              61,
              2_65,
          ]

          self.assertListEqual(snake_case__       ,		self.big_tokenizer.encode(snake_case__					)					)



   @require_torch
   @slow
   def 							_SCREAMING_SNAKE_CASE  (self						:   Any					)						->      List[Any]:


          '''simple docstring'''


          import torch

          from transformers import ReformerConfig, ReformerModel

          # Build sequence
          snake_case			:  str     							= list(self.big_tokenizer.get_vocab().keys()					)[:10]
          snake_case			:  str     							= " ".join(snake_case__					)
          snake_case			:  Any     							= self.big_tokenizer.encode_plus(snake_case__       ,		return_tensors="pt"					)
          snake_case			:  Optional[Any]     							= self.big_tokenizer.batch_encode_plus([sequence, sequence]       ,		return_tensors="pt"					)

          snake_case			:  Tuple     							= ReformerConfig()
          # The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
          snake_case			:  Tuple     							= encoded_sequence["input_ids"].shape
          snake_case			:  int     							= ReformerModel(snake_case__					)

          # Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
          assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

          with torch.no_grad():
                 model(**snake_case__					)
                 model(**snake_case__					)



   @slow
   def 							_SCREAMING_SNAKE_CASE  (self						:   str					)						->      Optional[int]:


          '''simple docstring'''


          snake_case			:  Union[str, Any]     							= {"input_ids": [[1_08, 2_65, 24, 1_11, 4, 2_58, 1_56, 7, 51, 2_79, 58, 7, 76, 25, 69, 2_78], [1_40, 2_43, 2_64, 1_34, 17, 2_67, 77, 2_63, 22, 2_62, 2_97, 2_58, 3_04, 1_77, 2_79, 2_66, 14, 89, 13, 35, 2_61, 2_99, 2_72, 1_37, 2_75, 2_78]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
          # fmt: on

          # This tokenizer does not know some characters like ")".
          # That is the reason why we use very simple texts here.
          # Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
          snake_case			:  Optional[int]     							= [
              "This is a very simple sentence.",
              "The quick brown fox jumps over the lazy dog.",
          ]

          self.tokenizer_integration_test_util(
              expected_encoding=snake_case__       ,		model_name="google/reformer-crime-and-punishment"       ,		revision="0e6c3decb8211d49bf881013425dc8b0448b3f5a"       ,		padding=snake_case__       ,		sequences=snake_case__       ,		)







import argparse
from collections import OrderedDict
from pathlib import Path

import torch

from transformers import (
    VisualBertConfig,
    VisualBertForMultipleChoice,
    VisualBertForPreTraining,
    VisualBertForQuestionAnswering,
    VisualBertForVisualReasoning,
)
from transformers.utils import logging


logging.set_verbosity_info()
__lowerCamelCase				=			logging.get_logger(__name__)

__lowerCamelCase				=			[
    ("""bert.bert""", """visual_bert"""),
    ("""bert.cls""", """cls"""),
    ("""bert.classifier""", """cls"""),
    ("""token_type_embeddings_visual""", """visual_token_type_embeddings"""),
    ("""position_embeddings_visual""", """visual_position_embeddings"""),
    ("""projection""", """visual_projection"""),
]

__lowerCamelCase				=			[
    """nlvr2_coco_pre_trained.th""",
    """nlvr2_fine_tuned.th""",
    """nlvr2_pre_trained.th""",
    """vcr_coco_pre_train.th""",
    """vcr_fine_tune.th""",
    """vcr_pre_train.th""",
    """vqa_coco_pre_trained.th""",
    """vqa_fine_tuned.th""",
    """vqa_pre_trained.th""",
]


def      UpperCamelCase   (       __lowerCamelCase	:					int       ):
       snake_case			:  List[Any]     							= torch.load(__lowerCamelCase							,   map_location="cpu"       )
       return sd


def      UpperCamelCase   (       __lowerCamelCase	:					List[Any]							,   __lowerCamelCase	:					Optional[int]							,   __lowerCamelCase	:					Optional[Any]=rename_keys_prefix       ):
       snake_case			:  Optional[Any]     							= OrderedDict()
       snake_case			:  Optional[int]     							= torch.arange(config.max_position_embeddings       ).expand((1, -1)       )
       # detector_d = OrderedDict()
       for key in d:
              if "detector" in key:
                     # detector_d[key.replace('detector.','')] = d[key]
                     continue
              snake_case			:  Any     							= key
              for name_pair in rename_keys_prefix:
                     snake_case			:  List[Any]     							= new_key.replace(name_pair[0]							,   name_pair[1]       )
              snake_case			:  List[str]     							= d[key]
              if key == "bert.cls.predictions.decoder.weight":
                     # Old bert code didn't have `decoder.bias`, but was added separately
                     snake_case			:  int     							= new_d["cls.predictions.bias"]
       return new_d


@torch.no_grad()
def      UpperCamelCase   (       __lowerCamelCase	:					List[Any]							,   __lowerCamelCase	:					Optional[Any]       ):
       assert (
           checkpoint_path.split("/"       )[-1] in ACCEPTABLE_CHECKPOINTS
       ), f"""The checkpoint provided must be in {ACCEPTABLE_CHECKPOINTS}."""

       # Get Config
       if "pre" in checkpoint_path:
              snake_case			:  Optional[Any]     							= "pretraining"
              if "vcr" in checkpoint_path:
                     snake_case			:  Optional[int]     							= {"visual_embedding_dim": 512}
              elif "vqa_advanced" in checkpoint_path:
                     snake_case			:  int     							= {"visual_embedding_dim": 2048}
              elif "vqa" in checkpoint_path:
                     snake_case			:  Dict     							= {"visual_embedding_dim": 2048}
              elif "nlvr" in checkpoint_path:
                     snake_case			:  Tuple     							= {"visual_embedding_dim": 1024}
              else:
                     raise NotImplementedError(f"""No implementation found for `{checkpoint_path}`."""       )
       else:
              if "vcr" in checkpoint_path:
                     snake_case			:  Optional[Any]     							= {"visual_embedding_dim": 512}
                     snake_case			:  List[str]     							= "multichoice"
              elif "vqa_advanced" in checkpoint_path:
                     snake_case			:  Dict     							= {"visual_embedding_dim": 2048}
                     snake_case			:  Tuple     							= "vqa_advanced"
              elif "vqa" in checkpoint_path:
                     snake_case			:  Optional[Any]     							= {"visual_embedding_dim": 2048, "num_labels": 3129}
                     snake_case			:  Tuple     							= "vqa"
              elif "nlvr" in checkpoint_path:
                     snake_case			:  str     							= {
                         "visual_embedding_dim": 1024,
                         "num_labels": 2,
                     }
                     snake_case			:  str     							= "nlvr"

       snake_case			:  int     							= VisualBertConfig(**__lowerCamelCase       )

       # Load State Dict
       snake_case			:  Any     							= load_state_dict(__lowerCamelCase       )

       snake_case			:  List[Any]     							= get_new_dict(__lowerCamelCase							,   __lowerCamelCase       )

       if model_type == "pretraining":
              snake_case			:  str     							= VisualBertForPreTraining(__lowerCamelCase       )
       elif model_type == "vqa":
              snake_case			:  Any     							= VisualBertForQuestionAnswering(__lowerCamelCase       )
       elif model_type == "nlvr":
              snake_case			:  Optional[Any]     							= VisualBertForVisualReasoning(__lowerCamelCase       )
       elif model_type == "multichoice":
              snake_case			:  Optional[int]     							= VisualBertForMultipleChoice(__lowerCamelCase       )

       model.load_state_dict(__lowerCamelCase       )
       # Save Checkpoints
       Path(__lowerCamelCase       ).mkdir(exist_ok=__lowerCamelCase       )
       model.save_pretrained(__lowerCamelCase       )


if __name__ == "__main__":
   __lowerCamelCase				=			argparse.ArgumentParser()
   # Required parameters
   parser.add_argument("""orig_checkpoint_path""", type=str, help="""A path to .th on local filesystem.""")
   parser.add_argument("""pytorch_dump_folder_path""", type=str, help="""Path to the output PyTorch model.""")
   __lowerCamelCase				=			parser.parse_args()
   convert_visual_bert_checkpoint(args.orig_checkpoint_path, args.pytorch_dump_folder_path)





