





"""simple docstring"""
import baseaa
import io
import json
import os
from copy import deepcopy

from ..optimizer import AcceleratedOptimizer
from ..scheduler import AcceleratedScheduler







class       a							:



 def __init__(     self				,       _snake_case     ):







  """simple docstring"""






  if isinstance(snake_case__				,       snake_case__     ):
   # Don't modify user's data should they want to reuse it (e.g. in tests), because once we
   # modified it, it will not be accepted here again, since `auto` values would have been overridden
   lowerCAmelCase    				= deepcopy(snake_case__     )
  elif os.path.exists(snake_case__     ):
   with io.open(snake_case__				,       'r'				,       encoding='utf-8'     ) as f:
    lowerCAmelCase    				= json.load(snake_case__     )
  else:
   try:
    lowerCAmelCase    				= baseaa.urlsafe_baadecode(snake_case__     ).decode('utf-8'     )
    lowerCAmelCase    				= json.loads(snake_case__     )
   except (UnicodeDecodeError, AttributeError, ValueError):
    raise ValueError(
        F'Expected a string path to an existing deepspeed config, or a dictionary, or a base64 encoded string. Received: {config_file_or_dict}'     )

  lowerCAmelCase    				= config

  self.set_stage_and_offload()



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  lowerCAmelCase    				= self.get_value('zero_optimization.stage'				,       -1     )

  # offload
  lowerCAmelCase    				= False
  if self.is_zeroa() or self.is_zeroa():
   lowerCAmelCase    				= set(['cpu', 'nvme']     )
   lowerCAmelCase    				= set(
       [
           self.get_value('zero_optimization.offload_optimizer.device'     ),
           self.get_value('zero_optimization.offload_param.device'     ),
       ]     )
   if len(offload_devices & offload_devices_valid     ) > 0:
    lowerCAmelCase    				= True



 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  lowerCAmelCase    				= self.config

  # find the config node of interest if it exists
  lowerCAmelCase    				= ds_key_long.split('.'     )
  lowerCAmelCase    				= nodes.pop()
  for node in nodes:
   lowerCAmelCase    				= config.get(snake_case__     )
   if config is None:
    return None, ds_key

  return config, ds_key



 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case=None     ):







  """simple docstring"""






  lowerCAmelCase    				= self.find_config_node(snake_case__     )
  if config is None:
   return default
  return config.get(snake_case__				,       snake_case__     )



 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case=False     ):







  """simple docstring"""






  lowerCAmelCase    				= self.config

  # find the config node of interest if it exists
  lowerCAmelCase    				= ds_key_long.split('.'     )
  for node in nodes:
   lowerCAmelCase    				= config
   lowerCAmelCase    				= config.get(snake_case__     )
   if config is None:
    if must_exist:
     raise ValueError(F'Can\'t find {ds_key_long} entry in the config: {self.config}'     )
    else:
     return

        # if found remove it
  if parent_config is not None:
   parent_config.pop(snake_case__     )



 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  lowerCAmelCase    				= self.get_value(snake_case__     )
  return False if value is None else bool(snake_case__     )



 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  lowerCAmelCase    				= self.get_value(snake_case__     )
  return False if value is None else not bool(snake_case__     )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  return self._stage == 2



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  return self._stage == 3



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  return self._offload







class       a							:



 def __init__(     self				,       _snake_case     ):







  """simple docstring"""






  lowerCAmelCase    				= engine



 def    UpperCamelCase__  (     self				,       _snake_case				,       **_snake_case     ):







  """simple docstring"""






  self.engine.backward(snake_case__				,       **snake_case__     )

  # Deepspeed's `engine.step` performs the following operations:
  # - gradient accumulation check
  # - gradient clipping
  # - optimizer step
  # - zero grad
  # - checking overflow
  # - lr_scheduler step (only if engine.lr_scheduler is not None)
  self.engine.step()
  # and this plugin overrides the above calls with no-ops when Accelerate runs under
  # Deepspeed, but allows normal functionality for non-Deepspeed cases thus enabling a simple
  # training loop that works transparently under many training regimes.







class       a							(   A_  ):



 def __init__(     self				,       _snake_case     ):







  """simple docstring"""






  super().__init__(snake_case__				,       device_placement=snake_case__				,       scaler=snake_case__     )
  lowerCAmelCase    				= hasattr(self.optimizer				,       'overflow'     )



 def    UpperCamelCase__  (     self				,       _snake_case=None     ):







  """simple docstring"""






  pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed



 @property
 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  if self.__has_overflow__:
   return self.optimizer.overflow
  return False







class       a							(   A_  ):



 def __init__(     self				,       _snake_case				,       _snake_case     ):







  """simple docstring"""






  super().__init__(snake_case__				,       snake_case__     )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed







class       a							:



 def __init__(     self				,       _snake_case				,       _snake_case=0.001				,       _snake_case=0				,       **_snake_case     ):







  """simple docstring"""






  lowerCAmelCase    				= params
  lowerCAmelCase    				= lr
  lowerCAmelCase    				= weight_decay
  lowerCAmelCase    				= kwargs







class       a							:



 def __init__(     self				,       _snake_case				,       _snake_case=None				,       _snake_case=0				,       **_snake_case     ):







  """simple docstring"""






  lowerCAmelCase    				= optimizer
  lowerCAmelCase    				= total_num_steps
  lowerCAmelCase    				= warmup_num_steps
  lowerCAmelCase    				= kwargs







import argparse
from collections import OrderedDict
from pathlib import Path

import torch

from transformers import (
    VisualBertConfig,
    VisualBertForMultipleChoice,
    VisualBertForPreTraining,
    VisualBertForQuestionAnswering,
    VisualBertForVisualReasoning,
)
from transformers.utils import logging


logging.set_verbosity_info()
__lowerCamelCase				=			logging.get_logger(__name__)

__lowerCamelCase				=			[
    ("""bert.bert""", """visual_bert"""),
    ("""bert.cls""", """cls"""),
    ("""bert.classifier""", """cls"""),
    ("""token_type_embeddings_visual""", """visual_token_type_embeddings"""),
    ("""position_embeddings_visual""", """visual_position_embeddings"""),
    ("""projection""", """visual_projection"""),
]

__lowerCamelCase				=			[
    """nlvr2_coco_pre_trained.th""",
    """nlvr2_fine_tuned.th""",
    """nlvr2_pre_trained.th""",
    """vcr_coco_pre_train.th""",
    """vcr_fine_tune.th""",
    """vcr_pre_train.th""",
    """vqa_coco_pre_trained.th""",
    """vqa_fine_tuned.th""",
    """vqa_pre_trained.th""",
]


def      UpperCamelCase   (       __lowerCamelCase	:					int       ):
       snake_case			:  List[Any]     							= torch.load(__lowerCamelCase							,   map_location="cpu"       )
       return sd


def      UpperCamelCase   (       __lowerCamelCase	:					List[Any]							,   __lowerCamelCase	:					Optional[int]							,   __lowerCamelCase	:					Optional[Any]=rename_keys_prefix       ):
       snake_case			:  Optional[Any]     							= OrderedDict()
       snake_case			:  Optional[int]     							= torch.arange(config.max_position_embeddings       ).expand((1, -1)       )
       # detector_d = OrderedDict()
       for key in d:
              if "detector" in key:
                     # detector_d[key.replace('detector.','')] = d[key]
                     continue
              snake_case			:  Any     							= key
              for name_pair in rename_keys_prefix:
                     snake_case			:  List[Any]     							= new_key.replace(name_pair[0]							,   name_pair[1]       )
              snake_case			:  List[str]     							= d[key]
              if key == "bert.cls.predictions.decoder.weight":
                     # Old bert code didn't have `decoder.bias`, but was added separately
                     snake_case			:  int     							= new_d["cls.predictions.bias"]
       return new_d


@torch.no_grad()
def      UpperCamelCase   (       __lowerCamelCase	:					List[Any]							,   __lowerCamelCase	:					Optional[Any]       ):
       assert (
           checkpoint_path.split("/"       )[-1] in ACCEPTABLE_CHECKPOINTS
       ), f"""The checkpoint provided must be in {ACCEPTABLE_CHECKPOINTS}."""

       # Get Config
       if "pre" in checkpoint_path:
              snake_case			:  Optional[Any]     							= "pretraining"
              if "vcr" in checkpoint_path:
                     snake_case			:  Optional[int]     							= {"visual_embedding_dim": 512}
              elif "vqa_advanced" in checkpoint_path:
                     snake_case			:  int     							= {"visual_embedding_dim": 2048}
              elif "vqa" in checkpoint_path:
                     snake_case			:  Dict     							= {"visual_embedding_dim": 2048}
              elif "nlvr" in checkpoint_path:
                     snake_case			:  Tuple     							= {"visual_embedding_dim": 1024}
              else:
                     raise NotImplementedError(f"""No implementation found for `{checkpoint_path}`."""       )
       else:
              if "vcr" in checkpoint_path:
                     snake_case			:  Optional[Any]     							= {"visual_embedding_dim": 512}
                     snake_case			:  List[str]     							= "multichoice"
              elif "vqa_advanced" in checkpoint_path:
                     snake_case			:  Dict     							= {"visual_embedding_dim": 2048}
                     snake_case			:  Tuple     							= "vqa_advanced"
              elif "vqa" in checkpoint_path:
                     snake_case			:  Optional[Any]     							= {"visual_embedding_dim": 2048, "num_labels": 3129}
                     snake_case			:  Tuple     							= "vqa"
              elif "nlvr" in checkpoint_path:
                     snake_case			:  str     							= {
                         "visual_embedding_dim": 1024,
                         "num_labels": 2,
                     }
                     snake_case			:  str     							= "nlvr"

       snake_case			:  int     							= VisualBertConfig(**__lowerCamelCase       )

       # Load State Dict
       snake_case			:  Any     							= load_state_dict(__lowerCamelCase       )

       snake_case			:  List[Any]     							= get_new_dict(__lowerCamelCase							,   __lowerCamelCase       )

       if model_type == "pretraining":
              snake_case			:  str     							= VisualBertForPreTraining(__lowerCamelCase       )
       elif model_type == "vqa":
              snake_case			:  Any     							= VisualBertForQuestionAnswering(__lowerCamelCase       )
       elif model_type == "nlvr":
              snake_case			:  Optional[Any]     							= VisualBertForVisualReasoning(__lowerCamelCase       )
       elif model_type == "multichoice":
              snake_case			:  Optional[int]     							= VisualBertForMultipleChoice(__lowerCamelCase       )

       model.load_state_dict(__lowerCamelCase       )
       # Save Checkpoints
       Path(__lowerCamelCase       ).mkdir(exist_ok=__lowerCamelCase       )
       model.save_pretrained(__lowerCamelCase       )


if __name__ == "__main__":
   __lowerCamelCase				=			argparse.ArgumentParser()
   # Required parameters
   parser.add_argument("""orig_checkpoint_path""", type=str, help="""A path to .th on local filesystem.""")
   parser.add_argument("""pytorch_dump_folder_path""", type=str, help="""Path to the output PyTorch model.""")
   __lowerCamelCase				=			parser.parse_args()
   convert_visual_bert_checkpoint(args.orig_checkpoint_path, args.pytorch_dump_folder_path)





