



"""simple docstring"""
import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version(""">=""", FSDP_PYTORCH_VERSION):
     import torch.distributed.checkpoint as dist_cp
     from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
     from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
     from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
     from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


SCREAMING_SNAKE_CASE_				= get_logger(__name__)



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__=0	)			->      int:
  os.makedirs(SCREAMING_SNAKE_CASE_, exist_ok=SCREAMING_SNAKE_CASE_	)
  with FSDP.state_dict_type(
      SCREAMING_SNAKE_CASE_, fsdp_plugin.state_dict_type, fsdp_plugin.state_dict_config, fsdp_plugin.optim_state_dict_config	):
    a_     :      str 						=     model.state_dict()
    if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
      a_     :      List[Any] 						=     F"""{MODEL_NAME}.bin""" if model_index == 0 else F"""{MODEL_NAME}_{model_index}.bin"""
      a_     :      int 						=     os.path.join(SCREAMING_SNAKE_CASE_, SCREAMING_SNAKE_CASE_	)
      if accelerator.process_index == 0:
        logger.info(F"""Saving model to {output_model_file}"""	)
        torch.save(SCREAMING_SNAKE_CASE_, SCREAMING_SNAKE_CASE_	)
        logger.info(F"""Model saved to {output_model_file}"""	)
    elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
      a_     :      str 						=     (
          F"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
          if model_index == 0
          else F"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
      )
      a_     :      int 						=     os.path.join(SCREAMING_SNAKE_CASE_, SCREAMING_SNAKE_CASE_	)
      logger.info(F"""Saving model to {output_model_file}"""	)
      torch.save(SCREAMING_SNAKE_CASE_, SCREAMING_SNAKE_CASE_	)
      logger.info(F"""Model saved to {output_model_file}"""	)
    elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
      a_     :      Optional[Any] 						=     os.path.join(SCREAMING_SNAKE_CASE_, F"""{MODEL_NAME}_{model_index}"""	)
      os.makedirs(SCREAMING_SNAKE_CASE_, exist_ok=SCREAMING_SNAKE_CASE_	)
      logger.info(F"""Saving model to {ckpt_dir}"""	)
      a_     :      Any 						=     {"model": state_dict}

      dist_cp.save_state_dict(
          state_dict=SCREAMING_SNAKE_CASE_, storage_writer=dist_cp.FileSystemWriter(SCREAMING_SNAKE_CASE_	), planner=DefaultSavePlanner(), )
      logger.info(F"""Model saved to {ckpt_dir}"""	)



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__=0	)			->      Union[str, Any]:
  accelerator.wait_for_everyone()
  with FSDP.state_dict_type(
      SCREAMING_SNAKE_CASE_, fsdp_plugin.state_dict_type, fsdp_plugin.state_dict_config, fsdp_plugin.optim_state_dict_config	):
    if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
      if type(SCREAMING_SNAKE_CASE_	) != FSDP and accelerator.process_index != 0:
        if not fsdp_plugin.sync_module_states:
          raise ValueError(
              "Set the `sync_module_states` flag to `True` so that model states are synced across processes when "
              "initializing FSDP object"	)
        return
      a_     :      int 						=     F"""{MODEL_NAME}.bin""" if model_index == 0 else F"""{MODEL_NAME}_{model_index}.bin"""
      a_     :      Optional[Any] 						=     os.path.join(SCREAMING_SNAKE_CASE_, SCREAMING_SNAKE_CASE_	)
      logger.info(F"""Loading model from {input_model_file}"""	)
      a_     :      Dict 						=     torch.load(SCREAMING_SNAKE_CASE_	)
      logger.info(F"""Model loaded from {input_model_file}"""	)
    elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
      a_     :      int 						=     (
          F"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
          if model_index == 0
          else F"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
      )
      a_     :      Dict 						=     os.path.join(SCREAMING_SNAKE_CASE_, SCREAMING_SNAKE_CASE_	)
      logger.info(F"""Loading model from {input_model_file}"""	)
      a_     :      List[str] 						=     torch.load(SCREAMING_SNAKE_CASE_	)
      logger.info(F"""Model loaded from {input_model_file}"""	)
    elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
      a_     :      Tuple 						=     (
          os.path.join(SCREAMING_SNAKE_CASE_, F"""{MODEL_NAME}_{model_index}"""	)
          if F"""{MODEL_NAME}""" not in input_dir
          else input_dir
      )
      logger.info(F"""Loading model from {ckpt_dir}"""	)
      a_     :      Dict 						=     {"model": model.state_dict()}
      dist_cp.load_state_dict(
          state_dict=SCREAMING_SNAKE_CASE_, storage_reader=dist_cp.FileSystemReader(SCREAMING_SNAKE_CASE_	), planner=DefaultLoadPlanner(), )
      a_     :      int 						=     state_dict["model"]
      logger.info(F"""Model loaded from {ckpt_dir}"""	)
    model.load_state_dict(SCREAMING_SNAKE_CASE_	)



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__=0	)			->      str:
  os.makedirs(SCREAMING_SNAKE_CASE_, exist_ok=SCREAMING_SNAKE_CASE_	)
  with FSDP.state_dict_type(
      SCREAMING_SNAKE_CASE_, fsdp_plugin.state_dict_type, fsdp_plugin.state_dict_config, fsdp_plugin.optim_state_dict_config	):
    a_     :      Tuple 						=     FSDP.optim_state_dict(SCREAMING_SNAKE_CASE_, SCREAMING_SNAKE_CASE_	)
    if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
      if accelerator.process_index == 0:
        a_     :      List[Any] 						=     (
            F"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else F"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
        )
        a_     :      str 						=     os.path.join(SCREAMING_SNAKE_CASE_, SCREAMING_SNAKE_CASE_	)
        logger.info(F"""Saving Optimizer state to {output_optimizer_file}"""	)
        torch.save(SCREAMING_SNAKE_CASE_, SCREAMING_SNAKE_CASE_	)
        logger.info(F"""Optimizer state saved in {output_optimizer_file}"""	)
    else:
      a_     :      Dict 						=     os.path.join(SCREAMING_SNAKE_CASE_, F"""{OPTIMIZER_NAME}_{optimizer_index}"""	)
      os.makedirs(SCREAMING_SNAKE_CASE_, exist_ok=SCREAMING_SNAKE_CASE_	)
      logger.info(F"""Saving Optimizer state to {ckpt_dir}"""	)
      dist_cp.save_state_dict(
          state_dict={"optimizer": optim_state}, storage_writer=dist_cp.FileSystemWriter(SCREAMING_SNAKE_CASE_	), planner=DefaultSavePlanner(), )
      logger.info(F"""Optimizer state saved in {ckpt_dir}"""	)



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__=0	)			->      List[Any]:
  accelerator.wait_for_everyone()
  with FSDP.state_dict_type(
      SCREAMING_SNAKE_CASE_, fsdp_plugin.state_dict_type, fsdp_plugin.state_dict_config, fsdp_plugin.optim_state_dict_config	):
    if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
      a_     :      Optional[Any] 						=     None
      # below check should work but currently it isn't working (mostly opytorch issue),
      # in the meantime disabling it at the cost of excess memory usage
      # if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
      a_     :      Union[str, Any] 						=     (
          F"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else F"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
      )
      a_     :      Optional[int] 						=     os.path.join(SCREAMING_SNAKE_CASE_, SCREAMING_SNAKE_CASE_	)
      logger.info(F"""Loading Optimizer state from {input_optimizer_file}"""	)
      a_     :      Optional[int] 						=     torch.load(SCREAMING_SNAKE_CASE_	)
      logger.info(F"""Optimizer state loaded from {input_optimizer_file}"""	)
    else:
      a_     :      Tuple 						=     (
          os.path.join(SCREAMING_SNAKE_CASE_, F"""{OPTIMIZER_NAME}_{optimizer_index}"""	)
          if F"""{OPTIMIZER_NAME}""" not in input_dir
          else input_dir
      )
      logger.info(F"""Loading Optimizer from {ckpt_dir}"""	)
      a_     :      Tuple 						=     load_sharded_optimizer_state_dict(
          model_state_dict=model.state_dict(), optimizer_key="optimizer", storage_reader=dist_cp.FileSystemReader(SCREAMING_SNAKE_CASE_	), )
      a_     :      Tuple 						=     optim_state["optimizer"]
      logger.info(F"""Optimizer loaded from {ckpt_dir}"""	)
    a_     :      Union[str, Any] 						=     FSDP.optim_state_dict_to_load(SCREAMING_SNAKE_CASE_, SCREAMING_SNAKE_CASE_, SCREAMING_SNAKE_CASE_	)
    optimizer.load_state_dict(SCREAMING_SNAKE_CASE_	)




"""simple docstring"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_				= logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_				= {
    """google/realm-cc-news-pretrained-embedder""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/config.json"""
    ),
    """google/realm-cc-news-pretrained-encoder""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/config.json"""
    ),
    """google/realm-cc-news-pretrained-scorer""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/config.json"""
    ),
    """google/realm-cc-news-pretrained-openqa""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/config.json"""
    ),
    """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/config.json""",
    """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/config.json""",
    """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/config.json""",
    """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/config.json""",
    # See all REALM models at https://huggingface.co/models?filter=realm
}




class 			snake_case_						(						a_				):
							__lowerCAmelCase    					=					"realm"






							def __init__(   self			,     a_=3_0_5_2_2			,     a_=7_6_8			,     a_=1_2_8			,     a_=1_2			,     a_=1_2			,     a_=8			,     a_=3_0_7_2			,     a_="gelu_new"			,     a_=0.1			,     a_=0.1			,     a_=5_1_2			,     a_=2			,     a_=0.02			,     a_=1e-12			,     a_=2_5_6			,     a_=1_0			,     a_=1e-3			,     a_=5			,     a_=3_2_0			,     a_=1_3_3_5_3_7_1_8			,     a_=5_0_0_0			,     a_=1			,     a_=0			,     a_=2			,     **a_			,     ):
									super().__init__(pad_token_id=a_			,     bos_token_id=a_			,     eos_token_id=a_			,     **a_      )

									# Common config
									a_     :      Optional[int] 						=     vocab_size
									a_     :      List[Any] 						=     max_position_embeddings
									a_     :      Optional[Any] 						=     hidden_size
									a_     :      Optional[Any] 						=     retriever_proj_size
									a_     :      List[str] 						=     num_hidden_layers
									a_     :      List[Any] 						=     num_attention_heads
									a_     :      Tuple 						=     num_candidates
									a_     :      str 						=     intermediate_size
									a_     :      Optional[int] 						=     hidden_act
									a_     :      List[str] 						=     hidden_dropout_prob
									a_     :      List[str] 						=     attention_probs_dropout_prob
									a_     :      Tuple 						=     initializer_range
									a_     :      Tuple 						=     type_vocab_size
									a_     :      str 						=     layer_norm_eps

									# Reader config
									a_     :      str 						=     span_hidden_size
									a_     :      Union[str, Any] 						=     max_span_width
									a_     :      Tuple 						=     reader_layer_norm_eps
									a_     :      List[Any] 						=     reader_beam_size
									a_     :      str 						=     reader_seq_len

									# Retrieval config
									a_     :      str 						=     num_block_records
									a_     :      int 						=     searcher_beam_size