


import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version('''>=''', FSDP_PYTORCH_VERSION):
     import torch.distributed.checkpoint as dist_cp
     from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
     from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
     from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
     from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


snake_case    :   List[Any]	    =					get_logger(__name__)
def     __lowerCamelCase						(			UpperCAmelCase_						:		List[Any]			,  UpperCAmelCase_						:		Union[str, Any]			,  UpperCAmelCase_						:		Any			,  UpperCAmelCase_						:		Union[str, Any]			,  UpperCAmelCase_						:		List[Any]=0						):




      """simple docstring"""





      os.makedirs(UpperCAmelCase_			,  exist_ok=UpperCAmelCase_						)
      with FSDP.state_dict_type(
          UpperCAmelCase_			,  fsdp_plugin.state_dict_type			,  fsdp_plugin.state_dict_config			,  fsdp_plugin.optim_state_dict_config						):
            a						:Union[str, Any]										=	model.state_dict()
            if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
                  a						:Optional[int]										=	F'''{MODEL_NAME}.bin''' if model_index == 0 else F'''{MODEL_NAME}_{model_index}.bin'''
                  a						:Optional[int]										=	os.path.join(UpperCAmelCase_			,  UpperCAmelCase_						)
                  if accelerator.process_index == 0:
                        logger.info(F'''Saving model to {output_model_file}'''						)
                        torch.save(UpperCAmelCase_			,  UpperCAmelCase_						)
                        logger.info(F'''Model saved to {output_model_file}'''						)
            elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
                  a						:List[str]										=	(
                      F'''{MODEL_NAME}_rank{accelerator.process_index}.bin'''
                      if model_index == 0
                      else F'''{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin'''
                  )
                  a						:List[Any]										=	os.path.join(UpperCAmelCase_			,  UpperCAmelCase_						)
                  logger.info(F'''Saving model to {output_model_file}'''						)
                  torch.save(UpperCAmelCase_			,  UpperCAmelCase_						)
                  logger.info(F'''Model saved to {output_model_file}'''						)
            elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
                  a						:List[Any]										=	os.path.join(UpperCAmelCase_			,  F'''{MODEL_NAME}_{model_index}'''						)
                  os.makedirs(UpperCAmelCase_			,  exist_ok=UpperCAmelCase_						)
                  logger.info(F'''Saving model to {ckpt_dir}'''						)
                  a						:Tuple										=	{'''model''': state_dict}

                  dist_cp.save_state_dict(
                      state_dict=UpperCAmelCase_			,  storage_writer=dist_cp.FileSystemWriter(UpperCAmelCase_						)			,  planner=DefaultSavePlanner()			,  )
                  logger.info(F'''Model saved to {ckpt_dir}'''						)
def     __lowerCamelCase						(			UpperCAmelCase_						:		Optional[int]			,  UpperCAmelCase_						:		int			,  UpperCAmelCase_						:		List[str]			,  UpperCAmelCase_						:		Union[str, Any]			,  UpperCAmelCase_						:		Optional[Any]=0						):




      """simple docstring"""





      accelerator.wait_for_everyone()
      with FSDP.state_dict_type(
          UpperCAmelCase_			,  fsdp_plugin.state_dict_type			,  fsdp_plugin.state_dict_config			,  fsdp_plugin.optim_state_dict_config						):
            if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
                  if type(UpperCAmelCase_						) != FSDP and accelerator.process_index != 0:
                        if not fsdp_plugin.sync_module_states:
                              raise ValueError(
                                  '''Set the `sync_module_states` flag to `True` so that model states are synced across processes when '''
                                  '''initializing FSDP object'''						)
                        return
                  a						:Any										=	F'''{MODEL_NAME}.bin''' if model_index == 0 else F'''{MODEL_NAME}_{model_index}.bin'''
                  a						:Tuple										=	os.path.join(UpperCAmelCase_			,  UpperCAmelCase_						)
                  logger.info(F'''Loading model from {input_model_file}'''						)
                  a						:Any										=	torch.load(UpperCAmelCase_						)
                  logger.info(F'''Model loaded from {input_model_file}'''						)
            elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
                  a						:List[str]										=	(
                      F'''{MODEL_NAME}_rank{accelerator.process_index}.bin'''
                      if model_index == 0
                      else F'''{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin'''
                  )
                  a						:Dict										=	os.path.join(UpperCAmelCase_			,  UpperCAmelCase_						)
                  logger.info(F'''Loading model from {input_model_file}'''						)
                  a						:List[Any]										=	torch.load(UpperCAmelCase_						)
                  logger.info(F'''Model loaded from {input_model_file}'''						)
            elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
                  a						:Union[str, Any]										=	(
                      os.path.join(UpperCAmelCase_			,  F'''{MODEL_NAME}_{model_index}'''						)
                      if F'''{MODEL_NAME}''' not in input_dir
                      else input_dir
                  )
                  logger.info(F'''Loading model from {ckpt_dir}'''						)
                  a						:Optional[Any]										=	{'''model''': model.state_dict()}
                  dist_cp.load_state_dict(
                      state_dict=UpperCAmelCase_			,  storage_reader=dist_cp.FileSystemReader(UpperCAmelCase_						)			,  planner=DefaultLoadPlanner()			,  )
                  a						:Optional[Any]										=	state_dict['''model''']
                  logger.info(F'''Model loaded from {ckpt_dir}'''						)
            model.load_state_dict(UpperCAmelCase_						)
def     __lowerCamelCase						(			UpperCAmelCase_						:		Union[str, Any]			,  UpperCAmelCase_						:		Any			,  UpperCAmelCase_						:		Optional[int]			,  UpperCAmelCase_						:		Dict			,  UpperCAmelCase_						:		Optional[Any]			,  UpperCAmelCase_						:		Union[str, Any]=0						):




      """simple docstring"""





      os.makedirs(UpperCAmelCase_			,  exist_ok=UpperCAmelCase_						)
      with FSDP.state_dict_type(
          UpperCAmelCase_			,  fsdp_plugin.state_dict_type			,  fsdp_plugin.state_dict_config			,  fsdp_plugin.optim_state_dict_config						):
            a						:Dict										=	FSDP.optim_state_dict(UpperCAmelCase_			,  UpperCAmelCase_						)
            if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
                  if accelerator.process_index == 0:
                        a						:str										=	(
                            F'''{OPTIMIZER_NAME}.bin''' if optimizer_index == 0 else F'''{OPTIMIZER_NAME}_{optimizer_index}.bin'''
                        )
                        a						:Optional[Any]										=	os.path.join(UpperCAmelCase_			,  UpperCAmelCase_						)
                        logger.info(F'''Saving Optimizer state to {output_optimizer_file}'''						)
                        torch.save(UpperCAmelCase_			,  UpperCAmelCase_						)
                        logger.info(F'''Optimizer state saved in {output_optimizer_file}'''						)
            else:
                  a						:List[str]										=	os.path.join(UpperCAmelCase_			,  F'''{OPTIMIZER_NAME}_{optimizer_index}'''						)
                  os.makedirs(UpperCAmelCase_			,  exist_ok=UpperCAmelCase_						)
                  logger.info(F'''Saving Optimizer state to {ckpt_dir}'''						)
                  dist_cp.save_state_dict(
                      state_dict={'''optimizer''': optim_state}			,  storage_writer=dist_cp.FileSystemWriter(UpperCAmelCase_						)			,  planner=DefaultSavePlanner()			,  )
                  logger.info(F'''Optimizer state saved in {ckpt_dir}'''						)






def     __lowerCamelCase						(			UpperCAmelCase_						:		Optional[Any]			,  UpperCAmelCase_						:		Union[str, Any]			,  UpperCAmelCase_						:		Tuple			,  UpperCAmelCase_						:		Any			,  UpperCAmelCase_						:		str			,  UpperCAmelCase_						:		int=0						):




      """simple docstring"""





      accelerator.wait_for_everyone()
      with FSDP.state_dict_type(
          UpperCAmelCase_			,  fsdp_plugin.state_dict_type			,  fsdp_plugin.state_dict_config			,  fsdp_plugin.optim_state_dict_config						):
            if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
                  a						:Union[str, Any]										=	None
                  # below check should work but currently it isn't working (mostly opytorch issue),
                  # in the meantime disabling it at the cost of excess memory usage
                  # if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
                  a						:Dict										=	(
                      F'''{OPTIMIZER_NAME}.bin''' if optimizer_index == 0 else F'''{OPTIMIZER_NAME}_{optimizer_index}.bin'''
                  )
                  a						:Tuple										=	os.path.join(UpperCAmelCase_			,  UpperCAmelCase_						)
                  logger.info(F'''Loading Optimizer state from {input_optimizer_file}'''						)
                  a						:Optional[Any]										=	torch.load(UpperCAmelCase_						)
                  logger.info(F'''Optimizer state loaded from {input_optimizer_file}'''						)
            else:
                  a						:List[Any]										=	(
                      os.path.join(UpperCAmelCase_			,  F'''{OPTIMIZER_NAME}_{optimizer_index}'''						)
                      if F'''{OPTIMIZER_NAME}''' not in input_dir
                      else input_dir
                  )
                  logger.info(F'''Loading Optimizer from {ckpt_dir}'''						)
                  a						:Any										=	load_sharded_optimizer_state_dict(
                      model_state_dict=model.state_dict()			,  optimizer_key='''optimizer'''			,  storage_reader=dist_cp.FileSystemReader(UpperCAmelCase_						)			,  )
                  a						:Tuple										=	optim_state['''optimizer''']
                  logger.info(F'''Optimizer loaded from {ckpt_dir}'''						)
            a						:Dict										=	FSDP.optim_state_dict_to_load(UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_						)
            optimizer.load_state_dict(UpperCAmelCase_						)









from . import __version__

# Backward compatibility imports, to make sure all those objects can be found in file_utils
from .utils import (
    CLOUDFRONT_DISTRIB_PREFIX,
    CONFIG_NAME,
    DISABLE_TELEMETRY,
    DUMMY_INPUTS,
    DUMMY_MASK,
    ENV_VARS_TRUE_AND_AUTO_VALUES,
    ENV_VARS_TRUE_VALUES,
    FEATURE_EXTRACTOR_NAME,
    FLAX_WEIGHTS_NAME,
    HF_MODULES_CACHE,
    HUGGINGFACE_CO_PREFIX,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    MODEL_CARD_NAME,
    MULTIPLE_CHOICE_DUMMY_INPUTS,
    PYTORCH_PRETRAINED_BERT_CACHE,
    PYTORCH_TRANSFORMERS_CACHE,
    S3_BUCKET_PREFIX,
    SENTENCEPIECE_UNDERLINE,
    SPIECE_UNDERLINE,
    TF2_WEIGHTS_NAME,
    TF_WEIGHTS_NAME,
    TORCH_FX_REQUIRED_VERSION,
    TRANSFORMERS_CACHE,
    TRANSFORMERS_DYNAMIC_MODULE_NAME,
    USE_JAX,
    USE_TF,
    USE_TORCH,
    WEIGHTS_INDEX_NAME,
    WEIGHTS_NAME,
    ContextManagers,
    DummyObject,
    EntryNotFoundError,
    ExplicitEnum,
    ModelOutput,
    PaddingStrategy,
    PushToHubMixin,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    TensorType,
    _LazyModule,
    add_code_sample_docstrings,
    add_end_docstrings,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    cached_property,
    copy_func,
    default_cache_path,
    define_sagemaker_information,
    get_cached_models,
    get_file_from_repo,
    get_full_repo_name,
    get_torch_version,
    has_file,
    http_user_agent,
    is_apex_available,
    is_bsa_available,
    is_coloredlogs_available,
    is_datasets_available,
    is_detectrona_available,
    is_faiss_available,
    is_flax_available,
    is_ftfy_available,
    is_in_notebook,
    is_ipex_available,
    is_librosa_available,
    is_offline_mode,
    is_onnx_available,
    is_pandas_available,
    is_phonemizer_available,
    is_protobuf_available,
    is_psutil_available,
    is_pyanvml_available,
    is_pyctcdecode_available,
    is_pytesseract_available,
    is_pytorch_quantization_available,
    is_rjieba_available,
    is_sagemaker_dp_enabled,
    is_sagemaker_mp_enabled,
    is_scipy_available,
    is_sentencepiece_available,
    is_seqio_available,
    is_sklearn_available,
    is_soundfile_availble,
    is_spacy_available,
    is_speech_available,
    is_tensor,
    is_tensorflow_probability_available,
    is_tfaonnx_available,
    is_tf_available,
    is_timm_available,
    is_tokenizers_available,
    is_torch_available,
    is_torch_bfaa_available,
    is_torch_cuda_available,
    is_torch_fx_available,
    is_torch_fx_proxy,
    is_torch_mps_available,
    is_torch_tfaa_available,
    is_torch_tpu_available,
    is_torchaudio_available,
    is_training_run_on_sagemaker,
    is_vision_available,
    replace_return_docstrings,
    requires_backends,
    to_numpy,
    to_py_obj,
    torch_only_method,
)





