

import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version('''>=''', FSDP_PYTORCH_VERSION):
 import torch.distributed.checkpoint as dist_cp
 from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
 from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
 from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
 from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


__A      		=     get_logger(__name__)



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase=0					)							->     List[Any]:



  """simple docstring"""






  os.makedirs(_UpperCamelCase			,	exist_ok=_UpperCamelCase					)
  with FSDP.state_dict_type(
      _UpperCamelCase			,	fsdp_plugin.state_dict_type			,	fsdp_plugin.state_dict_config			,	fsdp_plugin.optim_state_dict_config					):
    _snake_case						=							model.state_dict()
    if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
      _snake_case						=							F"""{MODEL_NAME}.bin""" if model_index == 0 else F"""{MODEL_NAME}_{model_index}.bin"""
      _snake_case						=							os.path.join(_UpperCamelCase			,	_UpperCamelCase					)
      if accelerator.process_index == 0:
        logger.info(F"""Saving model to {output_model_file}"""					)
        torch.save(_UpperCamelCase			,	_UpperCamelCase					)
        logger.info(F"""Model saved to {output_model_file}"""					)
    elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
      _snake_case						=							(
          F"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
          if model_index == 0
          else F"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
      )
      _snake_case						=							os.path.join(_UpperCamelCase			,	_UpperCamelCase					)
      logger.info(F"""Saving model to {output_model_file}"""					)
      torch.save(_UpperCamelCase			,	_UpperCamelCase					)
      logger.info(F"""Model saved to {output_model_file}"""					)
    elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
      _snake_case						=							os.path.join(_UpperCamelCase			,	F"""{MODEL_NAME}_{model_index}"""					)
      os.makedirs(_UpperCamelCase			,	exist_ok=_UpperCamelCase					)
      logger.info(F"""Saving model to {ckpt_dir}"""					)
      _snake_case						=							{'''model''': state_dict}

      dist_cp.save_state_dict(
          state_dict=_UpperCamelCase			,	storage_writer=dist_cp.FileSystemWriter(_UpperCamelCase					)			,	planner=DefaultSavePlanner()			,	)
      logger.info(F"""Model saved to {ckpt_dir}"""					)



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase=0					)							->     Optional[int]:



  """simple docstring"""






  accelerator.wait_for_everyone()
  with FSDP.state_dict_type(
      _UpperCamelCase			,	fsdp_plugin.state_dict_type			,	fsdp_plugin.state_dict_config			,	fsdp_plugin.optim_state_dict_config					):
    if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
      if type(_UpperCamelCase					) != FSDP and accelerator.process_index != 0:
        if not fsdp_plugin.sync_module_states:
          raise ValueError(
              '''Set the `sync_module_states` flag to `True` so that model states are synced across processes when '''
              '''initializing FSDP object'''					)
        return
      _snake_case						=							F"""{MODEL_NAME}.bin""" if model_index == 0 else F"""{MODEL_NAME}_{model_index}.bin"""
      _snake_case						=							os.path.join(_UpperCamelCase			,	_UpperCamelCase					)
      logger.info(F"""Loading model from {input_model_file}"""					)
      _snake_case						=							torch.load(_UpperCamelCase					)
      logger.info(F"""Model loaded from {input_model_file}"""					)
    elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
      _snake_case						=							(
          F"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
          if model_index == 0
          else F"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
      )
      _snake_case						=							os.path.join(_UpperCamelCase			,	_UpperCamelCase					)
      logger.info(F"""Loading model from {input_model_file}"""					)
      _snake_case						=							torch.load(_UpperCamelCase					)
      logger.info(F"""Model loaded from {input_model_file}"""					)
    elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
      _snake_case						=							(
          os.path.join(_UpperCamelCase			,	F"""{MODEL_NAME}_{model_index}"""					)
          if F"""{MODEL_NAME}""" not in input_dir
          else input_dir
      )
      logger.info(F"""Loading model from {ckpt_dir}"""					)
      _snake_case						=							{'''model''': model.state_dict()}
      dist_cp.load_state_dict(
          state_dict=_UpperCamelCase			,	storage_reader=dist_cp.FileSystemReader(_UpperCamelCase					)			,	planner=DefaultLoadPlanner()			,	)
      _snake_case						=							state_dict['''model''']
      logger.info(F"""Model loaded from {ckpt_dir}"""					)
    model.load_state_dict(_UpperCamelCase					)



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase=0					)							->     Dict:



  """simple docstring"""






  os.makedirs(_UpperCamelCase			,	exist_ok=_UpperCamelCase					)
  with FSDP.state_dict_type(
      _UpperCamelCase			,	fsdp_plugin.state_dict_type			,	fsdp_plugin.state_dict_config			,	fsdp_plugin.optim_state_dict_config					):
    _snake_case						=							FSDP.optim_state_dict(_UpperCamelCase			,	_UpperCamelCase					)
    if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
      if accelerator.process_index == 0:
        _snake_case						=							(
            F"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else F"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
        )
        _snake_case						=							os.path.join(_UpperCamelCase			,	_UpperCamelCase					)
        logger.info(F"""Saving Optimizer state to {output_optimizer_file}"""					)
        torch.save(_UpperCamelCase			,	_UpperCamelCase					)
        logger.info(F"""Optimizer state saved in {output_optimizer_file}"""					)
    else:
      _snake_case						=							os.path.join(_UpperCamelCase			,	F"""{OPTIMIZER_NAME}_{optimizer_index}"""					)
      os.makedirs(_UpperCamelCase			,	exist_ok=_UpperCamelCase					)
      logger.info(F"""Saving Optimizer state to {ckpt_dir}"""					)
      dist_cp.save_state_dict(
          state_dict={'''optimizer''': optim_state}			,	storage_writer=dist_cp.FileSystemWriter(_UpperCamelCase					)			,	planner=DefaultSavePlanner()			,	)
      logger.info(F"""Optimizer state saved in {ckpt_dir}"""					)



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase=0					)							->     Optional[Any]:



  """simple docstring"""






  accelerator.wait_for_everyone()
  with FSDP.state_dict_type(
      _UpperCamelCase			,	fsdp_plugin.state_dict_type			,	fsdp_plugin.state_dict_config			,	fsdp_plugin.optim_state_dict_config					):
    if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
      _snake_case						=							None
      # below check should work but currently it isn't working (mostly opytorch issue),
      # in the meantime disabling it at the cost of excess memory usage
      # if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
      _snake_case						=							(
          F"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else F"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
      )
      _snake_case						=							os.path.join(_UpperCamelCase			,	_UpperCamelCase					)
      logger.info(F"""Loading Optimizer state from {input_optimizer_file}"""					)
      _snake_case						=							torch.load(_UpperCamelCase					)
      logger.info(F"""Optimizer state loaded from {input_optimizer_file}"""					)
    else:
      _snake_case						=							(
          os.path.join(_UpperCamelCase			,	F"""{OPTIMIZER_NAME}_{optimizer_index}"""					)
          if F"""{OPTIMIZER_NAME}""" not in input_dir
          else input_dir
      )
      logger.info(F"""Loading Optimizer from {ckpt_dir}"""					)
      _snake_case						=							load_sharded_optimizer_state_dict(
          model_state_dict=model.state_dict()			,	optimizer_key='''optimizer'''			,	storage_reader=dist_cp.FileSystemReader(_UpperCamelCase					)			,	)
      _snake_case						=							optim_state['''optimizer''']
      logger.info(F"""Optimizer loaded from {ckpt_dir}"""					)
    _snake_case						=							FSDP.optim_state_dict_to_load(_UpperCamelCase			,	_UpperCamelCase			,	_UpperCamelCase					)
    optimizer.load_state_dict(_UpperCamelCase					)




__A      		=     {
    '''Pillow''': '''Pillow<10.0.0''',
    '''accelerate''': '''accelerate>=0.20.3''',
    '''av''': '''av==9.2.0''',
    '''beautifulsoup4''': '''beautifulsoup4''',
    '''black''': '''black~=23.1''',
    '''codecarbon''': '''codecarbon==1.2.0''',
    '''cookiecutter''': '''cookiecutter==1.7.3''',
    '''dataclasses''': '''dataclasses''',
    '''datasets''': '''datasets!=2.5.0''',
    '''decord''': '''decord==0.6.0''',
    '''deepspeed''': '''deepspeed>=0.9.3''',
    '''diffusers''': '''diffusers''',
    '''dill''': '''dill<0.3.5''',
    '''evaluate''': '''evaluate>=0.2.0''',
    '''fairscale''': '''fairscale>0.3''',
    '''faiss-cpu''': '''faiss-cpu''',
    '''fastapi''': '''fastapi''',
    '''filelock''': '''filelock''',
    '''flax''': '''flax>=0.4.1,<=0.7.0''',
    '''ftfy''': '''ftfy''',
    '''fugashi''': '''fugashi>=1.0''',
    '''GitPython''': '''GitPython<3.1.19''',
    '''hf-doc-builder''': '''hf-doc-builder>=0.3.0''',
    '''huggingface-hub''': '''huggingface-hub>=0.14.1,<1.0''',
    '''importlib_metadata''': '''importlib_metadata''',
    '''ipadic''': '''ipadic>=1.0.0,<2.0''',
    '''isort''': '''isort>=5.5.4''',
    '''jax''': '''jax>=0.2.8,!=0.3.2,<=0.4.13''',
    '''jaxlib''': '''jaxlib>=0.1.65,<=0.4.13''',
    '''jieba''': '''jieba''',
    '''kenlm''': '''kenlm''',
    '''keras-nlp''': '''keras-nlp>=0.3.1''',
    '''librosa''': '''librosa''',
    '''nltk''': '''nltk''',
    '''natten''': '''natten>=0.14.6''',
    '''numpy''': '''numpy>=1.17''',
    '''onnxconverter-common''': '''onnxconverter-common''',
    '''onnxruntime-tools''': '''onnxruntime-tools>=1.4.2''',
    '''onnxruntime''': '''onnxruntime>=1.4.0''',
    '''opencv-python''': '''opencv-python''',
    '''optuna''': '''optuna''',
    '''optax''': '''optax>=0.0.8,<=0.1.4''',
    '''packaging''': '''packaging>=20.0''',
    '''parameterized''': '''parameterized''',
    '''phonemizer''': '''phonemizer''',
    '''protobuf''': '''protobuf''',
    '''psutil''': '''psutil''',
    '''pyyaml''': '''pyyaml>=5.1''',
    '''pydantic''': '''pydantic<2''',
    '''pytest''': '''pytest>=7.2.0''',
    '''pytest-timeout''': '''pytest-timeout''',
    '''pytest-xdist''': '''pytest-xdist''',
    '''python''': '''python>=3.8.0''',
    '''ray[tune]''': '''ray[tune]''',
    '''regex''': '''regex!=2019.12.17''',
    '''requests''': '''requests''',
    '''rhoknp''': '''rhoknp>=1.1.0,<1.3.1''',
    '''rjieba''': '''rjieba''',
    '''rouge-score''': '''rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1''',
    '''ruff''': '''ruff>=0.0.241,<=0.0.259''',
    '''sacrebleu''': '''sacrebleu>=1.4.12,<2.0.0''',
    '''sacremoses''': '''sacremoses''',
    '''safetensors''': '''safetensors>=0.3.1''',
    '''sagemaker''': '''sagemaker>=2.31.0''',
    '''scikit-learn''': '''scikit-learn''',
    '''sentencepiece''': '''sentencepiece>=0.1.91,!=0.1.92''',
    '''sigopt''': '''sigopt''',
    '''starlette''': '''starlette''',
    '''sudachipy''': '''sudachipy>=0.6.6''',
    '''sudachidict_core''': '''sudachidict_core>=20220729''',
    '''tensorflow-cpu''': '''tensorflow-cpu>=2.6,<2.14''',
    '''tensorflow''': '''tensorflow>=2.6,<2.14''',
    '''tensorflow-text''': '''tensorflow-text<2.14''',
    '''tf2onnx''': '''tf2onnx''',
    '''timeout-decorator''': '''timeout-decorator''',
    '''timm''': '''timm''',
    '''tokenizers''': '''tokenizers>=0.11.1,!=0.11.3,<0.14''',
    '''torch''': '''torch>=1.9,!=1.12.0''',
    '''torchaudio''': '''torchaudio''',
    '''torchvision''': '''torchvision''',
    '''pyctcdecode''': '''pyctcdecode>=0.4.0''',
    '''tqdm''': '''tqdm>=4.27''',
    '''unidic''': '''unidic>=1.0.2''',
    '''unidic_lite''': '''unidic_lite>=1.0.7''',
    '''urllib3''': '''urllib3<2.0.0''',
    '''uvicorn''': '''uvicorn''',
}

