'''simple docstring'''





import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version(""">=""", FSDP_PYTORCH_VERSION):
			import torch.distributed.checkpoint as dist_cp
			from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
			from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
			from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
			from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


lowerCAmelCase_						:		List[str]								=  get_logger(__name__)
def     __A				(  UpperCAmelCase  ,UpperCAmelCase  ,UpperCAmelCase  ,UpperCAmelCase  ,UpperCAmelCase=0       )  ->      Union[str, Any]:




						'''simple docstring'''
						os.makedirs(__lowercase  ,exist_ok=__lowercase       )
						with FSDP.state_dict_type(
						    __lowercase  ,fsdp_plugin.state_dict_type  ,fsdp_plugin.state_dict_config  ,fsdp_plugin.optim_state_dict_config       ):
												_UpperCamelCase      :      Tuple       =    model.state_dict()
												if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
																		_UpperCamelCase      :      Optional[int]       =    f'''{MODEL_NAME}.bin''' if model_index == 0 else f'''{MODEL_NAME}_{model_index}.bin'''
																		_UpperCamelCase      :      Union[str, Any]       =    os.path.join(__lowercase  ,__lowercase       )
																		if accelerator.process_index == 0:
																								logger.info(f'''Saving model to {output_model_file}'''       )
																								torch.save(__lowercase  ,__lowercase       )
																								logger.info(f'''Model saved to {output_model_file}'''       )
												elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
																		_UpperCamelCase      :      int       =    (
																		    f'''{MODEL_NAME}_rank{accelerator.process_index}.bin'''
																		    if model_index == 0
																		    else f'''{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin'''
																		)
																		_UpperCamelCase      :      Dict       =    os.path.join(__lowercase  ,__lowercase       )
																		logger.info(f'''Saving model to {output_model_file}'''       )
																		torch.save(__lowercase  ,__lowercase       )
																		logger.info(f'''Model saved to {output_model_file}'''       )
												elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
																		_UpperCamelCase      :      Optional[int]       =    os.path.join(__lowercase  ,f'''{MODEL_NAME}_{model_index}'''       )
																		os.makedirs(__lowercase  ,exist_ok=__lowercase       )
																		logger.info(f'''Saving model to {ckpt_dir}'''       )
																		_UpperCamelCase      :      Tuple       =    {"model": state_dict}

																		dist_cp.save_state_dict(
																		    state_dict=__lowercase  ,storage_writer=dist_cp.FileSystemWriter(__lowercase       )  ,planner=DefaultSavePlanner()  ,)
																		logger.info(f'''Model saved to {ckpt_dir}'''       )
def     __A				(  UpperCAmelCase  ,UpperCAmelCase  ,UpperCAmelCase  ,UpperCAmelCase  ,UpperCAmelCase=0       )  ->      Any:




						'''simple docstring'''
						accelerator.wait_for_everyone()
						with FSDP.state_dict_type(
						    __lowercase  ,fsdp_plugin.state_dict_type  ,fsdp_plugin.state_dict_config  ,fsdp_plugin.optim_state_dict_config       ):
												if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
																		if type(__lowercase       ) != FSDP and accelerator.process_index != 0:
																								if not fsdp_plugin.sync_module_states:
																														raise ValueError(
																														    "Set the `sync_module_states` flag to `True` so that model states are synced across processes when "
																														    "initializing FSDP object"       )
																								return
																		_UpperCamelCase      :      Any       =    f'''{MODEL_NAME}.bin''' if model_index == 0 else f'''{MODEL_NAME}_{model_index}.bin'''
																		_UpperCamelCase      :      Any       =    os.path.join(__lowercase  ,__lowercase       )
																		logger.info(f'''Loading model from {input_model_file}'''       )
																		_UpperCamelCase      :      Optional[int]       =    torch.load(__lowercase       )
																		logger.info(f'''Model loaded from {input_model_file}'''       )
												elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
																		_UpperCamelCase      :      List[Any]       =    (
																		    f'''{MODEL_NAME}_rank{accelerator.process_index}.bin'''
																		    if model_index == 0
																		    else f'''{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin'''
																		)
																		_UpperCamelCase      :      Union[str, Any]       =    os.path.join(__lowercase  ,__lowercase       )
																		logger.info(f'''Loading model from {input_model_file}'''       )
																		_UpperCamelCase      :      Optional[Any]       =    torch.load(__lowercase       )
																		logger.info(f'''Model loaded from {input_model_file}'''       )
												elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
																		_UpperCamelCase      :      Tuple       =    (
																		    os.path.join(__lowercase  ,f'''{MODEL_NAME}_{model_index}'''       )
																		    if f'''{MODEL_NAME}''' not in input_dir
																		    else input_dir
																		)
																		logger.info(f'''Loading model from {ckpt_dir}'''       )
																		_UpperCamelCase      :      List[str]       =    {"model": model.state_dict()}
																		dist_cp.load_state_dict(
																		    state_dict=__lowercase  ,storage_reader=dist_cp.FileSystemReader(__lowercase       )  ,planner=DefaultLoadPlanner()  ,)
																		_UpperCamelCase      :      int       =    state_dict["model"]
																		logger.info(f'''Model loaded from {ckpt_dir}'''       )
												model.load_state_dict(__lowercase       )
def     __A				(  UpperCAmelCase  ,UpperCAmelCase  ,UpperCAmelCase  ,UpperCAmelCase  ,UpperCAmelCase  ,UpperCAmelCase=0       )  ->      Any:




						'''simple docstring'''
						os.makedirs(__lowercase  ,exist_ok=__lowercase       )
						with FSDP.state_dict_type(
						    __lowercase  ,fsdp_plugin.state_dict_type  ,fsdp_plugin.state_dict_config  ,fsdp_plugin.optim_state_dict_config       ):
												_UpperCamelCase      :      List[Any]       =    FSDP.optim_state_dict(__lowercase  ,__lowercase       )
												if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
																		if accelerator.process_index == 0:
																								_UpperCamelCase      :      Any       =    (
																								    f'''{OPTIMIZER_NAME}.bin''' if optimizer_index == 0 else f'''{OPTIMIZER_NAME}_{optimizer_index}.bin'''
																								)
																								_UpperCamelCase      :      Any       =    os.path.join(__lowercase  ,__lowercase       )
																								logger.info(f'''Saving Optimizer state to {output_optimizer_file}'''       )
																								torch.save(__lowercase  ,__lowercase       )
																								logger.info(f'''Optimizer state saved in {output_optimizer_file}'''       )
												else:
																		_UpperCamelCase      :      List[Any]       =    os.path.join(__lowercase  ,f'''{OPTIMIZER_NAME}_{optimizer_index}'''       )
																		os.makedirs(__lowercase  ,exist_ok=__lowercase       )
																		logger.info(f'''Saving Optimizer state to {ckpt_dir}'''       )
																		dist_cp.save_state_dict(
																		    state_dict={"optimizer": optim_state}  ,storage_writer=dist_cp.FileSystemWriter(__lowercase       )  ,planner=DefaultSavePlanner()  ,)
																		logger.info(f'''Optimizer state saved in {ckpt_dir}'''       )






def     __A				(  UpperCAmelCase  ,UpperCAmelCase  ,UpperCAmelCase  ,UpperCAmelCase  ,UpperCAmelCase  ,UpperCAmelCase=0       )  ->      Optional[Any]:




						'''simple docstring'''
						accelerator.wait_for_everyone()
						with FSDP.state_dict_type(
						    __lowercase  ,fsdp_plugin.state_dict_type  ,fsdp_plugin.state_dict_config  ,fsdp_plugin.optim_state_dict_config       ):
												if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
																		_UpperCamelCase      :      Tuple       =    None
																		# below check should work but currently it isn't working (mostly opytorch issue),
																		# in the meantime disabling it at the cost of excess memory usage
																		# if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
																		_UpperCamelCase      :      Optional[Any]       =    (
																		    f'''{OPTIMIZER_NAME}.bin''' if optimizer_index == 0 else f'''{OPTIMIZER_NAME}_{optimizer_index}.bin'''
																		)
																		_UpperCamelCase      :      Optional[Any]       =    os.path.join(__lowercase  ,__lowercase       )
																		logger.info(f'''Loading Optimizer state from {input_optimizer_file}'''       )
																		_UpperCamelCase      :      int       =    torch.load(__lowercase       )
																		logger.info(f'''Optimizer state loaded from {input_optimizer_file}'''       )
												else:
																		_UpperCamelCase      :      Tuple       =    (
																		    os.path.join(__lowercase  ,f'''{OPTIMIZER_NAME}_{optimizer_index}'''       )
																		    if f'''{OPTIMIZER_NAME}''' not in input_dir
																		    else input_dir
																		)
																		logger.info(f'''Loading Optimizer from {ckpt_dir}'''       )
																		_UpperCamelCase      :      List[Any]       =    load_sharded_optimizer_state_dict(
																		    model_state_dict=model.state_dict()  ,optimizer_key="optimizer"  ,storage_reader=dist_cp.FileSystemReader(__lowercase       )  ,)
																		_UpperCamelCase      :      Optional[Any]       =    optim_state["optimizer"]
																		logger.info(f'''Optimizer loaded from {ckpt_dir}'''       )
												_UpperCamelCase      :      Union[str, Any]       =    FSDP.optim_state_dict_to_load(__lowercase  ,__lowercase  ,__lowercase       )
												optimizer.load_state_dict(__lowercase       )










from __future__ import annotations

import unittest

from transformers import AutoTokenizer, MBartConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers import TFAutoModelForSeqaSeqLM, TFMBartForConditionalGeneration, TFMBartModel


@require_tf
class     _a							:






  """simple docstring"""





  A_						       = MBartConfig
  A_						       = {}
  A_						       = """gelu"""




  def __init__(  self		,		_UpperCAmelCase		,		_UpperCAmelCase=13		,		_UpperCAmelCase=7		,		_UpperCAmelCase=True		,		_UpperCAmelCase=False		,		_UpperCAmelCase=99		,		_UpperCAmelCase=32		,		_UpperCAmelCase=2		,		_UpperCAmelCase=4		,		_UpperCAmelCase=37		,		_UpperCAmelCase=0.1		,		_UpperCAmelCase=0.1		,		_UpperCAmelCase=20		,		_UpperCAmelCase=2		,		_UpperCAmelCase=1		,		_UpperCAmelCase=0		,		)      ->			Union[str, Any]:
     UpperCamelCase_						=  parent
     UpperCamelCase_						=  batch_size
     UpperCamelCase_						=  seq_length
     UpperCamelCase_						=  is_training
     UpperCamelCase_						=  use_labels
     UpperCamelCase_						=  vocab_size
     UpperCamelCase_						=  hidden_size
     UpperCamelCase_						=  num_hidden_layers
     UpperCamelCase_						=  num_attention_heads
     UpperCamelCase_						=  intermediate_size
     UpperCamelCase_						=  hidden_dropout_prob
     UpperCamelCase_						=  attention_probs_dropout_prob
     UpperCamelCase_						=  max_position_embeddings
     UpperCamelCase_						=  eos_token_id
     UpperCamelCase_						=  pad_token_id
     UpperCamelCase_						=  bos_token_id




  def  _UpperCAmelCase  (  self			)      ->			List[Any]:
     UpperCamelCase_						=  ids_tensor([self.batch_size, self.seq_length - 1]		,		self.vocab_size			)
     UpperCamelCase_						=  tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size			)		,		1			)
     UpperCamelCase_						=  tf.concat([input_ids, eos_tensor]		,		axis=1			)

     UpperCamelCase_						=  ids_tensor([self.batch_size, self.seq_length]		,		self.vocab_size			)

     UpperCamelCase_						=  self.config_cls(
         vocab_size=self.vocab_size		,		d_model=self.hidden_size		,		encoder_layers=self.num_hidden_layers		,		decoder_layers=self.num_hidden_layers		,		encoder_attention_heads=self.num_attention_heads		,		decoder_attention_heads=self.num_attention_heads		,		encoder_ffn_dim=self.intermediate_size		,		decoder_ffn_dim=self.intermediate_size		,		dropout=self.hidden_dropout_prob		,		attention_dropout=self.attention_probs_dropout_prob		,		max_position_embeddings=self.max_position_embeddings		,		eos_token_ids=[2]		,		bos_token_id=self.bos_token_id		,		pad_token_id=self.pad_token_id		,		decoder_start_token_id=self.pad_token_id		,		**self.config_updates		,		)
     UpperCamelCase_						=  prepare_mbart_inputs_dict(_UpperCAmelCase		,		_UpperCAmelCase		,		_UpperCAmelCase			)
     return config, inputs_dict




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			List[str]:
     UpperCamelCase_						=  TFMBartModel(config=_UpperCAmelCase			).get_decoder()
     UpperCamelCase_						=  inputs_dict['input_ids']

     UpperCamelCase_						=  input_ids[:1, :]
     UpperCamelCase_						=  inputs_dict['attention_mask'][:1, :]
     UpperCamelCase_						=  inputs_dict['head_mask']
     UpperCamelCase_						=  1

     # first forward pass
     UpperCamelCase_						=  model(_UpperCAmelCase		,		attention_mask=_UpperCAmelCase		,		head_mask=_UpperCAmelCase		,		use_cache=_UpperCAmelCase			)

     UpperCamelCase_		,   UpperCamelCase_						=  outputs.to_tuple()
     UpperCamelCase_						=  past_key_values[1]






def      _snake_case     (__lowercase     ,   __lowercase     ,   __lowercase     ,   __lowercase=None     ,   __lowercase=None     ,   __lowercase=None     ,   __lowercase=None     ,   __lowercase=None     ,   ):
   if attention_mask is None:
      UpperCamelCase_						=  tf.cast(tf.math.not_equal(__lowercase     ,   config.pad_token_id)     ,   tf.inta)
   if decoder_attention_mask is None:
      UpperCamelCase_						=  tf.concat(
          [
              tf.ones(decoder_input_ids[:, :1].shape     ,   dtype=tf.inta),
              tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]     ,   config.pad_token_id)     ,   tf.inta),
          ]     ,   axis=-1     ,   )
   if head_mask is None:
      UpperCamelCase_						=  tf.ones((config.encoder_layers, config.encoder_attention_heads))
   if decoder_head_mask is None:
      UpperCamelCase_						=  tf.ones((config.decoder_layers, config.decoder_attention_heads))
   if cross_attn_head_mask is None:
      UpperCamelCase_						=  tf.ones((config.decoder_layers, config.decoder_attention_heads))
   return {
       "input_ids": input_ids,
       "decoder_input_ids": decoder_input_ids,
       "attention_mask": attention_mask,
       "decoder_attention_mask": decoder_attention_mask,
       "head_mask": head_mask,
       "decoder_head_mask": decoder_head_mask,
       "cross_attn_head_mask": cross_attn_head_mask,
   }


@require_tf
class     _a							(	UpperCAmelCase__    ,							UpperCAmelCase__    ,							unittest.TestCase     ):






  """simple docstring"""





  A_						       = (TFMBartForConditionalGeneration, TFMBartModel) if is_tf_available() else ()
  A_						       = (TFMBartForConditionalGeneration,) if is_tf_available() else ()
  A_						       = (
      {
          """conversational""": TFMBartForConditionalGeneration,
          """feature-extraction""": TFMBartModel,
          """summarization""": TFMBartForConditionalGeneration,
          """text2text-generation""": TFMBartForConditionalGeneration,
          """translation""": TFMBartForConditionalGeneration,
      }
      if is_tf_available()
      else {}
  )
  A_						       = True
  A_						       = False
  A_						       = False




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase		,		_UpperCAmelCase		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Tuple:
     if pipeline_test_casse_name != "FeatureExtractionPipelineTests":
        # Exception encountered when calling layer '...'
        return True

     return False




  def  _UpperCAmelCase  (  self			)      ->			Optional[int]:
     UpperCamelCase_						=  TFMBartModelTester(self			)
     UpperCamelCase_						=  ConfigTester(self		,		config_class=_UpperCAmelCase			)




  def  _UpperCAmelCase  (  self			)      ->			Optional[int]:
     self.config_tester.run_common_tests()




  def  _UpperCAmelCase  (  self			)      ->			List[Any]:
     UpperCamelCase_						=  self.model_tester.prepare_config_and_inputs_for_common()
     self.model_tester.check_decoder_model_past_large_inputs(*_UpperCAmelCase			)




@require_sentencepiece
@require_tokenizers
@require_tf
class     _a							(	unittest.TestCase     ):






  """simple docstring"""





  A_						       = [
      """ UN Chief Says There Is No Military Solution in Syria""",
  ]
  A_						       = [
      """Şeful ONU declară că nu există o soluţie militară în Siria""",
  ]
  A_						       = """facebook/mbart-large-en-ro"""




  @cached_property
  def  _UpperCAmelCase  (  self			)      ->			Any:
     return AutoTokenizer.from_pretrained(self.model_name			)




  @cached_property
  def  _UpperCAmelCase  (  self			)      ->			List[str]:
     UpperCamelCase_						=  TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name			)
     return model




  def  _UpperCAmelCase  (  self		,		**_UpperCAmelCase			)      ->			int:
     UpperCamelCase_						=  self.translate_src_text(**_UpperCAmelCase			)
     self.assertListEqual(self.expected_text		,		_UpperCAmelCase			)




  def  _UpperCAmelCase  (  self		,		**_UpperCAmelCase			)      ->			List[str]:
     UpperCamelCase_						=  self.tokenizer(self.src_text		,		**_UpperCAmelCase		,		return_tensors='tf'			)
     UpperCamelCase_						=  self.model.generate(
         model_inputs.input_ids		,		attention_mask=model_inputs.attention_mask		,		num_beams=2			)
     UpperCamelCase_						=  self.tokenizer.batch_decode(_UpperCAmelCase		,		skip_special_tokens=_UpperCAmelCase			)
     return generated_words




  @slow
  def  _UpperCAmelCase  (  self			)      ->			List[Any]:
     self._assert_generated_batch_equal_expected()





