





import operator as op


UpperCAmelCase_							=				"""scaler.pt"""
UpperCAmelCase_							=				"""pytorch_model"""
UpperCAmelCase_							=				"""random_states"""
UpperCAmelCase_							=				"""optimizer"""
UpperCAmelCase_							=				"""scheduler"""
UpperCAmelCase_							=				"""pytorch_model.bin"""
UpperCAmelCase_							=				"""pytorch_model.bin.index.json"""
UpperCAmelCase_							=				"""model.safetensors"""
UpperCAmelCase_							=				"""model.safetensors.index.json"""
UpperCAmelCase_							=				"""1.10.2"""
UpperCAmelCase_							=				"""py38"""
UpperCAmelCase_							=				"""4.17.0"""
UpperCAmelCase_							=				["""ml.p3.16xlarge""", """ml.p3dn.24xlarge""", """ml.p4dn.24xlarge"""]
UpperCAmelCase_							=				["""FULL_SHARD""", """SHARD_GRAD_OP""", """NO_SHARD""", """HYBRID_SHARD""", """HYBRID_SHARD_ZERO2"""]
UpperCAmelCase_							=				["""TRANSFORMER_BASED_WRAP""", """SIZE_BASED_WRAP""", """NO_WRAP"""]
UpperCAmelCase_							=				["""BACKWARD_PRE""", """BACKWARD_POST""", """NO_PREFETCH"""]
UpperCAmelCase_							=				["""FULL_STATE_DICT""", """LOCAL_STATE_DICT""", """SHARDED_STATE_DICT"""]
UpperCAmelCase_							=				"""2.0.1"""
UpperCAmelCase_							=				["""pdsh""", """standard""", """openmpi""", """mvapich"""]
UpperCAmelCase_							=				["""default""", """reduce-overhead""", """max-autotune"""]

UpperCAmelCase_							=				{""">""": op.gt, """>=""": op.ge, """==""": op.eq, """!=""": op.ne, """<=""": op.le, """<""": op.lt}

# These are the args for `torch.distributed.launch` for pytorch < 1.9
UpperCAmelCase_							=				[
    """nnodes""",
    """nproc_per_node""",
    """rdzv_backend""",
    """rdzv_endpoint""",
    """rdzv_id""",
    """rdzv_conf""",
    """standalone""",
    """max_restarts""",
    """monitor_interval""",
    """start_method""",
    """role""",
    """module""",
    """m""",
    """no_python""",
    """run_path""",
    """log_dir""",
    """r""",
    """redirects""",
    """t""",
    """tee""",
    """node_rank""",
    """master_addr""",
    """master_port""",
]

UpperCAmelCase_							=				["""DEEPSPEED""", """MULTI_GPU""", """FSDP""", """MEGATRON_LM"""]
UpperCAmelCase_							=				["""DEEPSPEED""", """MULTI_XPU""", """FSDP"""]






import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import TaBlock, TaConfig, TaLayerNorm

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin
class     UpperCamelCase__					(     lowerCamelCase__				, lowerCamelCase__				, lowerCamelCase__				):






		'''simple docstring'''






		@register_to_config
		def __init__(							self,   snake_case__,   snake_case__,   snake_case__,   snake_case__,   snake_case__,   snake_case__,   snake_case__,   snake_case__,   snake_case__,   snake_case__ = False,   )       -> Any:






									"""simple docstring"""
									super().__init__()

									lowercase_	:   str  				=     nn.Embedding(snake_case__,   snake_case__  )

									lowercase_	:   Dict  				=     nn.Embedding(snake_case__,   snake_case__  )
									lowercase_	:   List[str]  				=     False

									lowercase_	:   Optional[int]  				=     nn.Dropout(p=snake_case__  )

									lowercase_	:   Any  				=     TaConfig(
									    vocab_size=snake_case__,   d_model=snake_case__,   num_heads=snake_case__,   d_kv=snake_case__,   d_ff=snake_case__,   dropout_rate=snake_case__,   feed_forward_proj=snake_case__,   is_decoder=snake_case__,   is_encoder_decoder=snake_case__,   )

									lowercase_	:   List[Any]  				=     nn.ModuleList()
									for lyr_num in range(snake_case__  ):
																lowercase_	:   List[str]  				=     TaBlock(snake_case__  )
																self.encoders.append(snake_case__  )

									lowercase_	:   Tuple  				=     TaLayerNorm(snake_case__  )
									lowercase_	:   Dict  				=     nn.Dropout(p=snake_case__  )






		def 						snake_case__				(							self,   snake_case__,   snake_case__  )       -> Union[str, Any]:






									"""simple docstring"""
									lowercase_	:   Union[str, Any]  				=     self.token_embedder(snake_case__  )

									lowercase_	:   Tuple  				=     encoder_input_tokens.shape[1]
									lowercase_	:   List[str]  				=     torch.arange(snake_case__,   device=encoder_input_tokens.device  )
									x += self.position_encoding(snake_case__  )

									lowercase_	:   Union[str, Any]  				=     self.dropout_pre(snake_case__  )

									# inverted the attention mask
									lowercase_	:   List[str]  				=     encoder_input_tokens.size()
									lowercase_	:   Union[str, Any]  				=     self.get_extended_attention_mask(snake_case__,   snake_case__  )

									for lyr in self.encoders:
																lowercase_	:   Optional[Any]  				=     lyr(snake_case__,   snake_case__  )[0]
									lowercase_	:   Dict  				=     self.layer_norm(snake_case__  )

									return self.dropout_post(snake_case__  ), encoder_inputs_mask