
from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase_    :			Any    =	logging.get_logger(__name__)

lowerCamelCase_    :			Optional[int]    =	{
    """facebook/nllb-moe-54B""": """https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json""",
}

class 							_UpperCamelCase					(       _A	):






			'''simple docstring'''

			__UpperCamelCase   :    Union[str, Any]          =			"""nllb-moe"""
			__UpperCamelCase   :    List[Any]          =			["""past_key_values"""]
			__UpperCamelCase   :    Union[str, Any]          =			{"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}

			def __init__(							self			:							Optional[Any]    ,						snake_case_			:							List[Any]=12_8112    ,						snake_case_			:							List[Any]=1024    ,						snake_case_			:							Dict=12    ,						snake_case_			:							Optional[Any]=4096    ,						snake_case_			:							Any=16    ,						snake_case_			:							Union[str, Any]=12    ,						snake_case_			:							Optional[Any]=4096    ,						snake_case_			:							str=16    ,						snake_case_			:							int=0.05    ,						snake_case_			:							Any=0.05    ,						snake_case_			:							int=True    ,						snake_case_			:							List[str]=True    ,						snake_case_			:							Tuple="relu"    ,						snake_case_			:							Union[str, Any]=1024    ,						snake_case_			:							Union[str, Any]=0.1    ,						snake_case_			:							Optional[int]=0.1    ,						snake_case_			:							int=0.0    ,						snake_case_			:							Optional[int]=0.02    ,						snake_case_			:							Dict=2    ,						snake_case_			:							Any=True    ,						snake_case_			:							Optional[int]=False    ,						snake_case_			:							Dict="float32"    ,						snake_case_			:							List[Any]=False    ,						snake_case_			:							Tuple=128    ,						snake_case_			:							Dict=64    ,						snake_case_			:							List[Any]=4    ,						snake_case_			:							List[str]=4    ,						snake_case_			:							List[str]=0.001    ,						snake_case_			:							str=0.001    ,						snake_case_			:							int="all"    ,						snake_case_			:							Any=False    ,						snake_case_			:							Dict=False    ,						snake_case_			:							Optional[int]=1.0    ,						snake_case_			:							str=0.2    ,						snake_case_			:							Optional[Any]=1    ,						snake_case_			:							Union[str, Any]=0    ,						snake_case_			:							str=2    ,						snake_case_			:							Any=False    ,						**snake_case_			:							str    ,						):
							UpperCamelCase_:				Tuple						=    vocab_size
							UpperCamelCase_:				List[Any]						=    max_position_embeddings
							UpperCamelCase_:				Optional[Any]						=    d_model
							UpperCamelCase_:				str						=    encoder_ffn_dim
							UpperCamelCase_:				Optional[Any]						=    encoder_layers
							UpperCamelCase_:				int						=    encoder_attention_heads
							UpperCamelCase_:				int						=    decoder_ffn_dim
							UpperCamelCase_:				Union[str, Any]						=    decoder_layers
							UpperCamelCase_:				Tuple						=    decoder_attention_heads
							UpperCamelCase_:				Any						=    dropout
							UpperCamelCase_:				Any						=    attention_dropout
							UpperCamelCase_:				Optional[Any]						=    activation_dropout
							UpperCamelCase_:				Union[str, Any]						=    activation_function
							UpperCamelCase_:				Any						=    init_std
							UpperCamelCase_:				Optional[Any]						=    encoder_layerdrop
							UpperCamelCase_:				List[str]						=    decoder_layerdrop
							UpperCamelCase_:				Dict						=    use_cache
							UpperCamelCase_:				Tuple						=    encoder_layers
							UpperCamelCase_:				Optional[int]						=    scale_embedding  # scale factor will be sqrt(d_model) if True
							UpperCamelCase_:				Optional[Any]						=    router_z_loss_coef
							UpperCamelCase_:				Dict						=    router_aux_loss_coef
							UpperCamelCase_:				Optional[int]						=    decoder_sparse_step
							UpperCamelCase_:				Union[str, Any]						=    encoder_sparse_step
							UpperCamelCase_:				Any						=    num_experts
							UpperCamelCase_:				Union[str, Any]						=    expert_capacity
							UpperCamelCase_:				Optional[int]						=    router_bias
							if router_dtype not in ["float32", "float16", "bfloat16"]:
											raise ValueError(f'''`router_dtype` must be one of \'float32\', \'float16\' or \'bfloat16\', got {router_dtype}'''   )
							UpperCamelCase_:				Optional[int]						=    router_dtype

							UpperCamelCase_:				str						=    router_ignore_padding_tokens
							UpperCamelCase_:				List[str]						=    batch_prioritized_routing
							UpperCamelCase_:				Optional[int]						=    second_expert_policy
							UpperCamelCase_:				List[str]						=    normalize_router_prob_before_dropping
							UpperCamelCase_:				Tuple						=    moe_eval_capacity_token_fraction
							UpperCamelCase_:				Union[str, Any]						=    moe_token_dropout
							UpperCamelCase_:				int						=    output_router_logits
							super().__init__(
							    pad_token_id=snake_case_    ,						bos_token_id=snake_case_    ,						eos_token_id=snake_case_    ,						is_encoder_decoder=snake_case_    ,						decoder_start_token_id=snake_case_    ,						**snake_case_    ,						)








import os
import sys
import tempfile

import torch

from .state import AcceleratorState
from .utils import PrecisionType, PrepareForLaunch, is_mps_available, patch_environment







def 							A__       (    lowerCamelCase  ,     lowerCamelCase=()  ,     lowerCamelCase=None  ,     lowerCamelCase="no"  ,     lowerCamelCase="29500"    )	->						List[Any]:
				UpperCamelCase_:				Any						=    False
				UpperCamelCase_:				int						=    False
				if any(key.startswith("""KAGGLE"""    ) for key in os.environ.keys()    ):
								UpperCamelCase_:				str						=    True
				elif "IPython" in sys.modules:
								UpperCamelCase_:				Tuple						=    """google.colab""" in str(sys.modules["""IPython"""].get_ipython()    )

				try:
								UpperCamelCase_:				Union[str, Any]						=    PrecisionType(mixed_precision.lower()    )
				except ValueError:
								raise ValueError(
								    F'''Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}.'''    )

				if (in_colab or in_kaggle) and (os.environ.get("""TPU_NAME"""  ,     lowerCamelCase    ) is not None):
								# TPU launch
								import torch_xla.distributed.xla_multiprocessing as xmp

								if len(AcceleratorState._shared_state    ) > 0:
												raise ValueError(
												    """To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside """
												    """your training function. Restart your notebook and make sure no cells initializes an """
												    """`Accelerator`."""    )
								if num_processes is None:
												UpperCamelCase_:				str						=    8

								UpperCamelCase_:				int						=    PrepareForLaunch(lowerCamelCase  ,     distributed_type="""TPU"""    )
								print(F'''Launching a training on {num_processes} TPU cores.'''    )
								xmp.spawn(lowerCamelCase  ,     args=lowerCamelCase  ,     nprocs=lowerCamelCase  ,     start_method="""fork"""    )
				elif in_colab:
								# No need for a distributed launch otherwise as it's either CPU or one GPU.
								if torch.cuda.is_available():
												print("""Launching training on one GPU."""    )
								else:
												print("""Launching training on one CPU."""    )
								function(*lowerCamelCase    )
				else:
								if num_processes is None:
												raise ValueError(
												    """You have to specify the number of GPUs you would like to use, add `num_processes=...` to your call."""    )

								if num_processes > 1:
												# Multi-GPU launch
												from torch.multiprocessing import start_processes
												from torch.multiprocessing.spawn import ProcessRaisedException

												if len(AcceleratorState._shared_state    ) > 0:
																raise ValueError(
																    """To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized """
																    """inside your training function. Restart your notebook and make sure no cells initializes an """
																    """`Accelerator`."""    )

												if torch.cuda.is_initialized():
																raise ValueError(
																    """To launch a multi-GPU training from your notebook, you need to avoid running any instruction """
																    """using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA """
																    """function."""    )

												# torch.distributed will expect a few environment variable to be here. We set the ones common to each
												# process here (the other ones will be set be the launcher).
												with patch_environment(
												    world_size=lowerCamelCase  ,     master_addr="""127.0.01"""  ,     master_port=lowerCamelCase  ,     mixed_precision=lowerCamelCase    ):
																UpperCamelCase_:				Optional[int]						=    PrepareForLaunch(lowerCamelCase  ,     distributed_type="""MULTI_GPU"""    )
																print(F'''Launching training on {num_processes} GPUs.'''    )
																try:
																				start_processes(lowerCamelCase  ,     args=lowerCamelCase  ,     nprocs=lowerCamelCase  ,     start_method="""fork"""    )
																except ProcessRaisedException as e:
																				if "Cannot re-initialize CUDA in forked subprocess" in e.args[0]:
																								raise RuntimeError(
																								    """CUDA has been initialized before the `notebook_launcher` could create a forked subprocess. """
																								    """This likely stems from an outside import causing issues once the `notebook_launcher()` is called. """
																								    """Please review your imports and test them when running the `notebook_launcher()` to identify """
																								    """which one is problematic."""    ) from e

								else:
												# No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
												if is_mps_available():
																UpperCamelCase_:				List[Any]						=    """1"""
																print("""Launching training on MPS."""    )
												elif torch.cuda.is_available():
																print("""Launching training on one GPU."""    )
												else:
																print("""Launching training on CPU."""    )
												function(*lowerCamelCase    )







def 							A__       (    lowerCamelCase  ,     lowerCamelCase=()  ,     lowerCamelCase=2    )	->						List[str]:
				from torch.multiprocessing import start_processes

				with tempfile.NamedTemporaryFile() as tmp_file:
								# torch.distributed will expect a few environment variable to be here. We set the ones common to each
								# process here (the other ones will be set be the launcher).
								with patch_environment(
								    world_size=lowerCamelCase  ,     master_addr="""127.0.01"""  ,     master_port="""29500"""  ,     accelerate_mixed_precision="""no"""  ,     accelerate_debug_rdv_file=tmp_file.name  ,     accelerate_use_cpu="""yes"""  ,     ):
												UpperCamelCase_:				str						=    PrepareForLaunch(lowerCamelCase  ,     debug=lowerCamelCase    )
												start_processes(lowerCamelCase  ,     args=lowerCamelCase  ,     nprocs=lowerCamelCase  ,     start_method="""fork"""    )






