
"""simple docstring"""




from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase__   :					List[Any]												=				logging.get_logger(__name__)

UpperCAmelCase__   :					Optional[Any]												=				{
    'facebook/nllb-moe-54B': 'https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json',
}



class 			lowerCAmelCase_		(a__						):


    """simple docstring"""



    __UpperCamelCase   :		Any       =     '''nllb-moe'''
    __UpperCamelCase   :		str       =     ['''past_key_values''']
    __UpperCamelCase   :		List[str]       =     {'''num_attention_heads''': '''encoder_attention_heads''', '''hidden_size''': '''d_model'''}



    def __init__(self     , SCREAMING_SNAKE_CASE__=12_81_12     , SCREAMING_SNAKE_CASE__=10_24     , SCREAMING_SNAKE_CASE__=12     , SCREAMING_SNAKE_CASE__=40_96     , SCREAMING_SNAKE_CASE__=16     , SCREAMING_SNAKE_CASE__=12     , SCREAMING_SNAKE_CASE__=40_96     , SCREAMING_SNAKE_CASE__=16     , SCREAMING_SNAKE_CASE__=0.05     , SCREAMING_SNAKE_CASE__=0.05     , SCREAMING_SNAKE_CASE__=True     , SCREAMING_SNAKE_CASE__=True     , SCREAMING_SNAKE_CASE__="relu"     , SCREAMING_SNAKE_CASE__=10_24     , SCREAMING_SNAKE_CASE__=0.1     , SCREAMING_SNAKE_CASE__=0.1     , SCREAMING_SNAKE_CASE__=0.0     , SCREAMING_SNAKE_CASE__=0.02     , SCREAMING_SNAKE_CASE__=2     , SCREAMING_SNAKE_CASE__=True     , SCREAMING_SNAKE_CASE__=False     , SCREAMING_SNAKE_CASE__="float32"     , SCREAMING_SNAKE_CASE__=False     , SCREAMING_SNAKE_CASE__=1_28     , SCREAMING_SNAKE_CASE__=64     , SCREAMING_SNAKE_CASE__=4     , SCREAMING_SNAKE_CASE__=4     , SCREAMING_SNAKE_CASE__=0.001     , SCREAMING_SNAKE_CASE__=0.001     , SCREAMING_SNAKE_CASE__="all"     , SCREAMING_SNAKE_CASE__=False     , SCREAMING_SNAKE_CASE__=False     , SCREAMING_SNAKE_CASE__=1.0     , SCREAMING_SNAKE_CASE__=0.2     , SCREAMING_SNAKE_CASE__=1     , SCREAMING_SNAKE_CASE__=0     , SCREAMING_SNAKE_CASE__=2     , SCREAMING_SNAKE_CASE__=False     , **SCREAMING_SNAKE_CASE__     , )      ->	Dict:



           """simple docstring"""




           SCREAMING_SNAKE_CASE__		: Dict     =     vocab_size
           SCREAMING_SNAKE_CASE__		: int     =     max_position_embeddings
           SCREAMING_SNAKE_CASE__		: Optional[Any]     =     d_model
           SCREAMING_SNAKE_CASE__		: int     =     encoder_ffn_dim
           SCREAMING_SNAKE_CASE__		: Union[str, Any]     =     encoder_layers
           SCREAMING_SNAKE_CASE__		: int     =     encoder_attention_heads
           SCREAMING_SNAKE_CASE__		: Optional[int]     =     decoder_ffn_dim
           SCREAMING_SNAKE_CASE__		: List[Any]     =     decoder_layers
           SCREAMING_SNAKE_CASE__		: Dict     =     decoder_attention_heads
           SCREAMING_SNAKE_CASE__		: Optional[int]     =     dropout
           SCREAMING_SNAKE_CASE__		: Optional[Any]     =     attention_dropout
           SCREAMING_SNAKE_CASE__		: List[Any]     =     activation_dropout
           SCREAMING_SNAKE_CASE__		: Tuple     =     activation_function
           SCREAMING_SNAKE_CASE__		: List[Any]     =     init_std
           SCREAMING_SNAKE_CASE__		: Tuple     =     encoder_layerdrop
           SCREAMING_SNAKE_CASE__		: Optional[int]     =     decoder_layerdrop
           SCREAMING_SNAKE_CASE__		: List[Any]     =     use_cache
           SCREAMING_SNAKE_CASE__		: Union[str, Any]     =     encoder_layers
           SCREAMING_SNAKE_CASE__		: Optional[Any]     =     scale_embedding  # scale factor will be sqrt(d_model) if True
           SCREAMING_SNAKE_CASE__		: Any     =     router_z_loss_coef
           SCREAMING_SNAKE_CASE__		: int     =     router_aux_loss_coef
           SCREAMING_SNAKE_CASE__		: List[str]     =     decoder_sparse_step
           SCREAMING_SNAKE_CASE__		: Dict     =     encoder_sparse_step
           SCREAMING_SNAKE_CASE__		: List[Any]     =     num_experts
           SCREAMING_SNAKE_CASE__		: Optional[Any]     =     expert_capacity
           SCREAMING_SNAKE_CASE__		: List[str]     =     router_bias
           if router_dtype not in ["float32", "float16", "bfloat16"]:
                  raise ValueError(F'''`router_dtype` must be one of \'float32\', \'float16\' or \'bfloat16\', got {router_dtype}'''					)
           SCREAMING_SNAKE_CASE__		: List[Any]     =     router_dtype

           SCREAMING_SNAKE_CASE__		: Dict     =     router_ignore_padding_tokens
           SCREAMING_SNAKE_CASE__		: str     =     batch_prioritized_routing
           SCREAMING_SNAKE_CASE__		: Union[str, Any]     =     second_expert_policy
           SCREAMING_SNAKE_CASE__		: Tuple     =     normalize_router_prob_before_dropping
           SCREAMING_SNAKE_CASE__		: List[Any]     =     moe_eval_capacity_token_fraction
           SCREAMING_SNAKE_CASE__		: int     =     moe_token_dropout
           SCREAMING_SNAKE_CASE__		: Dict     =     output_router_logits
           super().__init__(
               pad_token_id=SCREAMING_SNAKE_CASE__     , bos_token_id=SCREAMING_SNAKE_CASE__     , eos_token_id=SCREAMING_SNAKE_CASE__     , is_encoder_decoder=SCREAMING_SNAKE_CASE__     , decoder_start_token_id=SCREAMING_SNAKE_CASE__     , **SCREAMING_SNAKE_CASE__     , )






"""simple docstring"""




from manim import *



class 			lowerCAmelCase_		(a__						):


				"""simple docstring"""



				def 			__magic_name__					(self					)      ->	List[str]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		: Dict     =     Rectangle(height=0.5     , width=0.5					)
											SCREAMING_SNAKE_CASE__		: Union[str, Any]     =     Rectangle(height=0.46     , width=0.46					).set_stroke(width=0					)

											SCREAMING_SNAKE_CASE__		: Union[str, Any]     =     [mem.copy() for i in range(6					)]
											SCREAMING_SNAKE_CASE__		: str     =     [mem.copy() for i in range(6					)]
											SCREAMING_SNAKE_CASE__		: Union[str, Any]     =     VGroup(*SCREAMING_SNAKE_CASE__					).arrange(SCREAMING_SNAKE_CASE__     , buff=0					)
											SCREAMING_SNAKE_CASE__		: Optional[Any]     =     VGroup(*SCREAMING_SNAKE_CASE__					).arrange(SCREAMING_SNAKE_CASE__     , buff=0					)
											SCREAMING_SNAKE_CASE__		: Tuple     =     VGroup(SCREAMING_SNAKE_CASE__     , SCREAMING_SNAKE_CASE__					).arrange(SCREAMING_SNAKE_CASE__     , buff=0					)
											SCREAMING_SNAKE_CASE__		: int     =     Text("""CPU"""     , font_size=24					)
											SCREAMING_SNAKE_CASE__		: Any     =     Group(SCREAMING_SNAKE_CASE__     , SCREAMING_SNAKE_CASE__					).arrange(SCREAMING_SNAKE_CASE__     , buff=0.5     , aligned_edge=SCREAMING_SNAKE_CASE__					)
											cpu.move_to([-2.5, -0.5, 0]					)
											self.add(SCREAMING_SNAKE_CASE__					)

											SCREAMING_SNAKE_CASE__		: Optional[int]     =     [mem.copy() for i in range(1					)]
											SCREAMING_SNAKE_CASE__		: Optional[Any]     =     VGroup(*SCREAMING_SNAKE_CASE__					).arrange(SCREAMING_SNAKE_CASE__     , buff=0					)
											SCREAMING_SNAKE_CASE__		: str     =     Text("""GPU"""     , font_size=24					)
											SCREAMING_SNAKE_CASE__		: str     =     Group(SCREAMING_SNAKE_CASE__     , SCREAMING_SNAKE_CASE__					).arrange(SCREAMING_SNAKE_CASE__     , buff=0.5     , aligned_edge=SCREAMING_SNAKE_CASE__					)
											gpu.align_to(SCREAMING_SNAKE_CASE__     , SCREAMING_SNAKE_CASE__					)
											gpu.set_x(gpu.get_x() - 1					)

											self.add(SCREAMING_SNAKE_CASE__					)

											SCREAMING_SNAKE_CASE__		: List[Any]     =     [mem.copy() for i in range(6					)]
											SCREAMING_SNAKE_CASE__		: List[Any]     =     VGroup(*SCREAMING_SNAKE_CASE__					).arrange(SCREAMING_SNAKE_CASE__     , buff=0					)

											SCREAMING_SNAKE_CASE__		: Optional[Any]     =     Text("""Model"""     , font_size=24					)
											SCREAMING_SNAKE_CASE__		: Any     =     Group(SCREAMING_SNAKE_CASE__     , SCREAMING_SNAKE_CASE__					).arrange(SCREAMING_SNAKE_CASE__     , buff=0.5     , aligned_edge=SCREAMING_SNAKE_CASE__					)
											model.move_to([3, -1.0, 0]					)

											self.play(
											    Create(SCREAMING_SNAKE_CASE__     , run_time=1					)     , Create(SCREAMING_SNAKE_CASE__     , run_time=1					)     , Create(SCREAMING_SNAKE_CASE__     , run_time=1					)     , )

											SCREAMING_SNAKE_CASE__		: Tuple     =     MarkupText(
											    F'''First, an empty model skeleton is loaded\ninto <span fgcolor=\'{YELLOW}\'>memory</span> without using much RAM.'''     , font_size=24     , )

											SCREAMING_SNAKE_CASE__		: Dict     =     Square(side_length=2.2					)
											key.move_to([-5, 2, 0]					)

											SCREAMING_SNAKE_CASE__		: Tuple     =     MarkupText(
											    F'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''     , font_size=18     , )

											key_text.move_to([-5, 2.4, 0]					)

											step_a.move_to([2, 2, 0]					)
											self.play(Write(SCREAMING_SNAKE_CASE__     , run_time=2.5					)     , Write(SCREAMING_SNAKE_CASE__					)     , Write(SCREAMING_SNAKE_CASE__					)					)

											self.add(SCREAMING_SNAKE_CASE__					)

											SCREAMING_SNAKE_CASE__		: Optional[int]     =     []
											SCREAMING_SNAKE_CASE__		: Union[str, Any]     =     []
											SCREAMING_SNAKE_CASE__		: str     =     []
											for i, rect in enumerate(SCREAMING_SNAKE_CASE__					):
																		SCREAMING_SNAKE_CASE__		: Union[str, Any]     =     Rectangle(height=0.46     , width=0.46					).set_stroke(width=0.0					).set_fill(SCREAMING_SNAKE_CASE__     , opacity=0.7					)
																		cpu_target.move_to(SCREAMING_SNAKE_CASE__					)
																		cpu_target.generate_target()
																		SCREAMING_SNAKE_CASE__		: Tuple     =     0.46 / 4
																		SCREAMING_SNAKE_CASE__		: int     =     0.46 / 3

																		if i == 0:
																									cpu_target.target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT					)     , buff=0.02     , direction=SCREAMING_SNAKE_CASE__					)
																									cpu_target.target.set_x(cpu_target.target.get_x() + 0.1					)
																		elif i == 3:
																									cpu_target.target.next_to(cpu_targs[0].target     , direction=SCREAMING_SNAKE_CASE__     , buff=0.0					)
																		else:
																									cpu_target.target.next_to(cpu_targs[i - 1].target     , direction=SCREAMING_SNAKE_CASE__     , buff=0.0					)
																		cpu_targs.append(SCREAMING_SNAKE_CASE__					)

																		first_animations.append(rect.animate(run_time=0.5					).set_stroke(SCREAMING_SNAKE_CASE__					)					)
																		second_animations.append(MoveToTarget(SCREAMING_SNAKE_CASE__     , run_time=1.5					)					)

											self.play(*SCREAMING_SNAKE_CASE__					)
											self.play(*SCREAMING_SNAKE_CASE__					)

											self.wait()




