





from manim import *




class 			__UpperCamelCase    (							A__    ):





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   Rectangle(height=0.5				,							width=0.5					)
        _UpperCAmelCase           =   Rectangle(height=0.46				,							width=0.46					).set_stroke(width=0					)

        _UpperCAmelCase           =   Rectangle(height=0.25				,							width=0.25					)

        _UpperCAmelCase           =   [mem.copy() for i in range(6					)]
        _UpperCAmelCase           =   [mem.copy() for i in range(6					)]
        _UpperCAmelCase           =   VGroup(*_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0					)
        _UpperCAmelCase           =   VGroup(*_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0					)
        _UpperCAmelCase           =   VGroup(_UpperCamelCase				,							_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0					)
        _UpperCAmelCase           =   Text('''CPU'''				,							font_size=24					)
        _UpperCAmelCase           =   Group(_UpperCamelCase				,							_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0.5				,							aligned_edge=_UpperCamelCase					)
        cpu.move_to([-2.5, -0.5, 0]					)
        self.add(_UpperCamelCase					)

        _UpperCAmelCase           =   [mem.copy() for i in range(4					)]
        _UpperCAmelCase           =   VGroup(*_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0					)
        _UpperCAmelCase           =   Text('''GPU'''				,							font_size=24					)
        _UpperCAmelCase           =   Group(_UpperCamelCase				,							_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0.5				,							aligned_edge=_UpperCamelCase					)
        gpu.move_to([-1, -1, 0]					)
        self.add(_UpperCamelCase					)

        _UpperCAmelCase           =   [mem.copy() for i in range(6					)]
        _UpperCAmelCase           =   VGroup(*_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0					)

        _UpperCAmelCase           =   Text('''Model'''				,							font_size=24					)
        _UpperCAmelCase           =   Group(_UpperCamelCase				,							_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0.5				,							aligned_edge=_UpperCamelCase					)
        model.move_to([3, -1.0, 0]					)
        self.add(_UpperCamelCase					)

        _UpperCAmelCase           =   []
        _UpperCAmelCase           =   []

        for i, rect in enumerate(_UpperCamelCase					):
           _UpperCAmelCase           =   fill.copy().set_fill(_UpperCamelCase				,							opacity=0.8					)
           target.move_to(_UpperCamelCase					)
           model_arr.append(_UpperCamelCase					)

           _UpperCAmelCase           =   Rectangle(height=0.46				,							width=0.46					).set_stroke(width=0.0					).set_fill(_UpperCamelCase				,							opacity=0.8					)
           cpu_target.move_to(cpu_left_col_base[i]					)
           model_cpu_arr.append(_UpperCamelCase					)

        self.add(*_UpperCamelCase				,							*_UpperCamelCase					)

        _UpperCAmelCase           =   [meta_mem.copy() for i in range(6					)]
        _UpperCAmelCase           =   [meta_mem.copy() for i in range(6					)]
        _UpperCAmelCase           =   VGroup(*_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0					)
        _UpperCAmelCase           =   VGroup(*_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0					)
        _UpperCAmelCase           =   VGroup(_UpperCamelCase				,							_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0					)
        _UpperCAmelCase           =   Text('''Disk'''				,							font_size=24					)
        _UpperCAmelCase           =   Group(_UpperCamelCase				,							_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0.5				,							aligned_edge=_UpperCamelCase					)
        disk.move_to([-4, -1.25, 0]					)
        self.add(_UpperCamelCase				,							_UpperCamelCase					)

        _UpperCAmelCase           =   Square(side_length=2.2					)
        key.move_to([-5, 2, 0]					)

        _UpperCAmelCase           =   MarkupText(
            f'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''				,							font_size=18				,							)

        key_text.move_to([-5, 2.4, 0]					)

        self.add(_UpperCamelCase				,							_UpperCamelCase					)

        _UpperCAmelCase           =   MarkupText(
            f'''<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'''				,							font_size=18				,							)

        blue_text.next_to(_UpperCamelCase				,							DOWN * 2.4				,							aligned_edge=key_text.get_left()					)
        self.add(_UpperCamelCase					)

        _UpperCAmelCase           =   MarkupText(
            f'''Now watch as an input is passed through the model\nand how the memory is utilized and handled.'''				,							font_size=24				,							)
        step_a.move_to([2, 2, 0]					)

        self.play(Write(_UpperCamelCase					)					)

        _UpperCAmelCase           =   Square(0.3					)
        input.set_fill(_UpperCamelCase				,							opacity=1.0					)
        input.set_stroke(width=0.0					)
        input.next_to(model_base[0]				,							_UpperCamelCase				,							buff=0.5					)

        self.play(Write(_UpperCamelCase					)					)

        input.generate_target()
        input.target.next_to(model_arr[0]				,							direction=_UpperCamelCase				,							buff=0.02					)
        self.play(MoveToTarget(_UpperCamelCase					)					)

        self.play(FadeOut(_UpperCamelCase					)					)

        _UpperCAmelCase           =   Arrow(start=_UpperCamelCase				,							end=_UpperCamelCase				,							color=_UpperCamelCase				,							buff=0.5					)
        a.next_to(model_arr[0].get_left()				,							_UpperCamelCase				,							buff=0.2					)

        model_cpu_arr[0].generate_target()
        model_cpu_arr[0].target.move_to(gpu_rect[0]					)

        _UpperCAmelCase           =   MarkupText(
            f'''As the input reaches a layer, the hook triggers\nand weights are moved from the CPU\nto the GPU and back.'''				,							font_size=24				,							)
        step_a.move_to([2, 2, 0]					)

        self.play(Write(_UpperCamelCase				,							run_time=3					)					)

        _UpperCAmelCase           =   {'''run_time''': 1, '''fade_in''': True, '''fade_out''': True, '''buff''': 0.02}

        self.play(
            Write(_UpperCamelCase					)				,							Circumscribe(model_arr[0]				,							color=_UpperCamelCase				,							**_UpperCamelCase					)				,							Circumscribe(model_cpu_arr[0]				,							color=_UpperCamelCase				,							**_UpperCamelCase					)				,							Circumscribe(gpu_rect[0]				,							color=_UpperCamelCase				,							**_UpperCamelCase					)				,							)
        self.play(MoveToTarget(model_cpu_arr[0]					)					)

        _UpperCAmelCase           =   a.copy()
        for i in range(6					):
           a_c.next_to(model_arr[i].get_right() + 0.02				,							_UpperCamelCase				,							buff=0.2					)

           input.generate_target()
           input.target.move_to(model_arr[i].get_right() + 0.02					)

           _UpperCAmelCase           =   AnimationGroup(
               FadeOut(_UpperCamelCase				,							run_time=0.5					)				,							MoveToTarget(_UpperCamelCase				,							run_time=0.5					)				,							FadeIn(_UpperCamelCase				,							run_time=0.5					)				,							lag_ratio=0.2					)

           self.play(_UpperCamelCase					)

           model_cpu_arr[i].generate_target()
           model_cpu_arr[i].target.move_to(cpu_left_col_base[i]					)

           if i < 5:
              model_cpu_arr[i + 1].generate_target()
              model_cpu_arr[i + 1].target.move_to(gpu_rect[0]					)
              if i >= 1:
                 _UpperCAmelCase           =   0.7

              self.play(
                  Circumscribe(model_arr[i]				,							**_UpperCamelCase					)				,							Circumscribe(cpu_left_col_base[i]				,							**_UpperCamelCase					)				,							Circumscribe(cpu_left_col_base[i + 1]				,							color=_UpperCamelCase				,							**_UpperCamelCase					)				,							Circumscribe(gpu_rect[0]				,							color=_UpperCamelCase				,							**_UpperCamelCase					)				,							Circumscribe(model_arr[i + 1]				,							color=_UpperCamelCase				,							**_UpperCamelCase					)				,							)
              if i < 1:
                 self.play(
                     MoveToTarget(model_cpu_arr[i]					)				,							MoveToTarget(model_cpu_arr[i + 1]					)				,							)
              else:
                 self.play(
                     MoveToTarget(model_cpu_arr[i]				,							run_time=0.7					)				,							MoveToTarget(model_cpu_arr[i + 1]				,							run_time=0.7					)				,							)
           else:
              model_cpu_arr[i].generate_target()
              model_cpu_arr[i].target.move_to(cpu_left_col_base[-1]					)
              input.generate_target()
              input.target.next_to(model_arr[-1].get_right()				,							RIGHT + 0.02				,							buff=0.2					)

              self.play(
                  Circumscribe(model_arr[-1]				,							color=_UpperCamelCase				,							**_UpperCamelCase					)				,							Circumscribe(cpu_left_col_base[-1]				,							color=_UpperCamelCase				,							**_UpperCamelCase					)				,							Circumscribe(gpu_rect[0]				,							color=_UpperCamelCase				,							**_UpperCamelCase					)				,							)

              self.play(MoveToTarget(model_cpu_arr[i]					)					)

           _UpperCAmelCase           =   a_c
           _UpperCAmelCase           =   a_c.copy()

        input.generate_target()
        input.target.next_to(model_base[-1]				,							RIGHT + 0.02				,							buff=0.5					)
        self.play(
            FadeOut(_UpperCamelCase					)				,							FadeOut(_UpperCamelCase				,							run_time=0.5					)				,							)

        _UpperCAmelCase           =   MarkupText(f'''Inference on a model too large for GPU memory\nis successfully completed.'''				,							font_size=24					)
        step_a.move_to([2, 2, 0]					)

        self.play(Write(_UpperCamelCase				,							run_time=3					)				,							MoveToTarget(_UpperCamelCase					)					)

        self.wait()






'''simple docstring'''




from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


a__					:			str          =     logging.get_logger(__name__)

a__					:			Any          =     {
    '''xlm-mlm-en-2048''': '''https://huggingface.co/xlm-mlm-en-2048/resolve/main/config.json''',
    '''xlm-mlm-ende-1024''': '''https://huggingface.co/xlm-mlm-ende-1024/resolve/main/config.json''',
    '''xlm-mlm-enfr-1024''': '''https://huggingface.co/xlm-mlm-enfr-1024/resolve/main/config.json''',
    '''xlm-mlm-enro-1024''': '''https://huggingface.co/xlm-mlm-enro-1024/resolve/main/config.json''',
    '''xlm-mlm-tlm-xnli15-1024''': '''https://huggingface.co/xlm-mlm-tlm-xnli15-1024/resolve/main/config.json''',
    '''xlm-mlm-xnli15-1024''': '''https://huggingface.co/xlm-mlm-xnli15-1024/resolve/main/config.json''',
    '''xlm-clm-enfr-1024''': '''https://huggingface.co/xlm-clm-enfr-1024/resolve/main/config.json''',
    '''xlm-clm-ende-1024''': '''https://huggingface.co/xlm-clm-ende-1024/resolve/main/config.json''',
    '''xlm-mlm-17-1280''': '''https://huggingface.co/xlm-mlm-17-1280/resolve/main/config.json''',
    '''xlm-mlm-100-1280''': '''https://huggingface.co/xlm-mlm-100-1280/resolve/main/config.json''',
}






class 				__snake_case     (					__magic_name__							):
	__lowerCAmelCase       						=  '''xlm'''
	__lowerCAmelCase       						=  {
	    '''hidden_size''': '''emb_dim''',
	    '''num_attention_heads''': '''n_heads''',
	    '''num_hidden_layers''': '''n_layers''',
	    '''n_words''': '''vocab_size''',  # For backward compatibility
	}





	def __init__(		self      ,       UpperCamelCase_=3_0145      ,       UpperCamelCase_=2048      ,       UpperCamelCase_=12      ,       UpperCamelCase_=16      ,       UpperCamelCase_=0.1      ,       UpperCamelCase_=0.1      ,       UpperCamelCase_=True      ,       UpperCamelCase_=False      ,       UpperCamelCase_=False      ,       UpperCamelCase_=False      ,       UpperCamelCase_=1      ,       UpperCamelCase_=True      ,       UpperCamelCase_=512      ,       UpperCamelCase_=2048**-0.5      ,       UpperCamelCase_=1E-1_2      ,       UpperCamelCase_=0.0_2      ,       UpperCamelCase_=0      ,       UpperCamelCase_=1      ,       UpperCamelCase_=2      ,       UpperCamelCase_=3      ,       UpperCamelCase_=5      ,       UpperCamelCase_=True      ,       UpperCamelCase_="first"      ,       UpperCamelCase_=True      ,       UpperCamelCase_=None      ,       UpperCamelCase_=True      ,       UpperCamelCase_=0.1      ,       UpperCamelCase_=5      ,       UpperCamelCase_=5      ,       UpperCamelCase_=0      ,       UpperCamelCase_=0      ,       UpperCamelCase_=2      ,       UpperCamelCase_=0      ,       **UpperCamelCase_      ,       )      ->		List[str]:
		snake_case__	       =   vocab_size
		snake_case__	       =   emb_dim
		snake_case__	       =   n_layers
		snake_case__	       =   n_heads
		snake_case__	       =   dropout
		snake_case__	       =   attention_dropout
		snake_case__	       =   gelu_activation
		snake_case__	       =   sinusoidal_embeddings
		snake_case__	       =   causal
		snake_case__	       =   asm
		snake_case__	       =   n_langs
		snake_case__	       =   use_lang_emb
		snake_case__	       =   layer_norm_eps
		snake_case__	       =   bos_index
		snake_case__	       =   eos_index
		snake_case__	       =   pad_index
		snake_case__	       =   unk_index
		snake_case__	       =   mask_index
		snake_case__	       =   is_encoder
		snake_case__	       =   max_position_embeddings
		snake_case__	       =   embed_init_std
		snake_case__	       =   init_std
		snake_case__	       =   summary_type
		snake_case__	       =   summary_use_proj
		snake_case__	       =   summary_activation
		snake_case__	       =   summary_proj_to_labels
		snake_case__	       =   summary_first_dropout
		snake_case__	       =   start_n_top
		snake_case__	       =   end_n_top
		snake_case__	       =   mask_token_id
		snake_case__	       =   lang_id

		if "n_words" in kwargs:
			snake_case__	       =   kwargs['n_words']

		super().__init__(pad_token_id=UpperCamelCase_      ,       bos_token_id=UpperCamelCase_      ,       **UpperCamelCase_		)






class 				__snake_case     (					__magic_name__							):





	@property
	def   _snake_case   (		self		)      ->		Mapping[str, Mapping[int, str]]:
		if self.task == "multiple-choice":
			snake_case__	       =   {0: 'batch', 1: 'choice', 2: 'sequence'}
		else:
			snake_case__	       =   {0: 'batch', 1: 'sequence'}
		return OrderedDict(
		    [
		        ('input_ids', dynamic_axis),
		        ('attention_mask', dynamic_axis),
		        ('token_type_ids', dynamic_axis),
		    ]		)



