


from dataclasses import asdict, dataclass
from typing import Optional

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase__			:      int								=logging.get_logger(__name__)

# TODO Update this
lowerCAmelCase__			:      Optional[int]								={
    'facebook/esm-1b': 'https://huggingface.co/facebook/esm-1b/resolve/main/config.json',
    # See all ESM models at https://huggingface.co/models?filter=esm
}







class 	__lowercase    (__SCREAMING_SNAKE_CASE  ):





							"""simple docstring"""







							_UpperCAmelCase      =						"""esm"""






							def __init__(      self			,						lowerCAmelCase__=None			,						lowerCAmelCase__=None			,						lowerCAmelCase__=None			,						lowerCAmelCase__=7_6_8			,						lowerCAmelCase__=1_2			,						lowerCAmelCase__=1_2			,						lowerCAmelCase__=3_0_7_2			,						lowerCAmelCase__=0.1			,						lowerCAmelCase__=0.1			,						lowerCAmelCase__=1_0_2_6			,						lowerCAmelCase__=0.02			,						lowerCAmelCase__=1E-12			,						lowerCAmelCase__="absolute"			,						lowerCAmelCase__=True			,						lowerCAmelCase__=None			,						lowerCAmelCase__=False			,						lowerCAmelCase__=False			,						lowerCAmelCase__=None			,						lowerCAmelCase__=None			,						**lowerCAmelCase__			,						):





										"""simple docstring"""




										super().__init__(pad_token_id=lowerCAmelCase__			,						mask_token_id=lowerCAmelCase__			,						**lowerCAmelCase__       )

										SCREAMING_SNAKE_CASE_		:     int					=					vocab_size
										SCREAMING_SNAKE_CASE_		:     int					=					hidden_size
										SCREAMING_SNAKE_CASE_		:     str					=					num_hidden_layers
										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					num_attention_heads
										SCREAMING_SNAKE_CASE_		:     List[Any]					=					intermediate_size
										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					hidden_dropout_prob
										SCREAMING_SNAKE_CASE_		:     List[Any]					=					attention_probs_dropout_prob
										SCREAMING_SNAKE_CASE_		:     List[Any]					=					max_position_embeddings
										SCREAMING_SNAKE_CASE_		:     int					=					initializer_range
										SCREAMING_SNAKE_CASE_		:     Optional[int]					=					layer_norm_eps
										SCREAMING_SNAKE_CASE_		:     Dict					=					position_embedding_type
										SCREAMING_SNAKE_CASE_		:     List[str]					=					use_cache
										SCREAMING_SNAKE_CASE_		:     List[Any]					=					emb_layer_norm_before
										SCREAMING_SNAKE_CASE_		:     List[str]					=					token_dropout
										SCREAMING_SNAKE_CASE_		:     Tuple					=					is_folding_model
										if is_folding_model:
													if esmfold_config is None:
																logger.info('No esmfold_config supplied for folding model, using default values.'       )
																SCREAMING_SNAKE_CASE_		:     List[str]					=					EsmFoldConfig()
													elif isinstance(lowerCAmelCase__			,						lowerCAmelCase__       ):
																SCREAMING_SNAKE_CASE_		:     List[str]					=					EsmFoldConfig(**lowerCAmelCase__       )
													SCREAMING_SNAKE_CASE_		:     List[str]					=					esmfold_config
													if vocab_list is None:
																logger.warning('No vocab_list supplied for folding model, assuming the ESM-2 vocabulary!'       )
																SCREAMING_SNAKE_CASE_		:     Tuple					=					get_default_vocab_list()
													else:
																SCREAMING_SNAKE_CASE_		:     List[Any]					=					vocab_list
										else:
													SCREAMING_SNAKE_CASE_		:     List[str]					=					None
													SCREAMING_SNAKE_CASE_		:     Optional[int]					=					None
										if self.esmfold_config is not None and getattr(self.esmfold_config			,						'use_esm_attn_map'			,						lowerCAmelCase__       ):
													raise ValueError('The HuggingFace port of ESMFold does not support use_esm_attn_map at this time!'       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     List[str]					=					super().to_dict()
										if isinstance(self.esmfold_config			,						lowerCAmelCase__       ):
													SCREAMING_SNAKE_CASE_		:     Optional[int]					=					self.esmfold_config.to_dict()
										return output







@dataclass
class 	__lowercase    :





							"""simple docstring"""







							_UpperCAmelCase      =						None
							_UpperCAmelCase      =						True
							_UpperCAmelCase      =						False
							_UpperCAmelCase      =						False
							_UpperCAmelCase      =						False
							_UpperCAmelCase      =						0

							_UpperCAmelCase      =						True
							_UpperCAmelCase      =						False

							_UpperCAmelCase      =						1_2_8
							_UpperCAmelCase      =						None






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										if self.trunk is None:
													SCREAMING_SNAKE_CASE_		:     Any					=					TrunkConfig()
										elif isinstance(self.trunk			,						lowerCAmelCase__       ):
													SCREAMING_SNAKE_CASE_		:     Optional[int]					=					TrunkConfig(**self.trunk       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     int					=					asdict(self       )
										SCREAMING_SNAKE_CASE_		:     Optional[int]					=					self.trunk.to_dict()
										return output







@dataclass
class 	__lowercase    :





							"""simple docstring"""







							_UpperCAmelCase      =						4_8
							_UpperCAmelCase      =						1_0_2_4
							_UpperCAmelCase      =						1_2_8
							_UpperCAmelCase      =						3_2
							_UpperCAmelCase      =						3_2
							_UpperCAmelCase      =						3_2
							_UpperCAmelCase      =						0
							_UpperCAmelCase      =						0
							_UpperCAmelCase      =						False
							_UpperCAmelCase      =						4
							_UpperCAmelCase      =						1_2_8
							_UpperCAmelCase      =						None






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										if self.structure_module is None:
													SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					StructureModuleConfig()
										elif isinstance(self.structure_module			,						lowerCAmelCase__       ):
													SCREAMING_SNAKE_CASE_		:     List[str]					=					StructureModuleConfig(**self.structure_module       )

										if self.max_recycles <= 0:
													raise ValueError(F'''`max_recycles` should be positive, got {self.max_recycles}.'''       )
										if self.sequence_state_dim % self.sequence_state_dim != 0:
													raise ValueError(
													    '`sequence_state_dim` should be a round multiple of `sequence_state_dim`, got'
													    F''' {self.sequence_state_dim} and {self.sequence_state_dim}.'''       )
										if self.pairwise_state_dim % self.pairwise_state_dim != 0:
													raise ValueError(
													    '`pairwise_state_dim` should be a round multiple of `pairwise_state_dim`, got'
													    F''' {self.pairwise_state_dim} and {self.pairwise_state_dim}.'''       )

										SCREAMING_SNAKE_CASE_		:     List[str]					=					self.sequence_state_dim // self.sequence_head_width
										SCREAMING_SNAKE_CASE_		:     List[str]					=					self.pairwise_state_dim // self.pairwise_head_width

										if self.sequence_state_dim != sequence_num_heads * self.sequence_head_width:
													raise ValueError(
													    '`sequence_state_dim` should be equal to `sequence_num_heads * sequence_head_width, got'
													    F''' {self.sequence_state_dim} != {sequence_num_heads} * {self.sequence_head_width}.'''       )
										if self.pairwise_state_dim != pairwise_num_heads * self.pairwise_head_width:
													raise ValueError(
													    '`pairwise_state_dim` should be equal to `pairwise_num_heads * pairwise_head_width, got'
													    F''' {self.pairwise_state_dim} != {pairwise_num_heads} * {self.pairwise_head_width}.'''       )
										if self.pairwise_state_dim % 2 != 0:
													raise ValueError(F'''`pairwise_state_dim` should be even, got {self.pairwise_state_dim}.'''       )

										if self.dropout >= 0.4:
													raise ValueError(F'''`dropout` should not be greater than 0.4, got {self.dropout}.'''       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Dict					=					asdict(self       )
										SCREAMING_SNAKE_CASE_		:     Dict					=					self.structure_module.to_dict()
										return output







@dataclass
class 	__lowercase    :





							"""simple docstring"""







							_UpperCAmelCase      =						3_8_4
							_UpperCAmelCase      =						1_2_8
							_UpperCAmelCase      =						1_6
							_UpperCAmelCase      =						1_2_8
							_UpperCAmelCase      =						1_2
							_UpperCAmelCase      =						4
							_UpperCAmelCase      =						8
							_UpperCAmelCase      =						0.1
							_UpperCAmelCase      =						8
							_UpperCAmelCase      =						1
							_UpperCAmelCase      =						2
							_UpperCAmelCase      =						7
							_UpperCAmelCase      =						1_0
							_UpperCAmelCase      =						1E-8
							_UpperCAmelCase      =						1E5






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										return asdict(self       )







def 						a__			(				):
			return (
			    "<cls>",
			    "<pad>",
			    "<eos>",
			    "<unk>",
			    "L",
			    "A",
			    "G",
			    "V",
			    "S",
			    "E",
			    "R",
			    "T",
			    "I",
			    "D",
			    "P",
			    "K",
			    "Q",
			    "N",
			    "F",
			    "Y",
			    "M",
			    "H",
			    "W",
			    "C",
			    "X",
			    "B",
			    "U",
			    "Z",
			    "O",
			    ".",
			    "-",
			    "<null_1>",
			    "<mask>",
			)













from manim import *







class 		_A						( UpperCAmelCase_ ):

  def    a					(    self						:	Dict			):
     """simple docstring"""

     __UpperCamelCase	:			Tuple					=					Rectangle(height=0.5   ,				width=0.5			)
     __UpperCamelCase	:			Tuple					=					Rectangle(height=0.46   ,				width=0.46			).set_stroke(width=0			)

     __UpperCamelCase	:			Optional[Any]					=					Rectangle(height=0.25   ,				width=0.25			)

     __UpperCamelCase	:			Any					=					[mem.copy() for i in range(6			)]
     __UpperCamelCase	:			str					=					[mem.copy() for i in range(6			)]
     __UpperCamelCase	:			int					=					VGroup(*lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0			)
     __UpperCamelCase	:			Optional[int]					=					VGroup(*lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0			)
     __UpperCamelCase	:			Dict					=					VGroup(lowerCamelCase__   ,				lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0			)
     __UpperCamelCase	:			List[Any]					=					Text("""CPU"""   ,				font_size=24			)
     __UpperCamelCase	:			Any					=					Group(lowerCamelCase__   ,				lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0.5   ,				aligned_edge=lowerCamelCase__			)
     cpu.move_to([-2.5, -0.5, 0]			)
     self.add(lowerCamelCase__			)

     __UpperCamelCase	:			Tuple					=					[mem.copy() for i in range(4			)]
     __UpperCamelCase	:			Optional[Any]					=					VGroup(*lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0			)
     __UpperCamelCase	:			Optional[Any]					=					Text("""GPU"""   ,				font_size=24			)
     __UpperCamelCase	:			Union[str, Any]					=					Group(lowerCamelCase__   ,				lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0.5   ,				aligned_edge=lowerCamelCase__			)
     gpu.move_to([-1, -1, 0]			)
     self.add(lowerCamelCase__			)

     __UpperCamelCase	:			Dict					=					[mem.copy() for i in range(6			)]
     __UpperCamelCase	:			Tuple					=					VGroup(*lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0			)

     __UpperCamelCase	:			Tuple					=					Text("""Model"""   ,				font_size=24			)
     __UpperCamelCase	:			List[str]					=					Group(lowerCamelCase__   ,				lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0.5   ,				aligned_edge=lowerCamelCase__			)
     model.move_to([3, -1.0, 0]			)
     self.add(lowerCamelCase__			)

     __UpperCamelCase	:			Tuple					=					[]
     __UpperCamelCase	:			str					=					[]

     for i, rect in enumerate(lowerCamelCase__			):
        __UpperCamelCase	:			Optional[Any]					=					fill.copy().set_fill(lowerCamelCase__   ,				opacity=0.8			)
        target.move_to(lowerCamelCase__			)
        model_arr.append(lowerCamelCase__			)

        __UpperCamelCase	:			Tuple					=					Rectangle(height=0.46   ,				width=0.46			).set_stroke(width=0.0			).set_fill(lowerCamelCase__   ,				opacity=0.8			)
        cpu_target.move_to(cpu_left_col_base[i]			)
        model_cpu_arr.append(lowerCamelCase__			)

     self.add(*lowerCamelCase__   ,				*lowerCamelCase__			)

     __UpperCamelCase	:			Union[str, Any]					=					[meta_mem.copy() for i in range(6			)]
     __UpperCamelCase	:			Any					=					[meta_mem.copy() for i in range(6			)]
     __UpperCamelCase	:			int					=					VGroup(*lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0			)
     __UpperCamelCase	:			Union[str, Any]					=					VGroup(*lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0			)
     __UpperCamelCase	:			str					=					VGroup(lowerCamelCase__   ,				lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0			)
     __UpperCamelCase	:			Optional[Any]					=					Text("""Disk"""   ,				font_size=24			)
     __UpperCamelCase	:			Optional[Any]					=					Group(lowerCamelCase__   ,				lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0.5   ,				aligned_edge=lowerCamelCase__			)
     disk.move_to([-4, -1.25, 0]			)
     self.add(lowerCamelCase__   ,				lowerCamelCase__			)

     __UpperCamelCase	:			Union[str, Any]					=					Square(side_length=2.2			)
     key.move_to([-5, 2, 0]			)

     __UpperCamelCase	:			Optional[Any]					=					MarkupText(
         f'<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'   ,				font_size=18   ,				)

     key_text.move_to([-5, 2.4, 0]			)

     self.add(lowerCamelCase__   ,				lowerCamelCase__			)

     __UpperCamelCase	:			Optional[int]					=					MarkupText(
         f'<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'   ,				font_size=18   ,				)

     blue_text.next_to(lowerCamelCase__   ,				DOWN * 2.4   ,				aligned_edge=key_text.get_left()			)
     self.add(lowerCamelCase__			)

     __UpperCamelCase	:			Any					=					MarkupText(
         f'Now watch as an input is passed through the model\nand how the memory is utilized and handled.'   ,				font_size=24   ,				)
     step_a.move_to([2, 2, 0]			)

     self.play(Write(lowerCamelCase__			)			)

     __UpperCamelCase	:			List[str]					=					Square(0.3			)
     input.set_fill(lowerCamelCase__   ,				opacity=1.0			)
     input.set_stroke(width=0.0			)
     input.next_to(model_base[0]   ,				lowerCamelCase__   ,				buff=0.5			)

     self.play(Write(lowerCamelCase__			)			)

     input.generate_target()
     input.target.next_to(model_arr[0]   ,				direction=lowerCamelCase__   ,				buff=0.02			)
     self.play(MoveToTarget(lowerCamelCase__			)			)

     self.play(FadeOut(lowerCamelCase__			)			)

     __UpperCamelCase	:			Dict					=					Arrow(start=lowerCamelCase__   ,				end=lowerCamelCase__   ,				color=lowerCamelCase__   ,				buff=0.5			)
     a.next_to(model_arr[0].get_left()   ,				lowerCamelCase__   ,				buff=0.2			)

     model_cpu_arr[0].generate_target()
     model_cpu_arr[0].target.move_to(gpu_rect[0]			)

     __UpperCamelCase	:			int					=					MarkupText(
         f'As the input reaches a layer, the hook triggers\nand weights are moved from the CPU\nto the GPU and back.'   ,				font_size=24   ,				)
     step_a.move_to([2, 2, 0]			)

     self.play(Write(lowerCamelCase__   ,				run_time=3			)			)

     __UpperCamelCase	:			Dict					=					{"""run_time""": 1, """fade_in""": True, """fade_out""": True, """buff""": 0.02}

     self.play(
         Write(lowerCamelCase__			)   ,				Circumscribe(model_arr[0]   ,				color=lowerCamelCase__   ,				**lowerCamelCase__			)   ,				Circumscribe(model_cpu_arr[0]   ,				color=lowerCamelCase__   ,				**lowerCamelCase__			)   ,				Circumscribe(gpu_rect[0]   ,				color=lowerCamelCase__   ,				**lowerCamelCase__			)   ,				)
     self.play(MoveToTarget(model_cpu_arr[0]			)			)

     __UpperCamelCase	:			int					=					a.copy()
     for i in range(6			):
        a_c.next_to(model_arr[i].get_right() + 0.02   ,				lowerCamelCase__   ,				buff=0.2			)

        input.generate_target()
        input.target.move_to(model_arr[i].get_right() + 0.02			)

        __UpperCamelCase	:			Tuple					=					AnimationGroup(
            FadeOut(lowerCamelCase__   ,				run_time=0.5			)   ,				MoveToTarget(lowerCamelCase__   ,				run_time=0.5			)   ,				FadeIn(lowerCamelCase__   ,				run_time=0.5			)   ,				lag_ratio=0.2			)

        self.play(lowerCamelCase__			)

        model_cpu_arr[i].generate_target()
        model_cpu_arr[i].target.move_to(cpu_left_col_base[i]			)

        if i < 5:
           model_cpu_arr[i + 1].generate_target()
           model_cpu_arr[i + 1].target.move_to(gpu_rect[0]			)
           if i >= 1:
              __UpperCamelCase	:			Any					=					0.7

           self.play(
               Circumscribe(model_arr[i]   ,				**lowerCamelCase__			)   ,				Circumscribe(cpu_left_col_base[i]   ,				**lowerCamelCase__			)   ,				Circumscribe(cpu_left_col_base[i + 1]   ,				color=lowerCamelCase__   ,				**lowerCamelCase__			)   ,				Circumscribe(gpu_rect[0]   ,				color=lowerCamelCase__   ,				**lowerCamelCase__			)   ,				Circumscribe(model_arr[i + 1]   ,				color=lowerCamelCase__   ,				**lowerCamelCase__			)   ,				)
           if i < 1:
              self.play(
                  MoveToTarget(model_cpu_arr[i]			)   ,				MoveToTarget(model_cpu_arr[i + 1]			)   ,				)
           else:
              self.play(
                  MoveToTarget(model_cpu_arr[i]   ,				run_time=0.7			)   ,				MoveToTarget(model_cpu_arr[i + 1]   ,				run_time=0.7			)   ,				)
        else:
           model_cpu_arr[i].generate_target()
           model_cpu_arr[i].target.move_to(cpu_left_col_base[-1]			)
           input.generate_target()
           input.target.next_to(model_arr[-1].get_right()   ,				RIGHT + 0.02   ,				buff=0.2			)

           self.play(
               Circumscribe(model_arr[-1]   ,				color=lowerCamelCase__   ,				**lowerCamelCase__			)   ,				Circumscribe(cpu_left_col_base[-1]   ,				color=lowerCamelCase__   ,				**lowerCamelCase__			)   ,				Circumscribe(gpu_rect[0]   ,				color=lowerCamelCase__   ,				**lowerCamelCase__			)   ,				)

           self.play(MoveToTarget(model_cpu_arr[i]			)			)

        __UpperCamelCase	:			str					=					a_c
        __UpperCamelCase	:			List[Any]					=					a_c.copy()

     input.generate_target()
     input.target.next_to(model_base[-1]   ,				RIGHT + 0.02   ,				buff=0.5			)
     self.play(
         FadeOut(lowerCamelCase__			)   ,				FadeOut(lowerCamelCase__   ,				run_time=0.5			)   ,				)

     __UpperCamelCase	:			List[str]					=					MarkupText(f'Inference on a model too large for GPU memory\nis successfully completed.'   ,				font_size=24			)
     step_a.move_to([2, 2, 0]			)

     self.play(Write(lowerCamelCase__   ,				run_time=3			)   ,				MoveToTarget(lowerCamelCase__			)			)

     self.wait()



