


from dataclasses import dataclass
from typing import Optional, Tuple

import torch
from torch import nn
from transformers import RobertaPreTrainedModel, XLMRobertaConfig, XLMRobertaModel
from transformers.utils import ModelOutput



@dataclass
class        a					(			_UpperCAmelCase   ):
					UpperCAmelCase__ :  Optional[torch.FloatTensor]						      =	None
					UpperCAmelCase__ :  torch.FloatTensor						      =	None
					UpperCAmelCase__ :  Optional[Tuple[torch.FloatTensor]]						      =	None
					UpperCAmelCase__ :  Optional[Tuple[torch.FloatTensor]]						      =	None



class        a					(			_UpperCAmelCase   ):





					def __init__(    self					:				Optional[int]    ,	SCREAMING_SNAKE_CASE_					:				Any=1    ,	SCREAMING_SNAKE_CASE_					:				Tuple=0    ,	SCREAMING_SNAKE_CASE_					:				List[str]=2    ,	SCREAMING_SNAKE_CASE_					:				Tuple=512    ,	SCREAMING_SNAKE_CASE_					:				Dict="cls"    ,	SCREAMING_SNAKE_CASE_					:				Optional[int]=False    ,	SCREAMING_SNAKE_CASE_					:				int=True    ,	**SCREAMING_SNAKE_CASE_					:				List[str]    ,	):
								super().__init__(pad_token_id=SCREAMING_SNAKE_CASE_    ,	bos_token_id=SCREAMING_SNAKE_CASE_    ,	eos_token_id=SCREAMING_SNAKE_CASE_    ,	**SCREAMING_SNAKE_CASE_							)
								__lowerCamelCase:				int     =	project_dim
								__lowerCamelCase:				List[Any]     =	pooler_fn
								__lowerCamelCase:				Any     =	learn_encoder
								__lowerCamelCase:				Optional[int]     =	use_attention_mask



class        a					(			_UpperCAmelCase   ):
					UpperCAmelCase__ :  Any						      =	[R"pooler", R"logit_scale"]
					UpperCAmelCase__ :  List[Any]						      =	[R"position_ids", R"predictions.decoder.bias"]
					UpperCAmelCase__ :  List[str]						      =	"roberta"
					UpperCAmelCase__ :  Dict						      =	RobertaSeriesConfig





					def __init__(    self					:				Any    ,	SCREAMING_SNAKE_CASE_					:				str							):
								super().__init__(SCREAMING_SNAKE_CASE_							)
								__lowerCamelCase:				Dict     =	XLMRobertaModel(SCREAMING_SNAKE_CASE_							)
								__lowerCamelCase:				int     =	nn.Linear(config.hidden_size    ,	config.project_dim							)
								__lowerCamelCase:				Optional[Any]     =	getattr(SCREAMING_SNAKE_CASE_    ,	"""has_pre_transformation"""    ,	SCREAMING_SNAKE_CASE_							)
								if self.has_pre_transformation:
											__lowerCamelCase:				Any     =	nn.Linear(config.hidden_size    ,	config.project_dim							)
											__lowerCamelCase:				int     =	nn.LayerNorm(config.hidden_size    ,	eps=config.layer_norm_eps							)
								self.post_init()







					def  SCREAMING_SNAKE_CASE__       (    self					:				List[Any]    ,	SCREAMING_SNAKE_CASE_					:				Optional[torch.Tensor] = None    ,	SCREAMING_SNAKE_CASE_					:				Optional[torch.Tensor] = None    ,	SCREAMING_SNAKE_CASE_					:				Optional[torch.Tensor] = None    ,	SCREAMING_SNAKE_CASE_					:				Optional[torch.Tensor] = None    ,	SCREAMING_SNAKE_CASE_					:				Optional[torch.Tensor] = None    ,	SCREAMING_SNAKE_CASE_					:				Optional[torch.Tensor] = None    ,	SCREAMING_SNAKE_CASE_					:				Optional[torch.Tensor] = None    ,	SCREAMING_SNAKE_CASE_					:				Optional[torch.Tensor] = None    ,	SCREAMING_SNAKE_CASE_					:				Optional[bool] = None    ,	SCREAMING_SNAKE_CASE_					:				Optional[bool] = None    ,	SCREAMING_SNAKE_CASE_					:				Optional[bool] = None    ,	):
								__lowerCamelCase:				Tuple     =	return_dict if return_dict is not None else self.config.use_return_dict

								__lowerCamelCase:				str     =	self.base_model(
								    input_ids=SCREAMING_SNAKE_CASE_    ,	attention_mask=SCREAMING_SNAKE_CASE_    ,	token_type_ids=SCREAMING_SNAKE_CASE_    ,	position_ids=SCREAMING_SNAKE_CASE_    ,	head_mask=SCREAMING_SNAKE_CASE_    ,	inputs_embeds=SCREAMING_SNAKE_CASE_    ,	encoder_hidden_states=SCREAMING_SNAKE_CASE_    ,	encoder_attention_mask=SCREAMING_SNAKE_CASE_    ,	output_attentions=SCREAMING_SNAKE_CASE_    ,	output_hidden_states=True if self.has_pre_transformation else output_hidden_states    ,	return_dict=SCREAMING_SNAKE_CASE_    ,	)

								if self.has_pre_transformation:
											__lowerCamelCase:				Optional[Any]     =	outputs["""hidden_states"""][-2]
											__lowerCamelCase:				List[Any]     =	self.pre_LN(SCREAMING_SNAKE_CASE_							)
											__lowerCamelCase:				Tuple     =	self.transformation_pre(SCREAMING_SNAKE_CASE_							)

											return TransformationModelOutput(
											    projection_state=SCREAMING_SNAKE_CASE_    ,	last_hidden_state=outputs.last_hidden_state    ,	hidden_states=outputs.hidden_states    ,	attentions=outputs.attentions    ,	)
								else:
											__lowerCamelCase:				int     =	self.transformation(outputs.last_hidden_state							)
											return TransformationModelOutput(
											    projection_state=SCREAMING_SNAKE_CASE_    ,	last_hidden_state=outputs.last_hidden_state    ,	hidden_states=outputs.hidden_states    ,	attentions=outputs.attentions    ,	)






from manim import *



class        a					(			_UpperCAmelCase   ):





     def  SCREAMING_SNAKE_CASE__       (    self					:				int							):
        __lowerCamelCase:				int     =	Rectangle(height=0.5    ,	width=0.5							)
        __lowerCamelCase:				List[str]     =	Rectangle(height=0.25    ,	width=0.25							)
        __lowerCamelCase:				Optional[int]     =	Rectangle(height=0.46    ,	width=0.46							).set_stroke(width=0							)

        __lowerCamelCase:				str     =	[mem.copy() for i in range(6							)]
        __lowerCamelCase:				Dict     =	[mem.copy() for i in range(6							)]
        __lowerCamelCase:				Union[str, Any]     =	VGroup(*SCREAMING_SNAKE_CASE_							).arrange(SCREAMING_SNAKE_CASE_    ,	buff=0							)
        __lowerCamelCase:				Tuple     =	VGroup(*SCREAMING_SNAKE_CASE_							).arrange(SCREAMING_SNAKE_CASE_    ,	buff=0							)
        __lowerCamelCase:				Optional[int]     =	VGroup(SCREAMING_SNAKE_CASE_    ,	SCREAMING_SNAKE_CASE_							).arrange(SCREAMING_SNAKE_CASE_    ,	buff=0							)
        __lowerCamelCase:				Dict     =	Text("""CPU"""    ,	font_size=24							)
        __lowerCamelCase:				Any     =	Group(SCREAMING_SNAKE_CASE_    ,	SCREAMING_SNAKE_CASE_							).arrange(SCREAMING_SNAKE_CASE_    ,	buff=0.5    ,	aligned_edge=SCREAMING_SNAKE_CASE_							)
        cpu.move_to([-2.5, -0.5, 0]							)
        self.add(SCREAMING_SNAKE_CASE_							)

        __lowerCamelCase:				Optional[int]     =	[mem.copy() for i in range(4							)]
        __lowerCamelCase:				Optional[int]     =	VGroup(*SCREAMING_SNAKE_CASE_							).arrange(SCREAMING_SNAKE_CASE_    ,	buff=0							)
        __lowerCamelCase:				Dict     =	Text("""GPU"""    ,	font_size=24							)
        __lowerCamelCase:				Dict     =	Group(SCREAMING_SNAKE_CASE_    ,	SCREAMING_SNAKE_CASE_							).arrange(SCREAMING_SNAKE_CASE_    ,	buff=0.5    ,	aligned_edge=SCREAMING_SNAKE_CASE_							)
        gpu.move_to([-1, -1, 0]							)
        self.add(SCREAMING_SNAKE_CASE_							)

        __lowerCamelCase:				Any     =	[mem.copy() for i in range(6							)]
        __lowerCamelCase:				Dict     =	VGroup(*SCREAMING_SNAKE_CASE_							).arrange(SCREAMING_SNAKE_CASE_    ,	buff=0							)

        __lowerCamelCase:				List[Any]     =	Text("""Model"""    ,	font_size=24							)
        __lowerCamelCase:				Any     =	Group(SCREAMING_SNAKE_CASE_    ,	SCREAMING_SNAKE_CASE_							).arrange(SCREAMING_SNAKE_CASE_    ,	buff=0.5    ,	aligned_edge=SCREAMING_SNAKE_CASE_							)
        model.move_to([3, -1.0, 0]							)
        self.add(SCREAMING_SNAKE_CASE_							)

        __lowerCamelCase:				Tuple     =	[]
        __lowerCamelCase:				Any     =	[]
        __lowerCamelCase:				int     =	[]

        for i, rect in enumerate(SCREAMING_SNAKE_CASE_							):
           rect.set_stroke(SCREAMING_SNAKE_CASE_							)

           __lowerCamelCase:				int     =	Rectangle(height=0.46 / 4    ,	width=0.46 / 3							).set_stroke(width=0.0							).set_fill(SCREAMING_SNAKE_CASE_    ,	opacity=0.7							)

           if i == 0:
              cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT							)    ,	buff=0.02    ,	direction=SCREAMING_SNAKE_CASE_							)
              cpu_target.set_x(cpu_target.get_x() + 0.1							)
           elif i == 3:
              cpu_target.next_to(model_cpu_arr[0]    ,	direction=SCREAMING_SNAKE_CASE_    ,	buff=0.0							)
           else:
              cpu_target.next_to(model_cpu_arr[i - 1]    ,	direction=SCREAMING_SNAKE_CASE_    ,	buff=0.0							)
           self.add(SCREAMING_SNAKE_CASE_							)
           model_cpu_arr.append(SCREAMING_SNAKE_CASE_							)

        self.add(*SCREAMING_SNAKE_CASE_    ,	*SCREAMING_SNAKE_CASE_    ,	*SCREAMING_SNAKE_CASE_							)

        __lowerCamelCase:				List[Any]     =	[mem.copy() for i in range(6							)]
        __lowerCamelCase:				Any     =	VGroup(*SCREAMING_SNAKE_CASE_							).arrange(SCREAMING_SNAKE_CASE_    ,	buff=0							)

        __lowerCamelCase:				Tuple     =	Text("""Loaded Checkpoint"""    ,	font_size=24							)
        __lowerCamelCase:				Tuple     =	Group(SCREAMING_SNAKE_CASE_    ,	SCREAMING_SNAKE_CASE_							).arrange(SCREAMING_SNAKE_CASE_    ,	buff=0.5    ,	aligned_edge=SCREAMING_SNAKE_CASE_							)
        checkpoint.move_to([3, 0.5, 0]							)

        self.add(SCREAMING_SNAKE_CASE_							)

        __lowerCamelCase:				List[Any]     =	[]
        __lowerCamelCase:				Optional[int]     =	[]

        for i, rect in enumerate(SCREAMING_SNAKE_CASE_							):
           __lowerCamelCase:				Optional[int]     =	fill.copy().set_fill(SCREAMING_SNAKE_CASE_    ,	opacity=0.7							)
           target.move_to(SCREAMING_SNAKE_CASE_							)
           ckpt_arr.append(SCREAMING_SNAKE_CASE_							)

           __lowerCamelCase:				List[Any]     =	target.copy()
           if i < 5:
              cpu_target.move_to(cpu_left_col_base[i + 1]							)
           else:
              cpu_target.move_to(cpu_right_col_base[i - 5]							)
           ckpt_cpu_arr.append(SCREAMING_SNAKE_CASE_							)
        self.add(*SCREAMING_SNAKE_CASE_    ,	*SCREAMING_SNAKE_CASE_							)

        __lowerCamelCase:				Optional[Any]     =	Square(side_length=2.2							)
        key.move_to([-5, 2, 0]							)

        __lowerCamelCase:				Dict     =	MarkupText(
            F'<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'    ,	font_size=18    ,	)

        key_text.move_to([-5, 2.4, 0]							)

        self.add(SCREAMING_SNAKE_CASE_    ,	SCREAMING_SNAKE_CASE_							)

        __lowerCamelCase:				Optional[int]     =	MarkupText(
            F'<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'    ,	font_size=18    ,	)

        blue_text.next_to(SCREAMING_SNAKE_CASE_    ,	DOWN * 2.4    ,	aligned_edge=key_text.get_left()							)
        self.add(SCREAMING_SNAKE_CASE_							)

        __lowerCamelCase:				Tuple     =	MarkupText(
            F'Based on the passed in configuration, weights are stored in\na variety of np.memmaps on disk or to a particular device.'    ,	font_size=24    ,	)
        step_a.move_to([2, 2, 0]							)

        __lowerCamelCase:				List[Any]     =	[meta_mem.copy() for i in range(6							)]
        __lowerCamelCase:				Optional[int]     =	[meta_mem.copy() for i in range(6							)]
        __lowerCamelCase:				str     =	VGroup(*SCREAMING_SNAKE_CASE_							).arrange(SCREAMING_SNAKE_CASE_    ,	buff=0							)
        __lowerCamelCase:				Optional[int]     =	VGroup(*SCREAMING_SNAKE_CASE_							).arrange(SCREAMING_SNAKE_CASE_    ,	buff=0							)
        __lowerCamelCase:				List[str]     =	VGroup(SCREAMING_SNAKE_CASE_    ,	SCREAMING_SNAKE_CASE_							).arrange(SCREAMING_SNAKE_CASE_    ,	buff=0							)
        __lowerCamelCase:				Dict     =	Text("""Disk"""    ,	font_size=24							)
        __lowerCamelCase:				Any     =	Group(SCREAMING_SNAKE_CASE_    ,	SCREAMING_SNAKE_CASE_							).arrange(SCREAMING_SNAKE_CASE_    ,	buff=0.5    ,	aligned_edge=SCREAMING_SNAKE_CASE_							)
        disk.move_to([-4.0, -1.25, 0]							)
        self.play(Write(SCREAMING_SNAKE_CASE_    ,	run_time=3							)    ,	Write(SCREAMING_SNAKE_CASE_    ,	run_time=1							)    ,	Create(SCREAMING_SNAKE_CASE_    ,	run_time=1							)							)

        __lowerCamelCase:				int     =	[]
        for i, rect in enumerate(SCREAMING_SNAKE_CASE_							):
           __lowerCamelCase:				Optional[int]     =	rect.copy()
           target.generate_target()
           target.target.move_to(disk_left_col_base[i]							).scale(0.5							)
           animations.append(MoveToTarget(SCREAMING_SNAKE_CASE_    ,	run_time=1.5							)							)
        self.play(*SCREAMING_SNAKE_CASE_							)

        self.play(FadeOut(SCREAMING_SNAKE_CASE_							)							)

        __lowerCamelCase:				List[Any]     =	MarkupText(F'Then, the checkpoint is removed from memory\nthrough garbage collection.'    ,	font_size=24							)
        step_a.move_to([2, 2, 0]							)

        self.play(Write(SCREAMING_SNAKE_CASE_    ,	run_time=3							)							)

        self.play(
            FadeOut(SCREAMING_SNAKE_CASE_    ,	SCREAMING_SNAKE_CASE_    ,	*SCREAMING_SNAKE_CASE_    ,	*SCREAMING_SNAKE_CASE_							)    ,	)

        self.wait()


