


'''simple docstring'''

from manim import *





class 						__SCREAMING_SNAKE_CASE					(      lowerCamelCase__ ):






     def  a_					(			self		)       ->    int:
          _a						=		Rectangle(height=0.5     ,							width=0.5		)
          _a						=		Rectangle(height=0.25     ,							width=0.25		)
          _a						=		Rectangle(height=0.46     ,							width=0.46		).set_stroke(width=0		)

          _a						=		[mem.copy() for i in range(6		)]
          _a						=		[mem.copy() for i in range(6		)]
          _a						=		VGroup(*__UpperCamelCase		).arrange(__UpperCamelCase     ,							buff=0		)
          _a						=		VGroup(*__UpperCamelCase		).arrange(__UpperCamelCase     ,							buff=0		)
          _a						=		VGroup(__UpperCamelCase     ,							__UpperCamelCase		).arrange(__UpperCamelCase     ,							buff=0		)
          _a						=		Text("CPU"     ,							font_size=24		)
          _a						=		Group(__UpperCamelCase     ,							__UpperCamelCase		).arrange(__UpperCamelCase     ,							buff=0.5     ,							aligned_edge=__UpperCamelCase		)
          cpu.move_to([-2.5, -0.5, 0]		)
          self.add(__UpperCamelCase		)

          _a						=		[mem.copy() for i in range(4		)]
          _a						=		VGroup(*__UpperCamelCase		).arrange(__UpperCamelCase     ,							buff=0		)
          _a						=		Text("GPU"     ,							font_size=24		)
          _a						=		Group(__UpperCamelCase     ,							__UpperCamelCase		).arrange(__UpperCamelCase     ,							buff=0.5     ,							aligned_edge=__UpperCamelCase		)
          gpu.move_to([-1, -1, 0]		)
          self.add(__UpperCamelCase		)

          _a						=		[mem.copy() for i in range(6		)]
          _a						=		VGroup(*__UpperCamelCase		).arrange(__UpperCamelCase     ,							buff=0		)

          _a						=		Text("Model"     ,							font_size=24		)
          _a						=		Group(__UpperCamelCase     ,							__UpperCamelCase		).arrange(__UpperCamelCase     ,							buff=0.5     ,							aligned_edge=__UpperCamelCase		)
          model.move_to([3, -1.0, 0]		)
          self.add(__UpperCamelCase		)

          _a						=		[]
          _a						=		[]
          _a						=		[]

          for i, rect in enumerate(__UpperCamelCase		):
               rect.set_stroke(__UpperCamelCase		)

               _a						=		Rectangle(height=0.46 / 4     ,							width=0.46 / 3		).set_stroke(width=0.0		).set_fill(__UpperCamelCase     ,							opacity=0.7		)

               if i == 0:
                    cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT		)     ,							buff=0.02     ,							direction=__UpperCamelCase		)
                    cpu_target.set_x(cpu_target.get_x() + 0.1		)
               elif i == 3:
                    cpu_target.next_to(model_cpu_arr[0]     ,							direction=__UpperCamelCase     ,							buff=0.0		)
               else:
                    cpu_target.next_to(model_cpu_arr[i - 1]     ,							direction=__UpperCamelCase     ,							buff=0.0		)
               self.add(__UpperCamelCase		)
               model_cpu_arr.append(__UpperCamelCase		)

          self.add(*__UpperCamelCase     ,							*__UpperCamelCase     ,							*__UpperCamelCase		)

          _a						=		[mem.copy() for i in range(6		)]
          _a						=		VGroup(*__UpperCamelCase		).arrange(__UpperCamelCase     ,							buff=0		)

          _a						=		Text("Loaded Checkpoint"     ,							font_size=24		)
          _a						=		Group(__UpperCamelCase     ,							__UpperCamelCase		).arrange(__UpperCamelCase     ,							buff=0.5     ,							aligned_edge=__UpperCamelCase		)
          checkpoint.move_to([3, 0.5, 0]		)

          self.add(__UpperCamelCase		)

          _a						=		[]
          _a						=		[]

          for i, rect in enumerate(__UpperCamelCase		):
               _a						=		fill.copy().set_fill(__UpperCamelCase     ,							opacity=0.7		)
               target.move_to(__UpperCamelCase		)
               ckpt_arr.append(__UpperCamelCase		)

               _a						=		target.copy()
               if i < 5:
                    cpu_target.move_to(cpu_left_col_base[i + 1]		)
               else:
                    cpu_target.move_to(cpu_right_col_base[i - 5]		)
               ckpt_cpu_arr.append(__UpperCamelCase		)
          self.add(*__UpperCamelCase     ,							*__UpperCamelCase		)

          _a						=		Square(side_length=2.2		)
          key.move_to([-5, 2, 0]		)

          _a						=		MarkupText(
              f"<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model"     ,							font_size=18     ,							)

          key_text.move_to([-5, 2.4, 0]		)

          self.add(__UpperCamelCase     ,							__UpperCamelCase		)

          _a						=		MarkupText(
              f"<span fgcolor='{BLUE}'>●</span> Checkpoint"     ,							font_size=18     ,							)

          blue_text.next_to(__UpperCamelCase     ,							DOWN * 2.4     ,							aligned_edge=key_text.get_left()		)
          self.add(__UpperCamelCase		)

          _a						=		MarkupText(
              f"Based on the passed in configuration, weights are stored in\na variety of np.memmaps on disk or to a particular device."     ,							font_size=24     ,							)
          step_a.move_to([2, 2, 0]		)

          _a						=		[meta_mem.copy() for i in range(6		)]
          _a						=		[meta_mem.copy() for i in range(6		)]
          _a						=		VGroup(*__UpperCamelCase		).arrange(__UpperCamelCase     ,							buff=0		)
          _a						=		VGroup(*__UpperCamelCase		).arrange(__UpperCamelCase     ,							buff=0		)
          _a						=		VGroup(__UpperCamelCase     ,							__UpperCamelCase		).arrange(__UpperCamelCase     ,							buff=0		)
          _a						=		Text("Disk"     ,							font_size=24		)
          _a						=		Group(__UpperCamelCase     ,							__UpperCamelCase		).arrange(__UpperCamelCase     ,							buff=0.5     ,							aligned_edge=__UpperCamelCase		)
          disk.move_to([-4.0, -1.25, 0]		)
          self.play(Write(__UpperCamelCase     ,							run_time=3		)     ,							Write(__UpperCamelCase     ,							run_time=1		)     ,							Create(__UpperCamelCase     ,							run_time=1		)		)

          _a						=		[]
          for i, rect in enumerate(__UpperCamelCase		):
               _a						=		rect.copy()
               target.generate_target()
               target.target.move_to(disk_left_col_base[i]		).scale(0.5		)
               animations.append(MoveToTarget(__UpperCamelCase     ,							run_time=1.5		)		)
          self.play(*__UpperCamelCase		)

          self.play(FadeOut(__UpperCamelCase		)		)

          _a						=		MarkupText(f"Then, the checkpoint is removed from memory\nthrough garbage collection."     ,							font_size=24		)
          step_a.move_to([2, 2, 0]		)

          self.play(Write(__UpperCamelCase     ,							run_time=3		)		)

          self.play(
              FadeOut(__UpperCamelCase     ,							__UpperCamelCase     ,							*__UpperCamelCase     ,							*__UpperCamelCase		)     ,							)

          self.wait()





'''simple docstring'''

import re

from flax.core.frozen_dict import freeze
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.experimental import PartitionSpec as P


# Sentinels
lowercase__				     =		object()

# For specifying empty leaf dict `{}`
lowercase__				     =		object()






def __UpperCamelCase   (			__lowerCamelCase		: Tuple      ,  __lowerCamelCase		: Tuple				)      ->     Union[str, Any]:







     '''simple docstring'''






     _a						=		tuple((re.compile(x + "$"				) for x in qs)				)
     for i in range(len(__lowerCamelCase				) - len(__lowerCamelCase				) + 1				):
          _a						=		[x.match(__lowerCamelCase				) for x, y in zip(__lowerCamelCase      ,  ks[i:]				)]
          if matches and all(__lowerCamelCase				):
               return True
     return False






def __UpperCamelCase   (			__lowerCamelCase		: int				)      ->     Union[str, Any]:







     '''simple docstring'''






     def replace(__lowerCamelCase		: Tuple      ,  __lowerCamelCase		: Any				):
          for rule, replacement in rules:
               if _match(__lowerCamelCase      ,  __lowerCamelCase				):
                    return replacement
          return val

     return replace






def __UpperCamelCase   (			)      ->     Tuple:







     '''simple docstring'''






     return [
         # embeddings
         (("transformer", "wpe", "embedding"), P("mp"      ,  __lowerCamelCase				)),
         (("transformer", "wte", "embedding"), P("mp"      ,  __lowerCamelCase				)),
         # atention
         (("attention", "(q_proj|k_proj|v_proj)", "kernel"), P(__lowerCamelCase      ,  "mp"				)),
         (("attention", "out_proj", "kernel"), P("mp"      ,  __lowerCamelCase				)),
         (("attention", "out_proj", "bias"), None),
         # mlp
         (("mlp", "c_fc", "kernel"), P(__lowerCamelCase      ,  "mp"				)),
         (("mlp", "c_fc", "bias"), P("mp"				)),
         (("mlp", "c_proj", "kernel"), P("mp"      ,  __lowerCamelCase				)),
         (("mlp", "c_proj", "bias"), None),
         # layer norms
         ((r"ln_\d+", "bias"), None),
         ((r"\d+", r"ln_\d+", "scale"), None),
         (("ln_f", "bias"), None),
         (("ln_f", "scale"), None),
     ]






def __UpperCamelCase   (			__lowerCamelCase		: Dict				)      ->     Tuple:







     '''simple docstring'''






     _a						=		_get_partition_rules()
     _a						=		_replacement_rules(__lowerCamelCase				)
     _a						=		{k: _unmatched for k in flatten_dict(__lowerCamelCase				)}
     _a						=		{k: replace(__lowerCamelCase      ,  __lowerCamelCase				) for k, v in initd.items()}
     assert _unmatched not in result.values(), "Incomplete partition spec."
     return freeze(unflatten_dict(__lowerCamelCase				)				)

