





from manim import *




class 			__UpperCamelCase    (							A__    ):





     def 		UpperCamelCase(     self					):
        _UpperCAmelCase           =   Rectangle(height=0.5				,							width=0.5					)
        _UpperCAmelCase           =   Rectangle(height=0.46				,							width=0.46					).set_stroke(width=0					)

        _UpperCAmelCase           =   Rectangle(height=0.25				,							width=0.25					)

        _UpperCAmelCase           =   [mem.copy() for i in range(6					)]
        _UpperCAmelCase           =   [mem.copy() for i in range(6					)]
        _UpperCAmelCase           =   VGroup(*_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0					)
        _UpperCAmelCase           =   VGroup(*_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0					)
        _UpperCAmelCase           =   VGroup(_UpperCamelCase				,							_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0					)
        _UpperCAmelCase           =   Text('''CPU'''				,							font_size=24					)
        _UpperCAmelCase           =   Group(_UpperCamelCase				,							_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0.5				,							aligned_edge=_UpperCamelCase					)
        cpu.move_to([-2.5, -0.5, 0]					)
        self.add(_UpperCamelCase					)

        _UpperCAmelCase           =   [mem.copy() for i in range(4					)]
        _UpperCAmelCase           =   VGroup(*_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0					)
        _UpperCAmelCase           =   Text('''GPU'''				,							font_size=24					)
        _UpperCAmelCase           =   Group(_UpperCamelCase				,							_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0.5				,							aligned_edge=_UpperCamelCase					)
        gpu.move_to([-1, -1, 0]					)
        self.add(_UpperCamelCase					)

        _UpperCAmelCase           =   [mem.copy() for i in range(6					)]
        _UpperCAmelCase           =   VGroup(*_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0					)

        _UpperCAmelCase           =   Text('''Model'''				,							font_size=24					)
        _UpperCAmelCase           =   Group(_UpperCamelCase				,							_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0.5				,							aligned_edge=_UpperCamelCase					)
        model.move_to([3, -1.0, 0]					)
        self.add(_UpperCamelCase					)

        _UpperCAmelCase           =   []
        _UpperCAmelCase           =   []

        for i, rect in enumerate(_UpperCamelCase					):
           _UpperCAmelCase           =   fill.copy().set_fill(_UpperCamelCase				,							opacity=0.8					)
           target.move_to(_UpperCamelCase					)
           model_arr.append(_UpperCamelCase					)

           _UpperCAmelCase           =   Rectangle(height=0.46				,							width=0.46					).set_stroke(width=0.0					).set_fill(_UpperCamelCase				,							opacity=0.8					)
           cpu_target.move_to(cpu_left_col_base[i]					)
           model_cpu_arr.append(_UpperCamelCase					)

        self.add(*_UpperCamelCase				,							*_UpperCamelCase					)

        _UpperCAmelCase           =   [meta_mem.copy() for i in range(6					)]
        _UpperCAmelCase           =   [meta_mem.copy() for i in range(6					)]
        _UpperCAmelCase           =   VGroup(*_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0					)
        _UpperCAmelCase           =   VGroup(*_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0					)
        _UpperCAmelCase           =   VGroup(_UpperCamelCase				,							_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0					)
        _UpperCAmelCase           =   Text('''Disk'''				,							font_size=24					)
        _UpperCAmelCase           =   Group(_UpperCamelCase				,							_UpperCamelCase					).arrange(_UpperCamelCase				,							buff=0.5				,							aligned_edge=_UpperCamelCase					)
        disk.move_to([-4, -1.25, 0]					)
        self.add(_UpperCamelCase				,							_UpperCamelCase					)

        _UpperCAmelCase           =   Square(side_length=2.2					)
        key.move_to([-5, 2, 0]					)

        _UpperCAmelCase           =   MarkupText(
            f'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''				,							font_size=18				,							)

        key_text.move_to([-5, 2.4, 0]					)

        self.add(_UpperCamelCase				,							_UpperCamelCase					)

        _UpperCAmelCase           =   MarkupText(
            f'''<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'''				,							font_size=18				,							)

        blue_text.next_to(_UpperCamelCase				,							DOWN * 2.4				,							aligned_edge=key_text.get_left()					)
        self.add(_UpperCamelCase					)

        _UpperCAmelCase           =   MarkupText(
            f'''Now watch as an input is passed through the model\nand how the memory is utilized and handled.'''				,							font_size=24				,							)
        step_a.move_to([2, 2, 0]					)

        self.play(Write(_UpperCamelCase					)					)

        _UpperCAmelCase           =   Square(0.3					)
        input.set_fill(_UpperCamelCase				,							opacity=1.0					)
        input.set_stroke(width=0.0					)
        input.next_to(model_base[0]				,							_UpperCamelCase				,							buff=0.5					)

        self.play(Write(_UpperCamelCase					)					)

        input.generate_target()
        input.target.next_to(model_arr[0]				,							direction=_UpperCamelCase				,							buff=0.02					)
        self.play(MoveToTarget(_UpperCamelCase					)					)

        self.play(FadeOut(_UpperCamelCase					)					)

        _UpperCAmelCase           =   Arrow(start=_UpperCamelCase				,							end=_UpperCamelCase				,							color=_UpperCamelCase				,							buff=0.5					)
        a.next_to(model_arr[0].get_left()				,							_UpperCamelCase				,							buff=0.2					)

        model_cpu_arr[0].generate_target()
        model_cpu_arr[0].target.move_to(gpu_rect[0]					)

        _UpperCAmelCase           =   MarkupText(
            f'''As the input reaches a layer, the hook triggers\nand weights are moved from the CPU\nto the GPU and back.'''				,							font_size=24				,							)
        step_a.move_to([2, 2, 0]					)

        self.play(Write(_UpperCamelCase				,							run_time=3					)					)

        _UpperCAmelCase           =   {'''run_time''': 1, '''fade_in''': True, '''fade_out''': True, '''buff''': 0.02}

        self.play(
            Write(_UpperCamelCase					)				,							Circumscribe(model_arr[0]				,							color=_UpperCamelCase				,							**_UpperCamelCase					)				,							Circumscribe(model_cpu_arr[0]				,							color=_UpperCamelCase				,							**_UpperCamelCase					)				,							Circumscribe(gpu_rect[0]				,							color=_UpperCamelCase				,							**_UpperCamelCase					)				,							)
        self.play(MoveToTarget(model_cpu_arr[0]					)					)

        _UpperCAmelCase           =   a.copy()
        for i in range(6					):
           a_c.next_to(model_arr[i].get_right() + 0.02				,							_UpperCamelCase				,							buff=0.2					)

           input.generate_target()
           input.target.move_to(model_arr[i].get_right() + 0.02					)

           _UpperCAmelCase           =   AnimationGroup(
               FadeOut(_UpperCamelCase				,							run_time=0.5					)				,							MoveToTarget(_UpperCamelCase				,							run_time=0.5					)				,							FadeIn(_UpperCamelCase				,							run_time=0.5					)				,							lag_ratio=0.2					)

           self.play(_UpperCamelCase					)

           model_cpu_arr[i].generate_target()
           model_cpu_arr[i].target.move_to(cpu_left_col_base[i]					)

           if i < 5:
              model_cpu_arr[i + 1].generate_target()
              model_cpu_arr[i + 1].target.move_to(gpu_rect[0]					)
              if i >= 1:
                 _UpperCAmelCase           =   0.7

              self.play(
                  Circumscribe(model_arr[i]				,							**_UpperCamelCase					)				,							Circumscribe(cpu_left_col_base[i]				,							**_UpperCamelCase					)				,							Circumscribe(cpu_left_col_base[i + 1]				,							color=_UpperCamelCase				,							**_UpperCamelCase					)				,							Circumscribe(gpu_rect[0]				,							color=_UpperCamelCase				,							**_UpperCamelCase					)				,							Circumscribe(model_arr[i + 1]				,							color=_UpperCamelCase				,							**_UpperCamelCase					)				,							)
              if i < 1:
                 self.play(
                     MoveToTarget(model_cpu_arr[i]					)				,							MoveToTarget(model_cpu_arr[i + 1]					)				,							)
              else:
                 self.play(
                     MoveToTarget(model_cpu_arr[i]				,							run_time=0.7					)				,							MoveToTarget(model_cpu_arr[i + 1]				,							run_time=0.7					)				,							)
           else:
              model_cpu_arr[i].generate_target()
              model_cpu_arr[i].target.move_to(cpu_left_col_base[-1]					)
              input.generate_target()
              input.target.next_to(model_arr[-1].get_right()				,							RIGHT + 0.02				,							buff=0.2					)

              self.play(
                  Circumscribe(model_arr[-1]				,							color=_UpperCamelCase				,							**_UpperCamelCase					)				,							Circumscribe(cpu_left_col_base[-1]				,							color=_UpperCamelCase				,							**_UpperCamelCase					)				,							Circumscribe(gpu_rect[0]				,							color=_UpperCamelCase				,							**_UpperCamelCase					)				,							)

              self.play(MoveToTarget(model_cpu_arr[i]					)					)

           _UpperCAmelCase           =   a_c
           _UpperCAmelCase           =   a_c.copy()

        input.generate_target()
        input.target.next_to(model_base[-1]				,							RIGHT + 0.02				,							buff=0.5					)
        self.play(
            FadeOut(_UpperCamelCase					)				,							FadeOut(_UpperCamelCase				,							run_time=0.5					)				,							)

        _UpperCAmelCase           =   MarkupText(f'''Inference on a model too large for GPU memory\nis successfully completed.'''				,							font_size=24					)
        step_a.move_to([2, 2, 0]					)

        self.play(Write(_UpperCamelCase				,							run_time=3					)				,							MoveToTarget(_UpperCamelCase					)					)

        self.wait()



'''simple docstring'''
# We ignore warnings about stepping the scheduler since we step it ourselves during gradient accumulation

import warnings

from .state import AcceleratorState, GradientState


warnings.filterwarnings('ignore', category=UserWarning, module='torch.optim.lr_scheduler')

class     _lowerCamelCase       :




 '''simple docstring'''


 def __init__(   self   ,   __lowercase   ,   __lowercase   ,   __lowercase = True   ,   __lowercase = False    ):

  """simple docstring"""







  __A	:		List[str]	  =      scheduler
  __A	:		Dict	  =      optimizers if isinstance(__lowercase   ,   (list, tuple)    ) else [optimizers]
  __A	:		List[Any]	  =      split_batches
  __A	:		Any	  =      step_with_optimizer
  __A	:		List[Any]	  =      GradientState()


 def 			snake_case__ (   self   ,   *__lowercase   ,   **__lowercase    ):

  """simple docstring"""







  if not self.step_with_optimizer:
   # No link between scheduler and optimizer -> just step
   self.scheduler.step(*__lowercase   ,   **__lowercase    )
   return

  # Otherwise, first make sure the optimizer was stepped.
  if not self.gradient_state.sync_gradients:
   if self.gradient_state.adjust_scheduler:
    self.scheduler._step_count += 1
   return

  for opt in self.optimizers:
   if opt.step_was_skipped:
    return
  if self.split_batches:
   # Split batches -> the training dataloader batch size is not changed so one step per training step
   self.scheduler.step(*__lowercase   ,   **__lowercase    )
  else:
   # Otherwise the training dataloader batch size was multiplied by `num_processes`, so we need to do
   # num_processes steps per training step
   __A	:		Tuple	  =      AcceleratorState().num_processes
   for _ in range(__lowercase    ):
    # Special case when using OneCycle and `drop_last` was not used
    if hasattr(self.scheduler   ,   'total_steps'    ):
     if self.scheduler._step_count <= self.scheduler.total_steps:
      self.scheduler.step(*__lowercase   ,   **__lowercase    )
    else:
     self.scheduler.step(*__lowercase   ,   **__lowercase    )


 def 			snake_case__ (   self    ):

  """simple docstring"""







  return self.scheduler.get_last_lr()


 def 			snake_case__ (   self    ):

  """simple docstring"""







  return self.scheduler.state_dict()


 def 			snake_case__ (   self   ,   __lowercase    ):

  """simple docstring"""







  self.scheduler.load_state_dict(__lowercase    )


 def 			snake_case__ (   self    ):

  """simple docstring"""







  return self.scheduler.get_lr()







 def 			snake_case__ (   self   ,   *__lowercase   ,   **__lowercase    ):

  """simple docstring"""







  return self.scheduler.print_lr(*__lowercase   ,   **__lowercase    )






