


from typing import List, Optional, Union

import numpy as np
import tensorflow as tf

from .utils import logging


_SCREAMING_SNAKE_CASE			:       List[Any]      =						logging.get_logger(__name__)







def     _lowercase							(  __lowerCamelCase      :       Union[tf.Tensor, np.ndarray]      )	-> List[int]:





     '''simple docstring'''





     if isinstance(__lowerCAmelCase		,np.ndarray      ):
          return list(tensor.shape      )

     UpperCamelCase__				:	str										=  tf.shape(__lowerCAmelCase      )

     if tensor.shape == tf.TensorShape(__lowerCAmelCase      ):
          return dynamic

     UpperCamelCase__				:	List[str]										=  tensor.shape.as_list()

     return [dynamic[i] if s is None else s for i, s in enumerate(__lowerCAmelCase      )]







def     _lowercase							(  __lowerCamelCase      :       tf.Tensor		,__lowerCamelCase      :       Optional[int] = None		,__lowerCamelCase      :       Optional[str] = None      )	-> tf.Tensor:





     '''simple docstring'''





     return tf.nn.softmax(logits=logits + 1e-9		,axis=__lowerCAmelCase		,name=__lowerCAmelCase      )







def     _lowercase							(  __lowerCamelCase      :       Optional[int]		,__lowerCamelCase      :       Union[str, Any]		,__lowerCamelCase      :       Dict		,__lowerCamelCase      :       List[Any]=1e-5		,__lowerCamelCase      :       str=-1      )	-> Union[str, Any]:





     '''simple docstring'''





     if weight.shape.rank != 1 or bias.shape.rank != 1 or not isinstance(__lowerCAmelCase		,__lowerCAmelCase      ):
          raise NotImplementedError('''Only 1D weight and bias tensors are supported for now, with only a single axis.'''      )

     # Get mean and variance on the axis to be normalized
     UpperCamelCase__				:	Union[str, Any]										=  tf.nn.moments(__lowerCAmelCase		,axes=[axis]		,keepdims=__lowerCAmelCase      )

     if axis != -1:
          # Reshape scale and weight to have the same rank as inputs, but with 1 dimensions
          # on every dimension except axis
          UpperCamelCase__				:	Tuple										=  [1] * inputs.shape.rank
          UpperCamelCase__				:	Tuple										=  shape_list(__lowerCAmelCase      )[axis]
          UpperCamelCase__				:	Union[str, Any]										=  tf.reshape(__lowerCAmelCase		,__lowerCAmelCase      )
          UpperCamelCase__				:	Tuple										=  tf.reshape(__lowerCAmelCase		,__lowerCAmelCase      )

     # Compute layer normalization using the batch_normalization
     # function.
     UpperCamelCase__				:	Optional[int]										=  tf.nn.batch_normalization(
         __lowerCAmelCase		,__lowerCAmelCase		,__lowerCAmelCase		,offset=__lowerCAmelCase		,scale=__lowerCAmelCase		,variance_epsilon=__lowerCAmelCase		,)
     return outputs







def     _lowercase							(  __lowerCamelCase      :       Dict		,__lowerCamelCase      :       Optional[Any]=0		,__lowerCamelCase      :       Tuple=-1      )	-> List[str]:





     '''simple docstring'''





     if end_dim < 0:
          end_dim += input.shape.rank
     if start_dim < 0:
          start_dim += input.shape.rank

     if start_dim == end_dim:
          return input

     UpperCamelCase__				:	List[str]										=  tf.shape(__lowerCAmelCase      )
     UpperCamelCase__				:	Any										=  tf.math.reduce_prod(in_shape[start_dim : end_dim + 1]      )
     UpperCamelCase__				:	Union[str, Any]										=  tf.concat([in_shape[:start_dim], [flattened_dim], in_shape[end_dim + 1 :]]		,axis=0      )
     return tf.reshape(__lowerCAmelCase		,__lowerCAmelCase      )







def     _lowercase							(  __lowerCamelCase      :       tf.Tensor      )	-> tf.Tensor:





     '''simple docstring'''





     if not isinstance(__lowerCAmelCase		,tf.Tensor      ):
          UpperCamelCase__				:	List[str]										=  tf.convert_to_tensor(__lowerCAmelCase      )  # Catches stray NumPy inputs
     if encoder_attention_mask.shape.rank == 3:
          UpperCamelCase__				:	Optional[Any]										=  encoder_attention_mask[:, None, :, :]
     if encoder_attention_mask.shape.rank == 2:
          UpperCamelCase__				:	Optional[Any]										=  encoder_attention_mask[:, None, None, :]
     # T5 has a mask that can compare sequence ids, we can simulate this here with this transposition
     # Cf. https://github.com/tensorflow/mesh/blob/8d2465e9bc93129b913b5ccc6a59aa97abd96ec6/mesh_tensorflow
     # /transformer/transformer_layers.py#L270
     # encoder_extended_attention_mask = (encoder_extended_attention_mask ==
     # encoder_extended_attention_mask.transpose(-1, -2))
     UpperCamelCase__				:	str										=  (
         tf.cast(1		,encoder_attention_mask.dtype      ) - encoder_extended_attention_mask
     ) * encoder_extended_attention_mask.dtype.min

     return encoder_extended_attention_mask







def     _lowercase							(  __lowerCamelCase      :       tf.Tensor		,__lowerCamelCase      :       int		,__lowerCamelCase      :       str = "input_ids"      )	-> None:





     '''simple docstring'''





     tf.debugging.assert_less(
         __lowerCAmelCase		,tf.cast(__lowerCAmelCase		,dtype=tensor.dtype      )		,message=(
             F'The maximum value of {tensor_name} ({tf.math.reduce_max(__lowerCAmelCase      )}) must be smaller than the embedding '
             F'layer\'s input dimension ({embed_dim}). The likely cause is some problem at tokenization time.'
         )		,)







def     _lowercase							(  __lowerCamelCase      :       Any		,__lowerCamelCase      :       Tuple		,__lowerCamelCase      :       List[Any]      )	-> int:





     '''simple docstring'''





     UpperCamelCase__				:	Optional[int]										=  64512
     # Check that no item in `data` is larger than `HDF5_OBJECT_HEADER_LIMIT`
     # because in that case even chunking the array would not make the saving
     # possible.
     UpperCamelCase__				:	Dict										=  [x for x in data if len(__lowerCAmelCase      ) > HDF5_OBJECT_HEADER_LIMIT]

     # Expecting this to never be true.
     if bad_attributes:
          raise RuntimeError(
              '''The following attributes cannot be saved to HDF5 file because '''
              F'they are larger than {HDF5_OBJECT_HEADER_LIMIT} '
              F'bytes: {bad_attributes}'      )

     UpperCamelCase__				:	Optional[Any]										=  np.asarray(__lowerCAmelCase      )

     UpperCamelCase__				:	List[Any]										=  1
     UpperCamelCase__				:	List[str]										=  np.array_split(__lowerCAmelCase		,__lowerCAmelCase      )

     # This will never loop forever thanks to the test above.
     while any(x.nbytes > HDF5_OBJECT_HEADER_LIMIT for x in chunked_data      ):
          num_chunks += 1
          UpperCamelCase__				:	Union[str, Any]										=  np.array_split(__lowerCAmelCase		,__lowerCAmelCase      )

     if num_chunks > 1:
          for chunk_id, chunk_data in enumerate(__lowerCAmelCase      ):
               UpperCamelCase__				:	str										=  chunk_data
     else:
          UpperCamelCase__				:	Optional[int]										=  data







def     _lowercase							(  __lowerCamelCase      :       str		,__lowerCamelCase      :       Optional[int]      )	-> Union[str, Any]:





     '''simple docstring'''





     if name in group.attrs:
          UpperCamelCase__				:	List[Any]										=  [n.decode('''utf8'''      ) if hasattr(__lowerCAmelCase		,'''decode'''      ) else n for n in group.attrs[name]]
     else:
          UpperCamelCase__				:	Dict										=  []
          UpperCamelCase__				:	Any										=  0
          while "%s%d" % (name, chunk_id) in group.attrs:
               data.extend(
                   [n.decode('''utf8'''      ) if hasattr(__lowerCAmelCase		,'''decode'''      ) else n for n in group.attrs['''%s%d''' % (name, chunk_id)]]      )
               chunk_id += 1
     return data







def     _lowercase							(  __lowerCamelCase      :       Tuple      )	-> List[str]:





     '''simple docstring'''





     def _expand_single_ad_tensor(__lowerCamelCase      :       Optional[Any]      ):
          if isinstance(__lowerCAmelCase		,tf.Tensor      ) and t.shape.rank == 1:
               return tf.expand_dims(__lowerCAmelCase		,axis=-1      )
          return t

     return tf.nest.map_structure(_expand_single_ad_tensor		,__lowerCAmelCase      )












from manim import *







class 		_A						( UpperCAmelCase_ ):

  def    a					(    self						:	Dict			):
     """simple docstring"""

     __UpperCamelCase	:			Tuple					=					Rectangle(height=0.5   ,				width=0.5			)
     __UpperCamelCase	:			Tuple					=					Rectangle(height=0.46   ,				width=0.46			).set_stroke(width=0			)

     __UpperCamelCase	:			Optional[Any]					=					Rectangle(height=0.25   ,				width=0.25			)

     __UpperCamelCase	:			Any					=					[mem.copy() for i in range(6			)]
     __UpperCamelCase	:			str					=					[mem.copy() for i in range(6			)]
     __UpperCamelCase	:			int					=					VGroup(*lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0			)
     __UpperCamelCase	:			Optional[int]					=					VGroup(*lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0			)
     __UpperCamelCase	:			Dict					=					VGroup(lowerCamelCase__   ,				lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0			)
     __UpperCamelCase	:			List[Any]					=					Text("""CPU"""   ,				font_size=24			)
     __UpperCamelCase	:			Any					=					Group(lowerCamelCase__   ,				lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0.5   ,				aligned_edge=lowerCamelCase__			)
     cpu.move_to([-2.5, -0.5, 0]			)
     self.add(lowerCamelCase__			)

     __UpperCamelCase	:			Tuple					=					[mem.copy() for i in range(4			)]
     __UpperCamelCase	:			Optional[Any]					=					VGroup(*lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0			)
     __UpperCamelCase	:			Optional[Any]					=					Text("""GPU"""   ,				font_size=24			)
     __UpperCamelCase	:			Union[str, Any]					=					Group(lowerCamelCase__   ,				lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0.5   ,				aligned_edge=lowerCamelCase__			)
     gpu.move_to([-1, -1, 0]			)
     self.add(lowerCamelCase__			)

     __UpperCamelCase	:			Dict					=					[mem.copy() for i in range(6			)]
     __UpperCamelCase	:			Tuple					=					VGroup(*lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0			)

     __UpperCamelCase	:			Tuple					=					Text("""Model"""   ,				font_size=24			)
     __UpperCamelCase	:			List[str]					=					Group(lowerCamelCase__   ,				lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0.5   ,				aligned_edge=lowerCamelCase__			)
     model.move_to([3, -1.0, 0]			)
     self.add(lowerCamelCase__			)

     __UpperCamelCase	:			Tuple					=					[]
     __UpperCamelCase	:			str					=					[]

     for i, rect in enumerate(lowerCamelCase__			):
        __UpperCamelCase	:			Optional[Any]					=					fill.copy().set_fill(lowerCamelCase__   ,				opacity=0.8			)
        target.move_to(lowerCamelCase__			)
        model_arr.append(lowerCamelCase__			)

        __UpperCamelCase	:			Tuple					=					Rectangle(height=0.46   ,				width=0.46			).set_stroke(width=0.0			).set_fill(lowerCamelCase__   ,				opacity=0.8			)
        cpu_target.move_to(cpu_left_col_base[i]			)
        model_cpu_arr.append(lowerCamelCase__			)

     self.add(*lowerCamelCase__   ,				*lowerCamelCase__			)

     __UpperCamelCase	:			Union[str, Any]					=					[meta_mem.copy() for i in range(6			)]
     __UpperCamelCase	:			Any					=					[meta_mem.copy() for i in range(6			)]
     __UpperCamelCase	:			int					=					VGroup(*lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0			)
     __UpperCamelCase	:			Union[str, Any]					=					VGroup(*lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0			)
     __UpperCamelCase	:			str					=					VGroup(lowerCamelCase__   ,				lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0			)
     __UpperCamelCase	:			Optional[Any]					=					Text("""Disk"""   ,				font_size=24			)
     __UpperCamelCase	:			Optional[Any]					=					Group(lowerCamelCase__   ,				lowerCamelCase__			).arrange(lowerCamelCase__   ,				buff=0.5   ,				aligned_edge=lowerCamelCase__			)
     disk.move_to([-4, -1.25, 0]			)
     self.add(lowerCamelCase__   ,				lowerCamelCase__			)

     __UpperCamelCase	:			Union[str, Any]					=					Square(side_length=2.2			)
     key.move_to([-5, 2, 0]			)

     __UpperCamelCase	:			Optional[Any]					=					MarkupText(
         f'<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'   ,				font_size=18   ,				)

     key_text.move_to([-5, 2.4, 0]			)

     self.add(lowerCamelCase__   ,				lowerCamelCase__			)

     __UpperCamelCase	:			Optional[int]					=					MarkupText(
         f'<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'   ,				font_size=18   ,				)

     blue_text.next_to(lowerCamelCase__   ,				DOWN * 2.4   ,				aligned_edge=key_text.get_left()			)
     self.add(lowerCamelCase__			)

     __UpperCamelCase	:			Any					=					MarkupText(
         f'Now watch as an input is passed through the model\nand how the memory is utilized and handled.'   ,				font_size=24   ,				)
     step_a.move_to([2, 2, 0]			)

     self.play(Write(lowerCamelCase__			)			)

     __UpperCamelCase	:			List[str]					=					Square(0.3			)
     input.set_fill(lowerCamelCase__   ,				opacity=1.0			)
     input.set_stroke(width=0.0			)
     input.next_to(model_base[0]   ,				lowerCamelCase__   ,				buff=0.5			)

     self.play(Write(lowerCamelCase__			)			)

     input.generate_target()
     input.target.next_to(model_arr[0]   ,				direction=lowerCamelCase__   ,				buff=0.02			)
     self.play(MoveToTarget(lowerCamelCase__			)			)

     self.play(FadeOut(lowerCamelCase__			)			)

     __UpperCamelCase	:			Dict					=					Arrow(start=lowerCamelCase__   ,				end=lowerCamelCase__   ,				color=lowerCamelCase__   ,				buff=0.5			)
     a.next_to(model_arr[0].get_left()   ,				lowerCamelCase__   ,				buff=0.2			)

     model_cpu_arr[0].generate_target()
     model_cpu_arr[0].target.move_to(gpu_rect[0]			)

     __UpperCamelCase	:			int					=					MarkupText(
         f'As the input reaches a layer, the hook triggers\nand weights are moved from the CPU\nto the GPU and back.'   ,				font_size=24   ,				)
     step_a.move_to([2, 2, 0]			)

     self.play(Write(lowerCamelCase__   ,				run_time=3			)			)

     __UpperCamelCase	:			Dict					=					{"""run_time""": 1, """fade_in""": True, """fade_out""": True, """buff""": 0.02}

     self.play(
         Write(lowerCamelCase__			)   ,				Circumscribe(model_arr[0]   ,				color=lowerCamelCase__   ,				**lowerCamelCase__			)   ,				Circumscribe(model_cpu_arr[0]   ,				color=lowerCamelCase__   ,				**lowerCamelCase__			)   ,				Circumscribe(gpu_rect[0]   ,				color=lowerCamelCase__   ,				**lowerCamelCase__			)   ,				)
     self.play(MoveToTarget(model_cpu_arr[0]			)			)

     __UpperCamelCase	:			int					=					a.copy()
     for i in range(6			):
        a_c.next_to(model_arr[i].get_right() + 0.02   ,				lowerCamelCase__   ,				buff=0.2			)

        input.generate_target()
        input.target.move_to(model_arr[i].get_right() + 0.02			)

        __UpperCamelCase	:			Tuple					=					AnimationGroup(
            FadeOut(lowerCamelCase__   ,				run_time=0.5			)   ,				MoveToTarget(lowerCamelCase__   ,				run_time=0.5			)   ,				FadeIn(lowerCamelCase__   ,				run_time=0.5			)   ,				lag_ratio=0.2			)

        self.play(lowerCamelCase__			)

        model_cpu_arr[i].generate_target()
        model_cpu_arr[i].target.move_to(cpu_left_col_base[i]			)

        if i < 5:
           model_cpu_arr[i + 1].generate_target()
           model_cpu_arr[i + 1].target.move_to(gpu_rect[0]			)
           if i >= 1:
              __UpperCamelCase	:			Any					=					0.7

           self.play(
               Circumscribe(model_arr[i]   ,				**lowerCamelCase__			)   ,				Circumscribe(cpu_left_col_base[i]   ,				**lowerCamelCase__			)   ,				Circumscribe(cpu_left_col_base[i + 1]   ,				color=lowerCamelCase__   ,				**lowerCamelCase__			)   ,				Circumscribe(gpu_rect[0]   ,				color=lowerCamelCase__   ,				**lowerCamelCase__			)   ,				Circumscribe(model_arr[i + 1]   ,				color=lowerCamelCase__   ,				**lowerCamelCase__			)   ,				)
           if i < 1:
              self.play(
                  MoveToTarget(model_cpu_arr[i]			)   ,				MoveToTarget(model_cpu_arr[i + 1]			)   ,				)
           else:
              self.play(
                  MoveToTarget(model_cpu_arr[i]   ,				run_time=0.7			)   ,				MoveToTarget(model_cpu_arr[i + 1]   ,				run_time=0.7			)   ,				)
        else:
           model_cpu_arr[i].generate_target()
           model_cpu_arr[i].target.move_to(cpu_left_col_base[-1]			)
           input.generate_target()
           input.target.next_to(model_arr[-1].get_right()   ,				RIGHT + 0.02   ,				buff=0.2			)

           self.play(
               Circumscribe(model_arr[-1]   ,				color=lowerCamelCase__   ,				**lowerCamelCase__			)   ,				Circumscribe(cpu_left_col_base[-1]   ,				color=lowerCamelCase__   ,				**lowerCamelCase__			)   ,				Circumscribe(gpu_rect[0]   ,				color=lowerCamelCase__   ,				**lowerCamelCase__			)   ,				)

           self.play(MoveToTarget(model_cpu_arr[i]			)			)

        __UpperCamelCase	:			str					=					a_c
        __UpperCamelCase	:			List[Any]					=					a_c.copy()

     input.generate_target()
     input.target.next_to(model_base[-1]   ,				RIGHT + 0.02   ,				buff=0.5			)
     self.play(
         FadeOut(lowerCamelCase__			)   ,				FadeOut(lowerCamelCase__   ,				run_time=0.5			)   ,				)

     __UpperCamelCase	:			List[str]					=					MarkupText(f'Inference on a model too large for GPU memory\nis successfully completed.'   ,				font_size=24			)
     step_a.move_to([2, 2, 0]			)

     self.play(Write(lowerCamelCase__   ,				run_time=3			)   ,				MoveToTarget(lowerCamelCase__			)			)

     self.wait()



