



'''simple docstring'''

from typing import List, Optional, Union

import numpy as np
import tensorflow as tf

from .utils import logging


snake_case_			=     logging.get_logger(__name__)






def       __lowercase  (_SCREAMING_SNAKE_CASE      :Union[tf.Tensor, np.ndarray]       ):
				if isinstance(_SCREAMING_SNAKE_CASE    ,	np.ndarray       ):
								return list(tensor.shape       )

				SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= tf.shape(_SCREAMING_SNAKE_CASE       )

				if tensor.shape == tf.TensorShape(_SCREAMING_SNAKE_CASE       ):
								return dynamic

				SCREAMING_SNAKE_CASE  :						Dict      					= tensor.shape.as_list()

				return [dynamic[i] if s is None else s for i, s in enumerate(_SCREAMING_SNAKE_CASE       )]






def       __lowercase  (_SCREAMING_SNAKE_CASE      :tf.Tensor    ,	_SCREAMING_SNAKE_CASE      :Optional[int] = None    ,	_SCREAMING_SNAKE_CASE      :Optional[str] = None       ):
				return tf.nn.softmax(logits=logits + 1E-9    ,	axis=_SCREAMING_SNAKE_CASE    ,	name=_SCREAMING_SNAKE_CASE       )






def       __lowercase  (_SCREAMING_SNAKE_CASE      :int    ,	_SCREAMING_SNAKE_CASE      :Union[str, Any]    ,	_SCREAMING_SNAKE_CASE      :List[Any]    ,	_SCREAMING_SNAKE_CASE      :List[str]=1E-5    ,	_SCREAMING_SNAKE_CASE      :Union[str, Any]=-1       ):
				# This is a very simplified functional layernorm, designed to duplicate
				# the functionality of PyTorch nn.functional.layer_norm when this is needed to port
				# models in Transformers.

				if weight.shape.rank != 1 or bias.shape.rank != 1 or not isinstance(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE       ):
								raise NotImplementedError('''Only 1D weight and bias tensors are supported for now, with only a single axis.'''       )

				# Get mean and variance on the axis to be normalized
				SCREAMING_SNAKE_CASE					,					SCREAMING_SNAKE_CASE  :						Optional[int]      					= tf.nn.moments(_SCREAMING_SNAKE_CASE    ,	axes=[axis]    ,	keepdims=_SCREAMING_SNAKE_CASE       )

				if axis != -1:
								# Reshape scale and weight to have the same rank as inputs, but with 1 dimensions
								# on every dimension except axis
								SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= [1] * inputs.shape.rank
								SCREAMING_SNAKE_CASE  :						Dict      					= shape_list(_SCREAMING_SNAKE_CASE       )[axis]
								SCREAMING_SNAKE_CASE  :						Any      					= tf.reshape(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE       )
								SCREAMING_SNAKE_CASE  :						str      					= tf.reshape(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE       )

				# Compute layer normalization using the batch_normalization
				# function.
				SCREAMING_SNAKE_CASE  :						Optional[int]      					= tf.nn.batch_normalization(
				    _SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE    ,	offset=_SCREAMING_SNAKE_CASE    ,	scale=_SCREAMING_SNAKE_CASE    ,	variance_epsilon=_SCREAMING_SNAKE_CASE    ,	)
				return outputs






def       __lowercase  (_SCREAMING_SNAKE_CASE      :Optional[Any]    ,	_SCREAMING_SNAKE_CASE      :List[str]=0    ,	_SCREAMING_SNAKE_CASE      :Dict=-1       ):
				# Replicates the behavior of torch.flatten in TF

				# If end_dim or start_dim is negative, count them from the end
				if end_dim < 0:
								end_dim += input.shape.rank
				if start_dim < 0:
								start_dim += input.shape.rank

				if start_dim == end_dim:
								return input

				SCREAMING_SNAKE_CASE  :						Optional[int]      					= tf.shape(_SCREAMING_SNAKE_CASE       )
				SCREAMING_SNAKE_CASE  :						Optional[int]      					= tf.math.reduce_prod(in_shape[start_dim : end_dim + 1]       )
				SCREAMING_SNAKE_CASE  :						List[str]      					= tf.concat([in_shape[:start_dim], [flattened_dim], in_shape[end_dim + 1 :]]    ,	axis=0       )
				return tf.reshape(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE       )






def       __lowercase  (_SCREAMING_SNAKE_CASE      :tf.Tensor       ):
				if not isinstance(_SCREAMING_SNAKE_CASE    ,	tf.Tensor       ):
								SCREAMING_SNAKE_CASE  :						List[Any]      					= tf.convert_to_tensor(_SCREAMING_SNAKE_CASE       )  # Catches stray NumPy inputs
				if encoder_attention_mask.shape.rank == 3:
								SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= encoder_attention_mask[:, None, :, :]
				if encoder_attention_mask.shape.rank == 2:
								SCREAMING_SNAKE_CASE  :						List[str]      					= encoder_attention_mask[:, None, None, :]
				# T5 has a mask that can compare sequence ids, we can simulate this here with this transposition
				# Cf. https://github.com/tensorflow/mesh/blob/8d2465e9bc93129b913b5ccc6a59aa97abd96ec6/mesh_tensorflow
				# /transformer/transformer_layers.py#L270
				# encoder_extended_attention_mask = (encoder_extended_attention_mask ==
				# encoder_extended_attention_mask.transpose(-1, -2))
				SCREAMING_SNAKE_CASE  :						Optional[Any]      					= (
				    tf.cast(1    ,	encoder_attention_mask.dtype       ) - encoder_extended_attention_mask
				) * encoder_extended_attention_mask.dtype.min

				return encoder_extended_attention_mask






def       __lowercase  (_SCREAMING_SNAKE_CASE      :tf.Tensor    ,	_SCREAMING_SNAKE_CASE      :int    ,	_SCREAMING_SNAKE_CASE      :str = "input_ids"       ):
				tf.debugging.assert_less(
				    _SCREAMING_SNAKE_CASE    ,	tf.cast(_SCREAMING_SNAKE_CASE    ,	dtype=tensor.dtype       )    ,	message=(
				        F'''The maximum value of {tensor_name} ({tf.math.reduce_max(_SCREAMING_SNAKE_CASE       )}) must be smaller than the embedding '''
				        F'''layer\'s input dimension ({embed_dim}). The likely cause is some problem at tokenization time.'''
				    )    ,	)






def       __lowercase  (_SCREAMING_SNAKE_CASE      :List[str]    ,	_SCREAMING_SNAKE_CASE      :List[str]    ,	_SCREAMING_SNAKE_CASE      :List[str]       ):
				SCREAMING_SNAKE_CASE  :						List[str]      					= 6_45_12
				# Check that no item in `data` is larger than `HDF5_OBJECT_HEADER_LIMIT`
				# because in that case even chunking the array would not make the saving
				# possible.
				SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= [x for x in data if len(_SCREAMING_SNAKE_CASE       ) > HDF5_OBJECT_HEADER_LIMIT]

				# Expecting this to never be true.
				if bad_attributes:
								raise RuntimeError(
								    '''The following attributes cannot be saved to HDF5 file because '''
								    F'''they are larger than {HDF5_OBJECT_HEADER_LIMIT} '''
								    F'''bytes: {bad_attributes}'''       )

				SCREAMING_SNAKE_CASE  :						Optional[int]      					= np.asarray(_SCREAMING_SNAKE_CASE       )

				SCREAMING_SNAKE_CASE  :						List[str]      					= 1
				SCREAMING_SNAKE_CASE  :						str      					= np.array_split(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE       )

				# This will never loop forever thanks to the test above.
				while any(x.nbytes > HDF5_OBJECT_HEADER_LIMIT for x in chunked_data       ):
								num_chunks += 1
								SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= np.array_split(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE       )

				if num_chunks > 1:
								for chunk_id, chunk_data in enumerate(_SCREAMING_SNAKE_CASE       ):
												SCREAMING_SNAKE_CASE  :						Tuple      					= chunk_data
				else:
								SCREAMING_SNAKE_CASE  :						List[str]      					= data






def       __lowercase  (_SCREAMING_SNAKE_CASE      :Dict    ,	_SCREAMING_SNAKE_CASE      :Optional[Any]       ):
				if name in group.attrs:
								SCREAMING_SNAKE_CASE  :						Tuple      					= [n.decode('''utf8'''       ) if hasattr(_SCREAMING_SNAKE_CASE    ,	'''decode'''       ) else n for n in group.attrs[name]]
				else:
								SCREAMING_SNAKE_CASE  :						List[Any]      					= []
								SCREAMING_SNAKE_CASE  :						Optional[int]      					= 0
								while "%s%d" % (name, chunk_id) in group.attrs:
												data.extend(
												    [n.decode('''utf8'''       ) if hasattr(_SCREAMING_SNAKE_CASE    ,	'''decode'''       ) else n for n in group.attrs['''%s%d''' % (name, chunk_id)]]       )
												chunk_id += 1
				return data






def       __lowercase  (_SCREAMING_SNAKE_CASE      :Optional[Any]       ):

				def _expand_single_ad_tensor(_SCREAMING_SNAKE_CASE      :Optional[int]       ):
								if isinstance(_SCREAMING_SNAKE_CASE    ,	tf.Tensor       ) and t.shape.rank == 1:
												return tf.expand_dims(_SCREAMING_SNAKE_CASE    ,	axis=-1       )
								return t

				return tf.nest.map_structure(_expand_single_ad_tensor    ,	_SCREAMING_SNAKE_CASE       )









'''simple docstring'''

from manim import *


class 	a__     (				_lowercase	):


	def 				lowercase__  (self : Optional[Any]      )     ->			str:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						List[Any]      					= Rectangle(height=0.5,					width=0.5      )
					SCREAMING_SNAKE_CASE  :						str      					= Rectangle(height=0.46,					width=0.46      ).set_stroke(width=0      )

					SCREAMING_SNAKE_CASE  :						List[str]      					= [mem.copy() for i in range(6      )]
					SCREAMING_SNAKE_CASE  :						Optional[int]      					= [mem.copy() for i in range(6      )]
					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= VGroup(*__UpperCAmelCase      ).arrange(__UpperCAmelCase,					buff=0      )
					SCREAMING_SNAKE_CASE  :						List[str]      					= VGroup(*__UpperCAmelCase      ).arrange(__UpperCAmelCase,					buff=0      )
					SCREAMING_SNAKE_CASE  :						List[Any]      					= VGroup(__UpperCAmelCase,					__UpperCAmelCase      ).arrange(__UpperCAmelCase,					buff=0      )
					SCREAMING_SNAKE_CASE  :						Tuple      					= Text('''CPU''',					font_size=24      )
					SCREAMING_SNAKE_CASE  :						Dict      					= Group(__UpperCAmelCase,					__UpperCAmelCase      ).arrange(__UpperCAmelCase,					buff=0.5,					aligned_edge=__UpperCAmelCase      )
					cpu.move_to([-2.5, -0.5, 0]      )
					self.add(__UpperCAmelCase      )

					SCREAMING_SNAKE_CASE  :						Any      					= [mem.copy() for i in range(1      )]
					SCREAMING_SNAKE_CASE  :						Dict      					= VGroup(*__UpperCAmelCase      ).arrange(__UpperCAmelCase,					buff=0      )
					SCREAMING_SNAKE_CASE  :						List[str]      					= Text('''GPU''',					font_size=24      )
					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= Group(__UpperCAmelCase,					__UpperCAmelCase      ).arrange(__UpperCAmelCase,					buff=0.5,					aligned_edge=__UpperCAmelCase      )
					gpu.align_to(__UpperCAmelCase,					__UpperCAmelCase      )
					gpu.set_x(gpu.get_x() - 1      )

					self.add(__UpperCAmelCase      )

					SCREAMING_SNAKE_CASE  :						List[Any]      					= [mem.copy() for i in range(6      )]
					SCREAMING_SNAKE_CASE  :						Any      					= VGroup(*__UpperCAmelCase      ).arrange(__UpperCAmelCase,					buff=0      )

					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= Text('''Model''',					font_size=24      )
					SCREAMING_SNAKE_CASE  :						Dict      					= Group(__UpperCAmelCase,					__UpperCAmelCase      ).arrange(__UpperCAmelCase,					buff=0.5,					aligned_edge=__UpperCAmelCase      )
					model.move_to([3, -1.0, 0]      )

					self.play(
					    Create(__UpperCAmelCase,					run_time=1      ),					Create(__UpperCAmelCase,					run_time=1      ),					Create(__UpperCAmelCase,					run_time=1      ),					)

					SCREAMING_SNAKE_CASE  :						Tuple      					= MarkupText(
					    F'''First, an empty model skeleton is loaded\ninto <span fgcolor=\'{YELLOW}\'>memory</span> without using much RAM.''',					font_size=24,					)

					SCREAMING_SNAKE_CASE  :						Optional[Any]      					= Square(side_length=2.2      )
					key.move_to([-5, 2, 0]      )

					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= MarkupText(
					    F'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model''',					font_size=18,					)

					key_text.move_to([-5, 2.4, 0]      )

					step_a.move_to([2, 2, 0]      )
					self.play(Write(__UpperCAmelCase,					run_time=2.5      ),					Write(__UpperCAmelCase      ),					Write(__UpperCAmelCase      )      )

					self.add(__UpperCAmelCase      )

					SCREAMING_SNAKE_CASE  :						List[str]      					= []
					SCREAMING_SNAKE_CASE  :						List[str]      					= []
					SCREAMING_SNAKE_CASE  :						Dict      					= []
					for i, rect in enumerate(__UpperCAmelCase      ):
									SCREAMING_SNAKE_CASE  :						Optional[int]      					= Rectangle(height=0.46,					width=0.46      ).set_stroke(width=0.0      ).set_fill(__UpperCAmelCase,					opacity=0.7      )
									cpu_target.move_to(__UpperCAmelCase      )
									cpu_target.generate_target()
									SCREAMING_SNAKE_CASE  :						List[str]      					= 0.46 / 4
									SCREAMING_SNAKE_CASE  :						Any      					= 0.46 / 3

									if i == 0:
													cpu_target.target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT      ),					buff=0.02,					direction=__UpperCAmelCase      )
													cpu_target.target.set_x(cpu_target.target.get_x() + 0.1      )
									elif i == 3:
													cpu_target.target.next_to(cpu_targs[0].target,					direction=__UpperCAmelCase,					buff=0.0      )
									else:
													cpu_target.target.next_to(cpu_targs[i - 1].target,					direction=__UpperCAmelCase,					buff=0.0      )
									cpu_targs.append(__UpperCAmelCase      )

									first_animations.append(rect.animate(run_time=0.5      ).set_stroke(__UpperCAmelCase      )      )
									second_animations.append(MoveToTarget(__UpperCAmelCase,					run_time=1.5      )      )

					self.play(*__UpperCAmelCase      )
					self.play(*__UpperCAmelCase      )

					self.wait()




