





'''simple docstring'''


from __future__ import annotations

import unittest

from transformers import FunnelConfig, is_tf_available
from transformers.testing_utils import require_tf

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers import (
					    TFFunnelBaseModel,
					    TFFunnelForMaskedLM,
					    TFFunnelForMultipleChoice,
					    TFFunnelForPreTraining,
					    TFFunnelForQuestionAnswering,
					    TFFunnelForSequenceClassification,
					    TFFunnelForTokenClassification,
					    TFFunnelModel,
					)






class       UpperCamelCase__  :







	def __init__(				self			:   Union[str, Any] , lowerCamelCase			:   int , lowerCamelCase			:   List[Any]=1_3 , lowerCamelCase			:   str=7 , lowerCamelCase			:   Union[str, Any]=True , lowerCamelCase			:   Tuple=True , lowerCamelCase			:   Tuple=True , lowerCamelCase			:   Optional[Any]=True , lowerCamelCase			:   List[Any]=9_9 , lowerCamelCase			:   List[str]=[1, 1, 2] , lowerCamelCase			:   Optional[Any]=1 , lowerCamelCase			:   int=3_2 , lowerCamelCase			:   Optional[Any]=4 , lowerCamelCase			:   Optional[Any]=8 , lowerCamelCase			:   List[str]=3_7 , lowerCamelCase			:   Optional[int]="gelu_new" , lowerCamelCase			:   str=0.1 , lowerCamelCase			:   Union[str, Any]=0.1 , lowerCamelCase			:   Union[str, Any]=0.0 , lowerCamelCase			:   List[Any]=5_1_2 , lowerCamelCase			:   List[Any]=3 , lowerCamelCase			:   Optional[Any]=0.02 , lowerCamelCase			:   Optional[int]=3 , lowerCamelCase			:   List[str]=4 , lowerCamelCase			:   int=None , lowerCamelCase			:   Tuple=False , ):




				'''simple docstring'''
				a__     = parent
				a__     = batch_size
				a__     = seq_length
				a__     = is_training
				a__     = use_input_mask
				a__     = use_token_type_ids
				a__     = use_labels
				a__     = vocab_size
				a__     = block_sizes
				a__     = num_decoder_layers
				a__     = d_model
				a__     = n_head
				a__     = d_head
				a__     = d_inner
				a__     = hidden_act
				a__     = hidden_dropout
				a__     = attention_dropout
				a__     = activation_dropout
				a__     = max_position_embeddings
				a__     = type_vocab_size
				a__     = 2
				a__     = num_labels
				a__     = num_choices
				a__     = scope
				a__     = initializer_std

				# Used in the tests to check the size of the first attention layer
				a__     = n_head
				# Used in the tests to check the size of the first hidden state
				a__     = self.d_model
				# Used in the tests to check the number of output hidden states/attentions
				a__     = sum(self.block_sizes		) + (0 if base else self.num_decoder_layers)
				# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
				# the last hidden state of the first block (which is the first hidden state of the decoder).
				if not base:
							a__     = self.num_hidden_layers + 2







	def 						__a   (				self			:   List[str]		):




				'''simple docstring'''
				a__     = ids_tensor([self.batch_size, self.seq_length] , self.vocab_size		)

				a__     = None
				if self.use_input_mask:
							a__     = random_attention_mask([self.batch_size, self.seq_length]		)

				a__     = None
				if self.use_token_type_ids:
							a__     = ids_tensor([self.batch_size, self.seq_length] , self.type_vocab_size		)

				a__     = None
				a__     = None
				a__     = None
				if self.use_labels:
							a__     = ids_tensor([self.batch_size] , self.type_sequence_label_size		)
							a__     = ids_tensor([self.batch_size, self.seq_length] , self.num_labels		)
							a__     = ids_tensor([self.batch_size] , self.num_choices		)

				a__     = FunnelConfig(
				    vocab_size=self.vocab_size , block_sizes=self.block_sizes , num_decoder_layers=self.num_decoder_layers , d_model=self.d_model , n_head=self.n_head , d_head=self.d_head , d_inner=self.d_inner , hidden_act=self.hidden_act , hidden_dropout=self.hidden_dropout , attention_dropout=self.attention_dropout , activation_dropout=self.activation_dropout , max_position_embeddings=self.max_position_embeddings , type_vocab_size=self.type_vocab_size , initializer_std=self.initializer_std , )

				return (
				    config,
				    input_ids,
				    token_type_ids,
				    input_mask,
				    sequence_labels,
				    token_labels,
				    choice_labels,
				)







	def 						__a   (				self			:   Optional[Any] , lowerCamelCase			:   Optional[int] , lowerCamelCase			:   Any , lowerCamelCase			:   Tuple , lowerCamelCase			:   int , lowerCamelCase			:   Dict , lowerCamelCase			:   Union[str, Any] , lowerCamelCase			:   int , ):




				'''simple docstring'''
				a__     = TFFunnelModel(config=__A		)
				a__     = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
				a__     = model(__A		)

				a__     = [input_ids, input_mask]
				a__     = model(__A		)

				a__     = model(__A		)
				self.parent.assertEqual(result.last_hidden_state.shape , (self.batch_size, self.seq_length, self.d_model)		)

				a__     = False
				a__     = TFFunnelModel(config=__A		)
				a__     = model(__A		)
				self.parent.assertEqual(result.last_hidden_state.shape , (self.batch_size, self.seq_length, self.d_model)		)

				a__     = False
				a__     = TFFunnelModel(config=__A		)
				a__     = model(__A		)
				self.parent.assertEqual(result.last_hidden_state.shape , (self.batch_size, self.seq_length, self.d_model)		)







	def 						__a   (				self			:   Optional[int] , lowerCamelCase			:   Dict , lowerCamelCase			:   List[str] , lowerCamelCase			:   List[Any] , lowerCamelCase			:   Optional[int] , lowerCamelCase			:   Union[str, Any] , lowerCamelCase			:   int , lowerCamelCase			:   List[Any] , ):




				'''simple docstring'''
				a__     = TFFunnelBaseModel(config=__A		)
				a__     = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
				a__     = model(__A		)

				a__     = [input_ids, input_mask]
				a__     = model(__A		)

				a__     = model(__A		)
				self.parent.assertEqual(result.last_hidden_state.shape , (self.batch_size, 2, self.d_model)		)

				a__     = False
				a__     = TFFunnelBaseModel(config=__A		)
				a__     = model(__A		)
				self.parent.assertEqual(result.last_hidden_state.shape , (self.batch_size, 3, self.d_model)		)

				a__     = False
				a__     = TFFunnelBaseModel(config=__A		)
				a__     = model(__A		)
				self.parent.assertEqual(result.last_hidden_state.shape , (self.batch_size, 2, self.d_model)		)







	def 						__a   (				self			:   List[Any] , lowerCamelCase			:   List[str] , lowerCamelCase			:   int , lowerCamelCase			:   Optional[int] , lowerCamelCase			:   Dict , lowerCamelCase			:   Any , lowerCamelCase			:   Optional[Any] , lowerCamelCase			:   Dict , ):




				'''simple docstring'''
				a__     = TFFunnelForPreTraining(config=__A		)
				a__     = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
				a__     = model(__A		)
				self.parent.assertEqual(result.logits.shape , (self.batch_size, self.seq_length)		)







	def 						__a   (				self			:   str , lowerCamelCase			:   Optional[Any] , lowerCamelCase			:   Optional[int] , lowerCamelCase			:   Optional[Any] , lowerCamelCase			:   Optional[int] , lowerCamelCase			:   Optional[int] , lowerCamelCase			:   Dict , lowerCamelCase			:   Optional[Any] , ):




				'''simple docstring'''
				a__     = TFFunnelForMaskedLM(config=__A		)
				a__     = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
				a__     = model(__A		)
				self.parent.assertEqual(result.logits.shape , (self.batch_size, self.seq_length, self.vocab_size)		)







	def 						__a   (				self			:   List[str] , lowerCamelCase			:   List[str] , lowerCamelCase			:   Union[str, Any] , lowerCamelCase			:   Optional[Any] , lowerCamelCase			:   List[Any] , lowerCamelCase			:   Union[str, Any] , lowerCamelCase			:   str , lowerCamelCase			:   Tuple , ):




				'''simple docstring'''
				a__     = self.num_labels
				a__     = TFFunnelForSequenceClassification(config=__A		)
				a__     = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
				a__     = model(__A		)
				self.parent.assertEqual(result.logits.shape , (self.batch_size, self.num_labels)		)







	def 						__a   (				self			:   Any , lowerCamelCase			:   Optional[Any] , lowerCamelCase			:   List[Any] , lowerCamelCase			:   Any , lowerCamelCase			:   Any , lowerCamelCase			:   Any , lowerCamelCase			:   Dict , lowerCamelCase			:   Optional[Any] , ):




				'''simple docstring'''
				a__     = self.num_choices
				a__     = TFFunnelForMultipleChoice(config=__A		)
				a__     = tf.tile(tf.expand_dims(__A , 1		) , (1, self.num_choices, 1)		)
				a__     = tf.tile(tf.expand_dims(__A , 1		) , (1, self.num_choices, 1)		)
				a__     = tf.tile(tf.expand_dims(__A , 1		) , (1, self.num_choices, 1)		)
				a__     = {
				    "input_ids": multiple_choice_inputs_ids,
				    "attention_mask": multiple_choice_input_mask,
				    "token_type_ids": multiple_choice_token_type_ids,
				}
				a__     = model(__A		)
				self.parent.assertEqual(result.logits.shape , (self.batch_size, self.num_choices)		)







	def 						__a   (				self			:   Optional[int] , lowerCamelCase			:   int , lowerCamelCase			:   Optional[int] , lowerCamelCase			:   Dict , lowerCamelCase			:   str , lowerCamelCase			:   List[Any] , lowerCamelCase			:   Optional[int] , lowerCamelCase			:   List[str] , ):




				'''simple docstring'''
				a__     = self.num_labels
				a__     = TFFunnelForTokenClassification(config=__A		)
				a__     = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
				a__     = model(__A		)
				self.parent.assertEqual(result.logits.shape , (self.batch_size, self.seq_length, self.num_labels)		)







	def 						__a   (				self			:   Optional[int] , lowerCamelCase			:   Tuple , lowerCamelCase			:   List[Any] , lowerCamelCase			:   Union[str, Any] , lowerCamelCase			:   Dict , lowerCamelCase			:   Dict , lowerCamelCase			:   Dict , lowerCamelCase			:   Dict , ):




				'''simple docstring'''
				a__     = TFFunnelForQuestionAnswering(config=__A		)
				a__     = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
				a__     = model(__A		)
				self.parent.assertEqual(result.start_logits.shape , (self.batch_size, self.seq_length)		)
				self.parent.assertEqual(result.end_logits.shape , (self.batch_size, self.seq_length)		)







	def 						__a   (				self			:   int		):




				'''simple docstring'''
				a__     = self.prepare_config_and_inputs()
				(
				    a__		
				)     = config_and_inputs
				a__     = {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
				return config, inputs_dict






@require_tf
class       UpperCamelCase__  (		UpperCamelCase_	,UpperCamelCase_	,unittest.TestCase     ):
	lowerCAmelCase__					:			int								=						(
	    (
	        TFFunnelModel,
	        TFFunnelForMaskedLM,
	        TFFunnelForPreTraining,
	        TFFunnelForQuestionAnswering,
	        TFFunnelForTokenClassification,
	    )
	    if is_tf_available()
	    else ()
	)
	lowerCAmelCase__					:			Union[str, Any]								=						(
	    {
	        "feature-extraction": (TFFunnelBaseModel, TFFunnelModel),
	        "fill-mask": TFFunnelForMaskedLM,
	        "question-answering": TFFunnelForQuestionAnswering,
	        "text-classification": TFFunnelForSequenceClassification,
	        "token-classification": TFFunnelForTokenClassification,
	        "zero-shot": TFFunnelForSequenceClassification,
	    }
	    if is_tf_available()
	    else {}
	)
	lowerCAmelCase__					:			List[str]								=						False
	lowerCAmelCase__					:			int								=						False







	def 						__a   (				self			:   Dict		):




				'''simple docstring'''
				a__     = TFFunnelModelTester(self		)
				a__     = ConfigTester(self , config_class=__A		)







	def 						__a   (				self			:   Union[str, Any]		):




				'''simple docstring'''
				self.config_tester.run_common_tests()







	def 						__a   (				self			:   List[str]		):




				'''simple docstring'''
				a__     = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_model(*__A		)







	def 						__a   (				self			:   Optional[Any]		):




				'''simple docstring'''
				a__     = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_pretraining(*__A		)







	def 						__a   (				self			:   int		):




				'''simple docstring'''
				a__     = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_masked_lm(*__A		)







	def 						__a   (				self			:   Tuple		):




				'''simple docstring'''
				a__     = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_token_classification(*__A		)







	def 						__a   (				self			:   Optional[Any]		):




				'''simple docstring'''
				a__     = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_question_answering(*__A		)






@require_tf
class       UpperCamelCase__  (		UpperCamelCase_	,unittest.TestCase     ):
	lowerCAmelCase__					:			Dict								=						(
	    (TFFunnelBaseModel, TFFunnelForMultipleChoice, TFFunnelForSequenceClassification) if is_tf_available() else ()
	)
	lowerCAmelCase__					:			Optional[int]								=						False
	lowerCAmelCase__					:			Dict								=						False







	def 						__a   (				self			:   Any		):




				'''simple docstring'''
				a__     = TFFunnelModelTester(self , base=__A		)
				a__     = ConfigTester(self , config_class=__A		)







	def 						__a   (				self			:   List[str]		):




				'''simple docstring'''
				self.config_tester.run_common_tests()







	def 						__a   (				self			:   Optional[Any]		):




				'''simple docstring'''
				a__     = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_base_model(*__A		)







	def 						__a   (				self			:   List[Any]		):




				'''simple docstring'''
				a__     = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_sequence_classification(*__A		)







	def 						__a   (				self			:   Any		):




				'''simple docstring'''
				a__     = self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_multiple_choice(*__A		)










'''simple docstring'''


from manim import *






class       UpperCamelCase__  (		__lowerCAmelCase     ):







	def 						__a   (				self			:   List[Any]		):




				'''simple docstring'''
				a__     = Rectangle(height=0.5 , width=0.5		)
				a__     = Rectangle(height=0.25 , width=0.25		)
				a__     = Rectangle(height=0.46 , width=0.46		).set_stroke(width=0		)

				a__     = [mem.copy() for i in range(6		)]
				a__     = [mem.copy() for i in range(6		)]
				a__     = VGroup(*lowerCamelCase		).arrange(lowerCamelCase , buff=0		)
				a__     = VGroup(*lowerCamelCase		).arrange(lowerCamelCase , buff=0		)
				a__     = VGroup(lowerCamelCase , lowerCamelCase		).arrange(lowerCamelCase , buff=0		)
				a__     = Text("CPU" , font_size=2_4		)
				a__     = Group(lowerCamelCase , lowerCamelCase		).arrange(lowerCamelCase , buff=0.5 , aligned_edge=lowerCamelCase		)
				cpu.move_to([-2.5, -0.5, 0]		)
				self.add(lowerCamelCase		)

				a__     = [mem.copy() for i in range(4		)]
				a__     = VGroup(*lowerCamelCase		).arrange(lowerCamelCase , buff=0		)
				a__     = Text("GPU" , font_size=2_4		)
				a__     = Group(lowerCamelCase , lowerCamelCase		).arrange(lowerCamelCase , buff=0.5 , aligned_edge=lowerCamelCase		)
				gpu.move_to([-1, -1, 0]		)
				self.add(lowerCamelCase		)

				a__     = [mem.copy() for i in range(6		)]
				a__     = VGroup(*lowerCamelCase		).arrange(lowerCamelCase , buff=0		)

				a__     = Text("Model" , font_size=2_4		)
				a__     = Group(lowerCamelCase , lowerCamelCase		).arrange(lowerCamelCase , buff=0.5 , aligned_edge=lowerCamelCase		)
				model.move_to([3, -1.0, 0]		)
				self.add(lowerCamelCase		)

				a__     = []
				a__     = []
				a__     = []

				for i, rect in enumerate(lowerCamelCase		):
							rect.set_stroke(lowerCamelCase		)

							a__     = Rectangle(height=0.46 / 4 , width=0.46 / 3		).set_stroke(width=0.0		).set_fill(lowerCamelCase , opacity=0.7		)

							if i == 0:
										cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT		) , buff=0.02 , direction=lowerCamelCase		)
										cpu_target.set_x(cpu_target.get_x() + 0.1		)
							elif i == 3:
										cpu_target.next_to(model_cpu_arr[0] , direction=lowerCamelCase , buff=0.0		)
							else:
										cpu_target.next_to(model_cpu_arr[i - 1] , direction=lowerCamelCase , buff=0.0		)
							self.add(lowerCamelCase		)
							model_cpu_arr.append(lowerCamelCase		)

				self.add(*lowerCamelCase , *lowerCamelCase , *lowerCamelCase		)

				a__     = [mem.copy() for i in range(6		)]
				a__     = VGroup(*lowerCamelCase		).arrange(lowerCamelCase , buff=0		)

				a__     = Text("Loaded Checkpoint" , font_size=2_4		)
				a__     = Group(lowerCamelCase , lowerCamelCase		).arrange(lowerCamelCase , buff=0.5 , aligned_edge=lowerCamelCase		)
				checkpoint.move_to([3, 0.5, 0]		)

				self.add(lowerCamelCase		)

				a__     = []
				a__     = []

				for i, rect in enumerate(lowerCamelCase		):
							a__     = fill.copy().set_fill(lowerCamelCase , opacity=0.7		)
							target.move_to(lowerCamelCase		)
							ckpt_arr.append(lowerCamelCase		)

							a__     = target.copy()
							if i < 5:
										cpu_target.move_to(cpu_left_col_base[i + 1]		)
							else:
										cpu_target.move_to(cpu_right_col_base[i - 5]		)
							ckpt_cpu_arr.append(lowerCamelCase		)
				self.add(*lowerCamelCase , *lowerCamelCase		)

				a__     = Square(side_length=2.2		)
				key.move_to([-5, 2, 0]		)

				a__     = MarkupText(
				    F'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model''' , font_size=1_8 , )

				key_text.move_to([-5, 2.4, 0]		)

				self.add(lowerCamelCase , lowerCamelCase		)

				a__     = MarkupText(
				    F'''<span fgcolor=\'{BLUE}\'>●</span> Checkpoint''' , font_size=1_8 , )

				blue_text.next_to(lowerCamelCase , DOWN * 2.4 , aligned_edge=key_text.get_left()		)
				self.add(lowerCamelCase		)

				a__     = MarkupText(
				    F'''Based on the passed in configuration, weights are stored in\na variety of np.memmaps on disk or to a particular device.''' , font_size=2_4 , )
				step_a.move_to([2, 2, 0]		)

				a__     = [meta_mem.copy() for i in range(6		)]
				a__     = [meta_mem.copy() for i in range(6		)]
				a__     = VGroup(*lowerCamelCase		).arrange(lowerCamelCase , buff=0		)
				a__     = VGroup(*lowerCamelCase		).arrange(lowerCamelCase , buff=0		)
				a__     = VGroup(lowerCamelCase , lowerCamelCase		).arrange(lowerCamelCase , buff=0		)
				a__     = Text("Disk" , font_size=2_4		)
				a__     = Group(lowerCamelCase , lowerCamelCase		).arrange(lowerCamelCase , buff=0.5 , aligned_edge=lowerCamelCase		)
				disk.move_to([-4.0, -1.25, 0]		)
				self.play(Write(lowerCamelCase , run_time=3		) , Write(lowerCamelCase , run_time=1		) , Create(lowerCamelCase , run_time=1		)		)

				a__     = []
				for i, rect in enumerate(lowerCamelCase		):
							a__     = rect.copy()
							target.generate_target()
							target.target.move_to(disk_left_col_base[i]		).scale(0.5		)
							animations.append(MoveToTarget(lowerCamelCase , run_time=1.5		)		)
				self.play(*lowerCamelCase		)

				self.play(FadeOut(lowerCamelCase		)		)

				a__     = MarkupText(F'''Then, the checkpoint is removed from memory\nthrough garbage collection.''' , font_size=2_4		)
				step_a.move_to([2, 2, 0]		)

				self.play(Write(lowerCamelCase , run_time=3		)		)

				self.play(
				    FadeOut(lowerCamelCase , lowerCamelCase , *lowerCamelCase , *lowerCamelCase		) , )

				self.wait()



