

from __future__ import annotations

import random
import unittest

from transformers import TransfoXLConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers import (
					    TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
					    TFTransfoXLForSequenceClassification,
					    TFTransfoXLLMHeadModel,
					    TFTransfoXLModel,
					)

class __UpperCamelCase     :






					'''simple docstring'''



					def __init__(				self				,      lowerCamelCase__				,      ):
						UpperCAmelCase__:	List[Any]									= parent
						UpperCAmelCase__:	Any									= 1_3
						UpperCAmelCase__:	Optional[Any]									= 7
						UpperCAmelCase__:	List[Any]									= 3_0
						UpperCAmelCase__:	Tuple									= self.seq_length + self.mem_len
						UpperCAmelCase__:	List[str]									= 1_5
						UpperCAmelCase__:	Dict									= True
						UpperCAmelCase__:	Dict									= True
						UpperCAmelCase__:	Optional[int]									= 9_9
						UpperCAmelCase__:	Dict									= [1_0, 5_0, 8_0]
						UpperCAmelCase__:	List[str]									= 3_2
						UpperCAmelCase__:	Optional[Any]									= 3_2
						UpperCAmelCase__:	int									= 4
						UpperCAmelCase__:	List[Any]									= 8
						UpperCAmelCase__:	Tuple									= 1_2_8
						UpperCAmelCase__:	List[Any]									= 2
						UpperCAmelCase__:	Tuple									= 2
						UpperCAmelCase__:	List[Any]									= None
						UpperCAmelCase__:	List[str]									= 1
						UpperCAmelCase__:	Union[str, Any]									= 0
						UpperCAmelCase__:	Any									= 3
						UpperCAmelCase__:	List[str]									= self.vocab_size - 1
						UpperCAmelCase__:	str									= 0.01





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	List[Any]									= ids_tensor([self.batch_size, self.seq_length]				,      self.vocab_size						)
						UpperCAmelCase__:	Any									= ids_tensor([self.batch_size, self.seq_length]				,      self.vocab_size						)

						UpperCAmelCase__:	List[str]									= None
						if self.use_labels:
							UpperCAmelCase__:	Union[str, Any]									= ids_tensor([self.batch_size, self.seq_length]				,      self.vocab_size						)

						UpperCAmelCase__:	List[Any]									= TransfoXLConfig(
						    vocab_size=self.vocab_size				,      mem_len=self.mem_len				,      clamp_len=self.clamp_len				,      cutoffs=self.cutoffs				,      d_model=self.hidden_size				,      d_embed=self.d_embed				,      n_head=self.num_attention_heads				,      d_head=self.d_head				,      d_inner=self.d_inner				,      div_val=self.div_val				,      n_layer=self.num_hidden_layers				,      eos_token_id=self.eos_token_id				,      pad_token_id=self.vocab_size - 1				,      init_range=self.init_range				,      num_labels=self.num_labels				,      )

						return (config, input_ids_a, input_ids_a, lm_labels)





					def    _UpperCAmelCase    (				self						):
						random.seed(self.seed						)
						tf.random.set_seed(self.seed						)





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__				,      lowerCamelCase__				,      lowerCamelCase__				,      lowerCamelCase__						):
						UpperCAmelCase__:	Union[str, Any]									= TFTransfoXLModel(lowerCamelCase__						)

						UpperCAmelCase__							,						UpperCAmelCase__:	Any									= model(lowerCamelCase__						).to_tuple()

						UpperCAmelCase__:	int									= {"input_ids": input_ids_a, "mems": mems_a}

						UpperCAmelCase__							,						UpperCAmelCase__:	Dict									= model(lowerCamelCase__						).to_tuple()

						self.parent.assertEqual(hidden_states_a.shape				,      (self.batch_size, self.seq_length, self.hidden_size)						)
						self.parent.assertEqual(hidden_states_a.shape				,      (self.batch_size, self.seq_length, self.hidden_size)						)
						self.parent.assertListEqual(
						    [mem.shape for mem in mems_a]				,      [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers				,      )
						self.parent.assertListEqual(
						    [mem.shape for mem in mems_a]				,      [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers				,      )





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__				,      lowerCamelCase__				,      lowerCamelCase__				,      lowerCamelCase__						):
						UpperCAmelCase__:	int									= TFTransfoXLLMHeadModel(lowerCamelCase__						)

						UpperCAmelCase__							,						UpperCAmelCase__:	List[str]									= model(lowerCamelCase__						).to_tuple()

						UpperCAmelCase__:	Dict									= {"input_ids": input_ids_a, "labels": lm_labels}
						UpperCAmelCase__							,						UpperCAmelCase__:	Union[str, Any]									= model(lowerCamelCase__						).to_tuple()

						UpperCAmelCase__							,						UpperCAmelCase__:	Union[str, Any]									= model([input_ids_a, mems_a]						).to_tuple()

						UpperCAmelCase__:	Any									= {"input_ids": input_ids_a, "mems": mems_a, "labels": lm_labels}

						UpperCAmelCase__							,						UpperCAmelCase__:	int									= model(lowerCamelCase__						).to_tuple()

						self.parent.assertEqual(lm_logits_a.shape				,      (self.batch_size, self.seq_length, self.vocab_size)						)
						self.parent.assertListEqual(
						    [mem.shape for mem in mems_a]				,      [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers				,      )

						self.parent.assertEqual(lm_logits_a.shape				,      (self.batch_size, self.seq_length, self.vocab_size)						)
						self.parent.assertListEqual(
						    [mem.shape for mem in mems_a]				,      [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers				,      )





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__				,      lowerCamelCase__				,      lowerCamelCase__				,      lowerCamelCase__						):
						UpperCAmelCase__:	Union[str, Any]									= TFTransfoXLForSequenceClassification(lowerCamelCase__						)
						UpperCAmelCase__:	List[str]									= model(lowerCamelCase__						)
						self.parent.assertEqual(result.logits.shape				,      (self.batch_size, self.num_labels)						)





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Any									= self.prepare_config_and_inputs()
						((UpperCAmelCase__)							,						(UpperCAmelCase__)							,						(UpperCAmelCase__)							,						(UpperCAmelCase__)):	Dict									= config_and_inputs
						UpperCAmelCase__:	Dict									= {"input_ids": input_ids_a}
						return config, inputs_dict

@require_tf
class __UpperCamelCase     (				_a					,_a					,unittest.TestCase		):






					'''simple docstring'''



					__magic_name__        =   (
					    (TFTransfoXLModel, TFTransfoXLLMHeadModel, TFTransfoXLForSequenceClassification) if is_tf_available() else ()
					)
					__magic_name__        =   () if is_tf_available() else ()
					__magic_name__        =   (
					    {
					        "feature-extraction": TFTransfoXLModel,
					        "text-classification": TFTransfoXLForSequenceClassification,
					        "text-generation": TFTransfoXLLMHeadModel,
					        "zero-shot": TFTransfoXLForSequenceClassification,
					    }
					    if is_tf_available()
					    else {}
					)
					# TODO: add this test when TFTransfoXLLMHead has a linear output layer implemented
					__magic_name__        =   False
					__magic_name__        =   False
					__magic_name__        =   False
					__magic_name__        =   False





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__				,      lowerCamelCase__				,      lowerCamelCase__				,      lowerCamelCase__				,      lowerCamelCase__						):
						if pipeline_test_casse_name == "TextGenerationPipelineTests":
							# Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`.
							# `TransfoXLConfig` was never used in pipeline tests: cannot create a simple
							# tokenizer.
							return True

						return False





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	List[Any]									= TFTransfoXLModelTester(self						)
						UpperCAmelCase__:	Optional[int]									= ConfigTester(self				,      config_class=lowerCamelCase__				,      d_embed=3_7						)





					def    _UpperCAmelCase    (				self						):
						self.config_tester.run_common_tests()





					def    _UpperCAmelCase    (				self						):
						self.model_tester.set_seed()
						UpperCAmelCase__:	List[str]									= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_transfo_xl_model(*lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						self.model_tester.set_seed()
						UpperCAmelCase__:	Union[str, Any]									= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_transfo_xl_lm_head(*lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	List[str]									= self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_transfo_xl_for_sequence_classification(*lowerCamelCase__						)





					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__							,						UpperCAmelCase__:	str									= self.model_tester.prepare_config_and_inputs_for_common()
						UpperCAmelCase__:	List[str]									= [TFTransfoXLForSequenceClassification]

						for model_class in self.all_model_classes:
							UpperCAmelCase__:	Optional[int]									= model_class(lowerCamelCase__						)
							assert isinstance(model.get_input_embeddings()				,      tf.keras.layers.Layer						)
							if model_class in list_other_models_with_output_ebd:
								UpperCAmelCase__:	Tuple									= model.get_output_embeddings()
								assert isinstance(lowerCamelCase__				,      tf.keras.layers.Layer						)
								UpperCAmelCase__:	Any									= model.get_bias()
								assert name is None
							else:
								UpperCAmelCase__:	List[Any]									= model.get_output_embeddings()
								assert x is None
								UpperCAmelCase__:	List[Any]									= model.get_bias()
								assert name is None





					def    _UpperCAmelCase    (				self						):
						# TODO JP: Make TransfoXL XLA compliant
						pass





					@slow
					def    _UpperCAmelCase    (				self						):
						for model_name in TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							UpperCAmelCase__:	str									= TFTransfoXLModel.from_pretrained(lowerCamelCase__						)
							self.assertIsNotNone(lowerCamelCase__						)





					@unittest.skip(reason="This model doesn't play well with fit() due to not returning a single loss."						)
					def    _UpperCAmelCase    (				self						):
						pass

@require_tf
class __UpperCamelCase     (				unittest.TestCase		):






					'''simple docstring'''



					@unittest.skip("Skip test until #12651 is resolved."						)
					@slow
					def    _UpperCAmelCase    (				self						):
						UpperCAmelCase__:	Any									= TFTransfoXLLMHeadModel.from_pretrained("transfo-xl-wt103"						)
						# fmt: off
						UpperCAmelCase__:	Tuple									= tf.convert_to_tensor([[3_3,1_2_9_7,2,1,1_0_0_9,4,1_1_0_9,1_1_7_3_9,4_7_6_2,3_5_8,5,2_5,2_4_5,2_2,1_7_0_6,1_7,2_0_0_9_8,5,3_2_1_5,2_1,3_7,1_1_1_0,3,1_3,1_0_4_1,4,2_4,6_0_3,4_9_0,2,7_1_4_7_7,2_0_0_9_8,1_0_4_4_4_7,2,2_0_9_6_1,1,2_6_0_4,4,1,3_2_9,3,6_2_2_4,8_3_1,1_6_0_0_2,2,8,6_0_3,7_8_9_6_7,2_9_5_4_6,2_3,8_0_3,2_0,2_5,4_1_6,5,8,2_3_2,4,2_7_7,6,1_8_5_5,4_6_0_1,3,2_9_5_4_6,5_4,8,3_6_0_9,5,5_7_2_1_1,4_9,4,1,2_7_7,1_8,8,1_7_5_5,1_5_6_9_1,3,3_4_1,2_5,4_1_6,6_9_3,4_2_5_7_3,7_1,1_7,4_0_1,9_4,3_1,1_7_9_1_9,2,2_9_5_4_6,7_8_7_3,1_8,1,4_3_5,2_3,1_1_0_1_1,7_5_5,5,5_1_6_7,3,7_9_8_3,9_8,8_4,2,2_9_5_4_6,3_2_6_7,8,3_6_0_9,4,1,4_8_6_5,1_0_7_5,2,6_0_8_7,7_1,6,3_4_6,8,5_8_5_4,3,2_9_5_4_6,8_2_4,1_4_0_0,1_8_6_8,2,1_9,1_6_0,2,3_1_1,8,5_4_9_6,2,2_0_9_2_0,1_7,2_5,1_5_0_9_7,3,2_4,2_4,0]]				,      dtype=tf.intaa						)  # noqa: E231
						# fmt: on
						#  In 1991 , the remains of Russian Tsar Nicholas II and his family
						#  ( except for Alexei and Maria ) are discovered .
						#  The voice of Nicholas's young son , Tsarevich Alexei Nikolaevich , narrates the
						#  remainder of the story . 1883 Western Siberia ,
						#  a young Grigori Rasputin is asked by his father and a group of men to perform magic .
						#  Rasputin has a vision and denounces one of the men as a horse thief . Although his
						#  father initially slaps him for making such an accusation , Rasputin watches as the
						#  man is chased outside and beaten . Twenty years later , Rasputin sees a vision of
						#  the Virgin Mary , prompting him to become a priest . Rasputin quickly becomes famous ,
						#  with people , even a bishop , begging for his blessing . <eod> </s> <eos>

						# fmt: off
						UpperCAmelCase__:	str									= [3_3,1_2_9_7,2,1,1_0_0_9,4,1_1_0_9,1_1_7_3_9,4_7_6_2,3_5_8,5,2_5,2_4_5,2_2,1_7_0_6,1_7,2_0_0_9_8,5,3_2_1_5,2_1,3_7,1_1_1_0,3,1_3,1_0_4_1,4,2_4,6_0_3,4_9_0,2,7_1_4_7_7,2_0_0_9_8,1_0_4_4_4_7,2,2_0_9_6_1,1,2_6_0_4,4,1,3_2_9,3,6_2_2_4,8_3_1,1_6_0_0_2,2,8,6_0_3,7_8_9_6_7,2_9_5_4_6,2_3,8_0_3,2_0,2_5,4_1_6,5,8,2_3_2,4,2_7_7,6,1_8_5_5,4_6_0_1,3,2_9_5_4_6,5_4,8,3_6_0_9,5,5_7_2_1_1,4_9,4,1,2_7_7,1_8,8,1_7_5_5,1_5_6_9_1,3,3_4_1,2_5,4_1_6,6_9_3,4_2_5_7_3,7_1,1_7,4_0_1,9_4,3_1,1_7_9_1_9,2,2_9_5_4_6,7_8_7_3,1_8,1,4_3_5,2_3,1_1_0_1_1,7_5_5,5,5_1_6_7,3,7_9_8_3,9_8,8_4,2,2_9_5_4_6,3_2_6_7,8,3_6_0_9,4,1,4_8_6_5,1_0_7_5,2,6_0_8_7,7_1,6,3_4_6,8,5_8_5_4,3,2_9_5_4_6,8_2_4,1_4_0_0,1_8_6_8,2,1_9,1_6_0,2,3_1_1,8,5_4_9_6,2,2_0_9_2_0,1_7,2_5,1_5_0_9_7,3,2_4,2_4,0,3_3,1,1_8_5_7,2,1,1_0_0_9,4,1_1_0_9,1_1_7_3_9,4_7_6_2,3_5_8,5,2_5,2_4_5,2_8,1_1_1_0,3,1_3,1_0_4_1,4,2_4,6_0_3,4_9_0,2,7_1_4_7_7,2_0_0_9_8,1_0_4_4_4_7,2,2_0_9_6_1,1,2_6_0_4,4,1,3_2_9,3,0]  # noqa: E231
						# fmt: on
						#  In 1991, the remains of Russian Tsar Nicholas II and his family (
						#  except for Alexei and Maria ) are discovered. The voice of young son,
						#  Tsarevich Alexei Nikolaevich, narrates the remainder of the story.
						#  1883 Western Siberia, a young Grigori Rasputin is asked by his father
						#  and a group of men to perform magic. Rasputin has a vision and
						#  denounces one of the men as a horse thief. Although his father initially
						#  slaps him for making such an accusation, Rasputin watches as the man
						#  is chased outside and beaten. Twenty years later, Rasputin sees a vision
						#  of the Virgin Mary, prompting him to become a priest.
						#  Rasputin quickly becomes famous, with people, even a bishop, begging for
						#  his blessing. <unk> <unk> <eos> In the 1990s, the remains of Russian Tsar
						# Nicholas II and his family were discovered. The voice of <unk> young son,
						# Tsarevich Alexei Nikolaevich, narrates the remainder of the story.<eos>

						UpperCAmelCase__:	Optional[Any]									= model.generate(lowerCamelCase__				,      max_length=2_0_0				,      do_sample=lowerCamelCase__						)
						self.assertListEqual(output_ids[0].numpy().tolist()				,      lowerCamelCase__						)


import torch
from torch import nn

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin

class __UpperCamelCase     (				_a					,_a		):






					'''simple docstring'''



					@register_to_config
					def __init__(				self				,      *,
					    lowerCamelCase__ = 4				,      lowerCamelCase__ = 7_6_8				,      lowerCamelCase__				,      lowerCamelCase__				,      ):
						super().__init__()

						UpperCAmelCase__:	int									= nn.Parameter(torch.zeros(lowerCamelCase__						)						)

						# parameters for additional clip time embeddings
						UpperCAmelCase__:	Optional[Any]									= nn.Linear(lowerCamelCase__				,      lowerCamelCase__						)
						UpperCAmelCase__:	str									= nn.Linear(lowerCamelCase__				,      lowerCamelCase__						)

						# parameters for encoder hidden states
						UpperCAmelCase__:	Tuple									= clip_extra_context_tokens
						UpperCAmelCase__:	List[str]									= nn.Linear(
						    lowerCamelCase__				,      self.clip_extra_context_tokens * cross_attention_dim						)
						UpperCAmelCase__:	Union[str, Any]									= nn.Linear(lowerCamelCase__				,      lowerCamelCase__						)
						UpperCAmelCase__:	str									= nn.LayerNorm(lowerCamelCase__						)





					def    _UpperCAmelCase    (				self				,      *, lowerCamelCase__				,      lowerCamelCase__				,      lowerCamelCase__				,      lowerCamelCase__						):
						if do_classifier_free_guidance:
							# Add the classifier free guidance embeddings to the image embeddings
							UpperCAmelCase__:	Any									= image_embeddings.shape[0]
							UpperCAmelCase__:	Optional[Any]									= self.learned_classifier_free_guidance_embeddings.unsqueeze(0						)
							UpperCAmelCase__:	Union[str, Any]									= classifier_free_guidance_embeddings.expand(
							    lowerCamelCase__				,      -1						)
							UpperCAmelCase__:	Tuple									= torch.cat([classifier_free_guidance_embeddings, image_embeddings]				,      dim=0						)

						# The image embeddings batch size and the text embeddings batch size are equal
						assert image_embeddings.shape[0] == prompt_embeds.shape[0]

						UpperCAmelCase__:	str									= prompt_embeds.shape[0]

						# "Specifically, we modify the architecture described in Nichol et al. (2021) by projecting and
						# adding CLIP embeddings to the existing timestep embedding, ...
						UpperCAmelCase__:	List[str]									= self.embedding_proj(lowerCamelCase__						)
						UpperCAmelCase__:	Optional[int]									= self.clip_image_embeddings_project_to_time_embeddings(lowerCamelCase__						)
						UpperCAmelCase__:	Any									= time_projected_image_embeddings + time_projected_prompt_embeds

						# ... and by projecting CLIP embeddings into four
						# extra tokens of context that are concatenated to the sequence of outputs from the GLIDE text encoder"
						UpperCAmelCase__:	Dict									= self.clip_extra_context_tokens_proj(lowerCamelCase__						)
						UpperCAmelCase__:	Optional[Any]									= clip_extra_context_tokens.reshape(lowerCamelCase__				,      -1				,      self.clip_extra_context_tokens						)
						UpperCAmelCase__:	Dict									= clip_extra_context_tokens.permute(0				,      2				,      1						)

						UpperCAmelCase__:	Any									= self.encoder_hidden_states_proj(lowerCamelCase__						)
						UpperCAmelCase__:	List[str]									= self.text_encoder_hidden_states_norm(lowerCamelCase__						)
						UpperCAmelCase__:	Optional[int]									= torch.cat([clip_extra_context_tokens, text_encoder_hidden_states]				,      dim=1						)

						return text_encoder_hidden_states, additive_clip_time_embeddings