



"""simple docstring"""





import flax.linen as nn
import jax.numpy as jnp

from .attention_flax import FlaxTransformeraDModel
from .resnet_flax import FlaxDownsampleaD, FlaxResnetBlockaD, FlaxUpsampleaD



class  _lowerCAmelCase     (					nn.Module		):





		"""simple docstring"""






		_lowerCamelCase      	=							42
		_lowerCamelCase      	=							42
		_lowerCamelCase      	=							0.0
		_lowerCamelCase      	=							1
		_lowerCamelCase      	=							1
		_lowerCamelCase      	=							True
		_lowerCamelCase      	=							False
		_lowerCamelCase      	=							False
		_lowerCamelCase      	=							False
		_lowerCamelCase      	=							jnp.floataa





		def  UpperCAmelCase__				(       self )			->    Tuple:







								'''simple docstring'''

								snake_case_ :			Optional[int]            =      []
								snake_case_ :			Optional[int]            =      []

								for i in range(self.num_layers ):
														snake_case_ :			List[str]            =      self.in_channels if i == 0 else self.out_channels

														snake_case_ :			Optional[int]            =      FlaxResnetBlockaD(
														    in_channels=_lowercase				,     out_channels=self.out_channels				,     dropout_prob=self.dropout				,     dtype=self.dtype				,     )
														resnets.append(_lowercase )

														snake_case_ :			Any            =      FlaxTransformeraDModel(
														    in_channels=self.out_channels				,     n_heads=self.num_attention_heads				,     d_head=self.out_channels // self.num_attention_heads				,     depth=1				,     use_linear_projection=self.use_linear_projection				,     only_cross_attention=self.only_cross_attention				,     use_memory_efficient_attention=self.use_memory_efficient_attention				,     dtype=self.dtype				,     )
														attentions.append(_lowercase )

								snake_case_ :			Any            =      resnets
								snake_case_ :			Any            =      attentions

								if self.add_downsample:
														snake_case_ :			Any            =      FlaxDownsampleaD(self.out_channels				,     dtype=self.dtype )





		def __call__(       self				,     _lowercase				,     _lowercase				,     _lowercase				,     _lowercase=True )			->    Optional[int]:







								'''simple docstring'''

								snake_case_ :			Union[str, Any]            =      ()

								for resnet, attn in zip(self.resnets				,     self.attentions ):
														snake_case_ :			List[Any]            =      resnet(_lowercase				,     _lowercase				,     deterministic=_lowercase )
														snake_case_ :			Optional[int]            =      attn(_lowercase				,     _lowercase				,     deterministic=_lowercase )
														output_states += (hidden_states,)

								if self.add_downsample:
														snake_case_ :			List[Any]            =      self.downsamplers_a(_lowercase )
														output_states += (hidden_states,)

								return hidden_states, output_states




class  _lowerCAmelCase     (					nn.Module		):





		"""simple docstring"""






		_lowerCamelCase      	=							42
		_lowerCamelCase      	=							42
		_lowerCamelCase      	=							0.0
		_lowerCamelCase      	=							1
		_lowerCamelCase      	=							True
		_lowerCamelCase      	=							jnp.floataa





		def  UpperCAmelCase__				(       self )			->    Optional[Any]:







								'''simple docstring'''

								snake_case_ :			int            =      []

								for i in range(self.num_layers ):
														snake_case_ :			int            =      self.in_channels if i == 0 else self.out_channels

														snake_case_ :			Optional[Any]            =      FlaxResnetBlockaD(
														    in_channels=_lowercase				,     out_channels=self.out_channels				,     dropout_prob=self.dropout				,     dtype=self.dtype				,     )
														resnets.append(_lowercase )
								snake_case_ :			Dict            =      resnets

								if self.add_downsample:
														snake_case_ :			Any            =      FlaxDownsampleaD(self.out_channels				,     dtype=self.dtype )





		def __call__(       self				,     _lowercase				,     _lowercase				,     _lowercase=True )			->    List[Any]:







								'''simple docstring'''

								snake_case_ :			Dict            =      ()

								for resnet in self.resnets:
														snake_case_ :			List[Any]            =      resnet(_lowercase				,     _lowercase				,     deterministic=_lowercase )
														output_states += (hidden_states,)

								if self.add_downsample:
														snake_case_ :			Union[str, Any]            =      self.downsamplers_a(_lowercase )
														output_states += (hidden_states,)

								return hidden_states, output_states




class  _lowerCAmelCase     (					nn.Module		):





		"""simple docstring"""






		_lowerCamelCase      	=							42
		_lowerCamelCase      	=							42
		_lowerCamelCase      	=							42
		_lowerCamelCase      	=							0.0
		_lowerCamelCase      	=							1
		_lowerCamelCase      	=							1
		_lowerCamelCase      	=							True
		_lowerCamelCase      	=							False
		_lowerCamelCase      	=							False
		_lowerCamelCase      	=							False
		_lowerCamelCase      	=							jnp.floataa





		def  UpperCAmelCase__				(       self )			->    Optional[int]:







								'''simple docstring'''

								snake_case_ :			List[Any]            =      []
								snake_case_ :			Tuple            =      []

								for i in range(self.num_layers ):
														snake_case_ :			Dict            =      self.in_channels if (i == self.num_layers - 1) else self.out_channels
														snake_case_ :			Tuple            =      self.prev_output_channel if i == 0 else self.out_channels

														snake_case_ :			str            =      FlaxResnetBlockaD(
														    in_channels=resnet_in_channels + res_skip_channels				,     out_channels=self.out_channels				,     dropout_prob=self.dropout				,     dtype=self.dtype				,     )
														resnets.append(_lowercase )

														snake_case_ :			Optional[int]            =      FlaxTransformeraDModel(
														    in_channels=self.out_channels				,     n_heads=self.num_attention_heads				,     d_head=self.out_channels // self.num_attention_heads				,     depth=1				,     use_linear_projection=self.use_linear_projection				,     only_cross_attention=self.only_cross_attention				,     use_memory_efficient_attention=self.use_memory_efficient_attention				,     dtype=self.dtype				,     )
														attentions.append(_lowercase )

								snake_case_ :			Union[str, Any]            =      resnets
								snake_case_ :			Any            =      attentions

								if self.add_upsample:
														snake_case_ :			Tuple            =      FlaxUpsampleaD(self.out_channels				,     dtype=self.dtype )





		def __call__(       self				,     _lowercase				,     _lowercase				,     _lowercase				,     _lowercase				,     _lowercase=True )			->    Union[str, Any]:







								'''simple docstring'''

								for resnet, attn in zip(self.resnets				,     self.attentions ):
														# pop res hidden states
														snake_case_ :			Union[str, Any]            =      res_hidden_states_tuple[-1]
														snake_case_ :			Optional[Any]            =      res_hidden_states_tuple[:-1]
														snake_case_ :			List[Any]            =      jnp.concatenate((hidden_states, res_hidden_states)				,     axis=-1 )

														snake_case_ :			List[str]            =      resnet(_lowercase				,     _lowercase				,     deterministic=_lowercase )
														snake_case_ :			int            =      attn(_lowercase				,     _lowercase				,     deterministic=_lowercase )

								if self.add_upsample:
														snake_case_ :			List[Any]            =      self.upsamplers_a(_lowercase )

								return hidden_states




class  _lowerCAmelCase     (					nn.Module		):





		"""simple docstring"""






		_lowerCamelCase      	=							42
		_lowerCamelCase      	=							42
		_lowerCamelCase      	=							42
		_lowerCamelCase      	=							0.0
		_lowerCamelCase      	=							1
		_lowerCamelCase      	=							True
		_lowerCamelCase      	=							jnp.floataa





		def  UpperCAmelCase__				(       self )			->    Any:







								'''simple docstring'''

								snake_case_ :			Union[str, Any]            =      []

								for i in range(self.num_layers ):
														snake_case_ :			int            =      self.in_channels if (i == self.num_layers - 1) else self.out_channels
														snake_case_ :			List[str]            =      self.prev_output_channel if i == 0 else self.out_channels

														snake_case_ :			Optional[int]            =      FlaxResnetBlockaD(
														    in_channels=resnet_in_channels + res_skip_channels				,     out_channels=self.out_channels				,     dropout_prob=self.dropout				,     dtype=self.dtype				,     )
														resnets.append(_lowercase )

								snake_case_ :			Optional[int]            =      resnets

								if self.add_upsample:
														snake_case_ :			str            =      FlaxUpsampleaD(self.out_channels				,     dtype=self.dtype )





		def __call__(       self				,     _lowercase				,     _lowercase				,     _lowercase				,     _lowercase=True )			->    Optional[int]:







								'''simple docstring'''

								for resnet in self.resnets:
														# pop res hidden states
														snake_case_ :			Optional[Any]            =      res_hidden_states_tuple[-1]
														snake_case_ :			List[str]            =      res_hidden_states_tuple[:-1]
														snake_case_ :			str            =      jnp.concatenate((hidden_states, res_hidden_states)				,     axis=-1 )

														snake_case_ :			Optional[int]            =      resnet(_lowercase				,     _lowercase				,     deterministic=_lowercase )

								if self.add_upsample:
														snake_case_ :			int            =      self.upsamplers_a(_lowercase )

								return hidden_states




class  _lowerCAmelCase     (					nn.Module		):





		"""simple docstring"""






		_lowerCamelCase      	=							42
		_lowerCamelCase      	=							0.0
		_lowerCamelCase      	=							1
		_lowerCamelCase      	=							1
		_lowerCamelCase      	=							False
		_lowerCamelCase      	=							False
		_lowerCamelCase      	=							jnp.floataa





		def  UpperCAmelCase__				(       self )			->    Tuple:







								'''simple docstring'''

								snake_case_ :			Any            =      [
								    FlaxResnetBlockaD(
								        in_channels=self.in_channels				,     out_channels=self.in_channels				,     dropout_prob=self.dropout				,     dtype=self.dtype				,     )
								]

								snake_case_ :			Tuple            =      []

								for _ in range(self.num_layers ):
														snake_case_ :			List[Any]            =      FlaxTransformeraDModel(
														    in_channels=self.in_channels				,     n_heads=self.num_attention_heads				,     d_head=self.in_channels // self.num_attention_heads				,     depth=1				,     use_linear_projection=self.use_linear_projection				,     use_memory_efficient_attention=self.use_memory_efficient_attention				,     dtype=self.dtype				,     )
														attentions.append(_lowercase )

														snake_case_ :			List[str]            =      FlaxResnetBlockaD(
														    in_channels=self.in_channels				,     out_channels=self.in_channels				,     dropout_prob=self.dropout				,     dtype=self.dtype				,     )
														resnets.append(_lowercase )

								snake_case_ :			Optional[Any]            =      resnets
								snake_case_ :			List[Any]            =      attentions





		def __call__(       self				,     _lowercase				,     _lowercase				,     _lowercase				,     _lowercase=True )			->    Union[str, Any]:







								'''simple docstring'''

								snake_case_ :			Any            =      self.resnets[0](_lowercase				,     _lowercase )
								for attn, resnet in zip(self.attentions				,     self.resnets[1:] ):
														snake_case_ :			Dict            =      attn(_lowercase				,     _lowercase				,     deterministic=_lowercase )
														snake_case_ :			Optional[int]            =      resnet(_lowercase				,     _lowercase				,     deterministic=_lowercase )

								return hidden_states







from __future__ import annotations

import unittest

from transformers import XGLMConfig, XGLMTokenizer, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
						import tensorflow as tf

						from transformers.models.xglm.modeling_tf_xglm import (
						    TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFXGLMForCausalLM,
						    TFXGLMModel,
						)







@require_tf
class 				lowerCAmelCase_       :
				__a     :       List[Any]						=     XGLMConfig
				__a     :       List[Any]						=     {}
				__a     :       List[str]						=     "gelu"






				def __init__(			self      ,snake_case__      ,snake_case__=14      ,snake_case__=7      ,snake_case__=True      ,snake_case__=True      ,snake_case__=True      ,snake_case__=99      ,snake_case__=32      ,snake_case__=2      ,snake_case__=4      ,snake_case__=37      ,snake_case__="gelu"      ,snake_case__=0.1      ,snake_case__=0.1      ,snake_case__=512      ,snake_case__=0.02      ,):
							SCREAMING_SNAKE_CASE_   : int					  =      parent
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      batch_size
							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      seq_length
							SCREAMING_SNAKE_CASE_   : List[str]					  =      is_training
							SCREAMING_SNAKE_CASE_   : int					  =      use_input_mask
							SCREAMING_SNAKE_CASE_   : Tuple					  =      use_labels
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      vocab_size
							SCREAMING_SNAKE_CASE_   : int					  =      d_model
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      num_hidden_layers
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      num_attention_heads
							SCREAMING_SNAKE_CASE_   : List[str]					  =      ffn_dim
							SCREAMING_SNAKE_CASE_   : Any					  =      activation_function
							SCREAMING_SNAKE_CASE_   : Dict					  =      activation_dropout
							SCREAMING_SNAKE_CASE_   : int					  =      attention_dropout
							SCREAMING_SNAKE_CASE_   : str					  =      max_position_embeddings
							SCREAMING_SNAKE_CASE_   : Any					  =      initializer_range
							SCREAMING_SNAKE_CASE_   : int					  =      None
							SCREAMING_SNAKE_CASE_   : str					  =      0
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      2
							SCREAMING_SNAKE_CASE_   : Any					  =      1






				def 		snake_case	(			self ):
							return XGLMConfig.from_pretrained('facebook/xglm-564M' )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      tf.clip_by_value(
							    ids_tensor([self.batch_size, self.seq_length]      ,self.vocab_size )      ,clip_value_min=0      ,clip_value_max=3 )

							SCREAMING_SNAKE_CASE_   : List[str]					  =      None
							if self.use_input_mask:
										SCREAMING_SNAKE_CASE_   : List[str]					  =      random_attention_mask([self.batch_size, self.seq_length] )

							SCREAMING_SNAKE_CASE_   : List[Any]					  =      self.get_config()

							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      floats_tensor([self.num_hidden_layers, self.num_attention_heads]      ,2 )

							return (
							    config,
							    input_ids,
							    input_mask,
							    head_mask,
							)






				def 		snake_case	(			self ):
							return XGLMConfig(
							    vocab_size=self.vocab_size      ,d_model=self.hidden_size      ,num_layers=self.num_hidden_layers      ,attention_heads=self.num_attention_heads      ,ffn_dim=self.ffn_dim      ,activation_function=self.activation_function      ,activation_dropout=self.activation_dropout      ,attention_dropout=self.attention_dropout      ,max_position_embeddings=self.max_position_embeddings      ,initializer_range=self.initializer_range      ,use_cache=snake_case__      ,bos_token_id=self.bos_token_id      ,eos_token_id=self.eos_token_id      ,pad_token_id=self.pad_token_id      ,return_dict=snake_case__      ,)






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : str					  =      self.prepare_config_and_inputs()

							(
							    (
							    SCREAMING_SNAKE_CASE_
							)  ,    (
							    SCREAMING_SNAKE_CASE_
							)  ,    (
							    SCREAMING_SNAKE_CASE_
							)  ,    (
							    SCREAMING_SNAKE_CASE_
							)  ,    
							)   : List[Any]					  =      config_and_inputs

							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      {
							    'input_ids': input_ids,
							    'head_mask': head_mask,
							}

							return config, inputs_dict







@require_tf
class 				lowerCAmelCase_       (					lowerCamelCase_						,     lowerCamelCase_						,     unittest.TestCase							):
				__a     :       Dict						=     (TFXGLMModel, TFXGLMForCausalLM) if is_tf_available() else ()
				__a     :       Dict						=     (TFXGLMForCausalLM,) if is_tf_available() else ()
				__a     :       Union[str, Any]						=     (
				    {"feature-extraction": TFXGLMModel, "text-generation": TFXGLMForCausalLM} if is_tf_available() else {}
				)
				__a     :       Dict						=     False
				__a     :       Union[str, Any]						=     False
				__a     :       List[str]						=     False






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : int					  =      TFXGLMModelTester(self )
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      ConfigTester(self      ,config_class=snake_case__      ,n_embd=37 )






				def 		snake_case	(			self ):
							self.config_tester.run_common_tests()






				@slow
				def 		snake_case	(			self ):
							for model_name in TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
										SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      TFXGLMModel.from_pretrained(snake_case__ )
										self.assertIsNotNone(snake_case__ )






				@unittest.skip(reason='Currently, model embeddings are going to undergo a major refactor.' )
				def 		snake_case	(			self ):
							super().test_resize_token_embeddings()







@require_tf
class 				lowerCAmelCase_       (					unittest.TestCase							):






				@slow
				def 		snake_case	(			self      ,snake_case__=True ):
							SCREAMING_SNAKE_CASE_   : Dict					  =      TFXGLMForCausalLM.from_pretrained('facebook/xglm-564M' )
							SCREAMING_SNAKE_CASE_   : Any					  =      tf.convert_to_tensor([[2, 268, 9865]]      ,dtype=tf.intaa )  # The dog
							# </s> The dog is a very friendly dog. He is very affectionate and loves to play with other
							# fmt: off
							SCREAMING_SNAKE_CASE_   : Tuple					  =      [2, 268, 9865, 67, 11, 1988, 57252, 9865, 5, 984, 67, 1988, 213838, 1658, 53, 70446, 33, 6657, 278, 1581]
							# fmt: on
							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      model.generate(snake_case__      ,do_sample=snake_case__      ,num_beams=1 )
							if verify_outputs:
										self.assertListEqual(output_ids[0].numpy().tolist()      ,snake_case__ )






				@slow
				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Tuple					  =      XGLMTokenizer.from_pretrained('facebook/xglm-564M' )
							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      TFXGLMForCausalLM.from_pretrained('facebook/xglm-564M' )

							tf.random.set_seed(0 )
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      tokenizer('Today is a nice day and'      ,return_tensors='tf' )
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      tokenized.input_ids
							# forces the generation to happen on CPU, to avoid GPU-related quirks (and assure same output regardless of the available devices)
							with tf.device(':/CPU:0' ):
										SCREAMING_SNAKE_CASE_   : List[Any]					  =      model.generate(snake_case__      ,do_sample=snake_case__      ,seed=[7, 0] )
							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      tokenizer.decode(output_ids[0]      ,skip_special_tokens=snake_case__ )

							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      (
							    'Today is a nice day and warm evening here over Southern Alberta!! Today when they closed schools due'
							)
							self.assertEqual(snake_case__      ,snake_case__ )






				@slow
				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      TFXGLMForCausalLM.from_pretrained('facebook/xglm-564M' )
							SCREAMING_SNAKE_CASE_   : Tuple					  =      XGLMTokenizer.from_pretrained('facebook/xglm-564M' )

							SCREAMING_SNAKE_CASE_   : List[Any]					  =      'left'

							# use different length sentences to test batching
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      [
							    'This is an extremelly long sentence that only exists to test the ability of the model to cope with '
							    'left-padding, such as in batched generation. The output for the sequence below should be the same '
							    'regardless of whether left padding is applied or not. When',
							    'Hello, my dog is a little',
							]

							SCREAMING_SNAKE_CASE_   : Any					  =      tokenizer(snake_case__      ,return_tensors='tf'      ,padding=snake_case__ )
							SCREAMING_SNAKE_CASE_   : int					  =      inputs['input_ids']

							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      model.generate(input_ids=snake_case__      ,attention_mask=inputs['attention_mask']      ,max_new_tokens=12 )

							SCREAMING_SNAKE_CASE_   : List[str]					  =      tokenizer(sentences[0]      ,return_tensors='tf' ).input_ids
							SCREAMING_SNAKE_CASE_   : Any					  =      model.generate(input_ids=snake_case__      ,max_new_tokens=12 )

							SCREAMING_SNAKE_CASE_   : List[str]					  =      tokenizer(sentences[1]      ,return_tensors='tf' ).input_ids
							SCREAMING_SNAKE_CASE_   : Any					  =      model.generate(input_ids=snake_case__      ,max_new_tokens=12 )

							SCREAMING_SNAKE_CASE_   : List[str]					  =      tokenizer.batch_decode(snake_case__      ,skip_special_tokens=snake_case__ )
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      tokenizer.decode(output_non_padded[0]      ,skip_special_tokens=snake_case__ )
							SCREAMING_SNAKE_CASE_   : Tuple					  =      tokenizer.decode(output_padded[0]      ,skip_special_tokens=snake_case__ )

							SCREAMING_SNAKE_CASE_   : str					  =      [
							    'This is an extremelly long sentence that only exists to test the ability of the model to cope with '
							    'left-padding, such as in batched generation. The output for the sequence below should be the same '
							    'regardless of whether left padding is applied or not. When left padding is applied, the sequence will be '
							    'a single',
							    'Hello, my dog is a little bit of a shy one, but he is very friendly',
							]
							self.assertListEqual(snake_case__      ,snake_case__ )
							self.assertListEqual(snake_case__      ,[non_padded_sentence, padded_sentence] )



