






import unittest

from transformers import (
    MODEL_FOR_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
    TextGenerationPipeline,
    logging,
    pipeline,
)
from transformers.testing_utils import (
    CaptureLogger,
    is_pipeline_test,
    require_accelerate,
    require_tf,
    require_torch,
    require_torch_gpu,
    require_torch_or_tf,
)

from .test_pipelines_common import ANY
@is_pipeline_test
@require_torch_or_tf
class  lowerCAmelCase_       (				unittest.TestCase					):
							UpperCamelCase_		:Any		      =    MODEL_FOR_CAUSAL_LM_MAPPING
							UpperCamelCase_		:str		      =    TF_MODEL_FOR_CAUSAL_LM_MAPPING




							@require_torch
							def 						__snake_case					(   self     :   Union[str, Any]	):
											lowerCAmelCase__      			=						pipeline(task='''text-generation''' ,							model='''sshleifer/tiny-ctrl''' ,							framework='''pt'''	)
											# Using `do_sample=False` to force deterministic output
											lowerCAmelCase__      			=						text_generator('''This is a test''' ,							do_sample=SCREAMING_SNAKE_CASE_	)
											self.assertEqual(
											    SCREAMING_SNAKE_CASE_ ,							[
											        {
											            '''generated_text''': (
											                '''This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope.'''
											                ''' oscope. FiliFili@@'''
											            )
											        }
											    ] ,							)

											lowerCAmelCase__      			=						text_generator(['''This is a test''', '''This is a second test''']	)
											self.assertEqual(
											    SCREAMING_SNAKE_CASE_ ,							[
											        [
											            {
											                '''generated_text''': (
											                    '''This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope.'''
											                    ''' oscope. FiliFili@@'''
											                )
											            }
											        ],
											        [
											            {
											                '''generated_text''': (
											                    '''This is a second test ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy'''
											                    ''' oscope. oscope. FiliFili@@'''
											                )
											            }
											        ],
											    ] ,							)

											lowerCAmelCase__      			=						text_generator('''This is a test''' ,							do_sample=SCREAMING_SNAKE_CASE_ ,							num_return_sequences=2 ,							return_tensors=SCREAMING_SNAKE_CASE_	)
											self.assertEqual(
											    SCREAMING_SNAKE_CASE_ ,							[
											        {'''generated_token_ids''': ANY(SCREAMING_SNAKE_CASE_	)},
											        {'''generated_token_ids''': ANY(SCREAMING_SNAKE_CASE_	)},
											    ] ,							)
											lowerCAmelCase__      			=						text_generator.model.config.eos_token_id
											lowerCAmelCase__      			=						'''<pad>'''
											lowerCAmelCase__      			=						text_generator(
											    ['''This is a test''', '''This is a second test'''] ,							do_sample=SCREAMING_SNAKE_CASE_ ,							num_return_sequences=2 ,							batch_size=2 ,							return_tensors=SCREAMING_SNAKE_CASE_ ,							)
											self.assertEqual(
											    SCREAMING_SNAKE_CASE_ ,							[
											        [
											            {'''generated_token_ids''': ANY(SCREAMING_SNAKE_CASE_	)},
											            {'''generated_token_ids''': ANY(SCREAMING_SNAKE_CASE_	)},
											        ],
											        [
											            {'''generated_token_ids''': ANY(SCREAMING_SNAKE_CASE_	)},
											            {'''generated_token_ids''': ANY(SCREAMING_SNAKE_CASE_	)},
											        ],
											    ] ,							)




							@require_tf
							def 						__snake_case					(   self     :   Optional[Any]	):
											lowerCAmelCase__      			=						pipeline(task='''text-generation''' ,							model='''sshleifer/tiny-ctrl''' ,							framework='''tf'''	)

											# Using `do_sample=False` to force deterministic output
											lowerCAmelCase__      			=						text_generator('''This is a test''' ,							do_sample=SCREAMING_SNAKE_CASE_	)
											self.assertEqual(
											    SCREAMING_SNAKE_CASE_ ,							[
											        {
											            '''generated_text''': (
											                '''This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵'''
											                ''' please,'''
											            )
											        }
											    ] ,							)

											lowerCAmelCase__      			=						text_generator(['''This is a test''', '''This is a second test'''] ,							do_sample=SCREAMING_SNAKE_CASE_	)
											self.assertEqual(
											    SCREAMING_SNAKE_CASE_ ,							[
											        [
											            {
											                '''generated_text''': (
											                    '''This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵'''
											                    ''' please,'''
											                )
											            }
											        ],
											        [
											            {
											                '''generated_text''': (
											                    '''This is a second test Chieftain Chieftain prefecture prefecture prefecture Cannes Cannes'''
											                    ''' Cannes 閲閲Cannes Cannes Cannes 攵 please,'''
											                )
											            }
											        ],
											    ] ,							)




							def 						__snake_case					(   self     :   int ,							SCREAMING_SNAKE_CASE_     :   Optional[Any] ,							SCREAMING_SNAKE_CASE_     :   Any ,							SCREAMING_SNAKE_CASE_     :   Dict	):
											lowerCAmelCase__      			=						TextGenerationPipeline(model=SCREAMING_SNAKE_CASE_ ,							tokenizer=SCREAMING_SNAKE_CASE_	)
											return text_generator, ["This is a test", "Another test"]




							def 						__snake_case					(   self     :   Optional[Any]	):
											lowerCAmelCase__      			=						'''Hello I believe in'''
											lowerCAmelCase__      			=						pipeline('''text-generation''' ,							model='''hf-internal-testing/tiny-random-gpt2'''	)
											lowerCAmelCase__      			=						text_generator(SCREAMING_SNAKE_CASE_	)
											self.assertEqual(
											    SCREAMING_SNAKE_CASE_ ,							[{'''generated_text''': '''Hello I believe in fe fe fe fe fe fe fe fe fe fe fe fe'''}] ,							)

											lowerCAmelCase__      			=						text_generator(SCREAMING_SNAKE_CASE_ ,							stop_sequence=''' fe'''	)
											self.assertEqual(SCREAMING_SNAKE_CASE_ ,							[{'''generated_text''': '''Hello I believe in fe'''}]	)




							def 						__snake_case					(   self     :   Optional[int] ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   int	):
											lowerCAmelCase__      			=						text_generator.model
											lowerCAmelCase__      			=						text_generator.tokenizer

											lowerCAmelCase__      			=						text_generator('''This is a test'''	)
											self.assertEqual(SCREAMING_SNAKE_CASE_ ,							[{'''generated_text''': ANY(SCREAMING_SNAKE_CASE_	)}]	)
											self.assertTrue(outputs[0]['''generated_text'''].startswith('''This is a test'''	)	)

											lowerCAmelCase__      			=						text_generator('''This is a test''' ,							return_full_text=SCREAMING_SNAKE_CASE_	)
											self.assertEqual(SCREAMING_SNAKE_CASE_ ,							[{'''generated_text''': ANY(SCREAMING_SNAKE_CASE_	)}]	)
											self.assertNotIn('''This is a test''' ,							outputs[0]['''generated_text''']	)

											lowerCAmelCase__      			=						pipeline(task='''text-generation''' ,							model=SCREAMING_SNAKE_CASE_ ,							tokenizer=SCREAMING_SNAKE_CASE_ ,							return_full_text=SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						text_generator('''This is a test'''	)
											self.assertEqual(SCREAMING_SNAKE_CASE_ ,							[{'''generated_text''': ANY(SCREAMING_SNAKE_CASE_	)}]	)
											self.assertNotIn('''This is a test''' ,							outputs[0]['''generated_text''']	)

											lowerCAmelCase__      			=						text_generator('''This is a test''' ,							return_full_text=SCREAMING_SNAKE_CASE_	)
											self.assertEqual(SCREAMING_SNAKE_CASE_ ,							[{'''generated_text''': ANY(SCREAMING_SNAKE_CASE_	)}]	)
											self.assertTrue(outputs[0]['''generated_text'''].startswith('''This is a test'''	)	)

											lowerCAmelCase__      			=						text_generator(['''This is great !''', '''Something else'''] ,							num_return_sequences=2 ,							do_sample=SCREAMING_SNAKE_CASE_	)
											self.assertEqual(
											    SCREAMING_SNAKE_CASE_ ,							[
											        [{'''generated_text''': ANY(SCREAMING_SNAKE_CASE_	)}, {'''generated_text''': ANY(SCREAMING_SNAKE_CASE_	)}],
											        [{'''generated_text''': ANY(SCREAMING_SNAKE_CASE_	)}, {'''generated_text''': ANY(SCREAMING_SNAKE_CASE_	)}],
											    ] ,							)

											if text_generator.tokenizer.pad_token is not None:
															lowerCAmelCase__      			=						text_generator(
															    ['''This is great !''', '''Something else'''] ,							num_return_sequences=2 ,							batch_size=2 ,							do_sample=SCREAMING_SNAKE_CASE_	)
															self.assertEqual(
															    SCREAMING_SNAKE_CASE_ ,							[
															        [{'''generated_text''': ANY(SCREAMING_SNAKE_CASE_	)}, {'''generated_text''': ANY(SCREAMING_SNAKE_CASE_	)}],
															        [{'''generated_text''': ANY(SCREAMING_SNAKE_CASE_	)}, {'''generated_text''': ANY(SCREAMING_SNAKE_CASE_	)}],
															    ] ,							)

											with self.assertRaises(SCREAMING_SNAKE_CASE_	):
															lowerCAmelCase__      			=						text_generator('''test''' ,							return_full_text=SCREAMING_SNAKE_CASE_ ,							return_text=SCREAMING_SNAKE_CASE_	)
											with self.assertRaises(SCREAMING_SNAKE_CASE_	):
															lowerCAmelCase__      			=						text_generator('''test''' ,							return_full_text=SCREAMING_SNAKE_CASE_ ,							return_tensors=SCREAMING_SNAKE_CASE_	)
											with self.assertRaises(SCREAMING_SNAKE_CASE_	):
															lowerCAmelCase__      			=						text_generator('''test''' ,							return_text=SCREAMING_SNAKE_CASE_ ,							return_tensors=SCREAMING_SNAKE_CASE_	)

											# Empty prompt is slighly special
											# it requires BOS token to exist.
											# Special case for Pegasus which will always append EOS so will
											# work even without BOS.
											if (
											    text_generator.tokenizer.bos_token_id is not None
											    or "Pegasus" in tokenizer.__class__.__name__
											    or "Git" in model.__class__.__name__
											):
															lowerCAmelCase__      			=						text_generator(''''''	)
															self.assertEqual(SCREAMING_SNAKE_CASE_ ,							[{'''generated_text''': ANY(SCREAMING_SNAKE_CASE_	)}]	)
											else:
															with self.assertRaises((ValueError, AssertionError)	):
																			lowerCAmelCase__      			=						text_generator(''''''	)

											if text_generator.framework == "tf":
															# TF generation does not support max_new_tokens, and it's impossible
															# to control long generation with only max_length without
															# fancy calculation, dismissing tests for now.
															return
											# We don't care about infinite range models.
											# They already work.
											# Skip this test for XGLM, since it uses sinusoidal positional embeddings which are resized on-the-fly.
											lowerCAmelCase__      			=						['''RwkvForCausalLM''', '''XGLMForCausalLM''', '''GPTNeoXForCausalLM''']
											if (
											    tokenizer.model_max_length < 10_000
											    and text_generator.model.__class__.__name__ not in EXTRA_MODELS_CAN_HANDLE_LONG_INPUTS
											):
															# Handling of large generations
															with self.assertRaises((RuntimeError, IndexError, ValueError, AssertionError)	):
																			text_generator('''This is a test''' * 500 ,							max_new_tokens=20	)

															lowerCAmelCase__      			=						text_generator('''This is a test''' * 500 ,							handle_long_generation='''hole''' ,							max_new_tokens=20	)
															# Hole strategy cannot work
															with self.assertRaises(SCREAMING_SNAKE_CASE_	):
																			text_generator(
																			    '''This is a test''' * 500 ,							handle_long_generation='''hole''' ,							max_new_tokens=tokenizer.model_max_length + 10 ,							)




							@require_torch
							@require_accelerate
							@require_torch_gpu
							def 						__snake_case					(   self     :   Optional[int]	):
											import torch

											# Classic `model_kwargs`
											lowerCAmelCase__      			=						pipeline(
											    model='''hf-internal-testing/tiny-random-bloom''' ,							model_kwargs={'''device_map''': '''auto''', '''torch_dtype''': torch.bfloataa} ,							)
											self.assertEqual(pipe.model.device ,							torch.device(0	)	)
											self.assertEqual(pipe.model.lm_head.weight.dtype ,							torch.bfloataa	)
											lowerCAmelCase__      			=						pipe('''This is a test'''	)
											self.assertEqual(
											    SCREAMING_SNAKE_CASE_ ,							[
											        {
											            '''generated_text''': (
											                '''This is a test test test test test test test test test test test test test test test test'''
											                ''' test'''
											            )
											        }
											    ] ,							)

											# Upgraded those two to real pipeline arguments (they just get sent for the model as they're unlikely to mean anything else.)
											lowerCAmelCase__      			=						pipeline(model='''hf-internal-testing/tiny-random-bloom''' ,							device_map='''auto''' ,							torch_dtype=torch.bfloataa	)
											self.assertEqual(pipe.model.device ,							torch.device(0	)	)
											self.assertEqual(pipe.model.lm_head.weight.dtype ,							torch.bfloataa	)
											lowerCAmelCase__      			=						pipe('''This is a test'''	)
											self.assertEqual(
											    SCREAMING_SNAKE_CASE_ ,							[
											        {
											            '''generated_text''': (
											                '''This is a test test test test test test test test test test test test test test test test'''
											                ''' test'''
											            )
											        }
											    ] ,							)

											# torch_dtype will be automatically set to float32 if not provided - check: https://github.com/huggingface/transformers/pull/20602
											lowerCAmelCase__      			=						pipeline(model='''hf-internal-testing/tiny-random-bloom''' ,							device_map='''auto'''	)
											self.assertEqual(pipe.model.device ,							torch.device(0	)	)
											self.assertEqual(pipe.model.lm_head.weight.dtype ,							torch.floataa	)
											lowerCAmelCase__      			=						pipe('''This is a test'''	)
											self.assertEqual(
											    SCREAMING_SNAKE_CASE_ ,							[
											        {
											            '''generated_text''': (
											                '''This is a test test test test test test test test test test test test test test test test'''
											                ''' test'''
											            )
											        }
											    ] ,							)




							@require_torch
							@require_torch_gpu
							def 						__snake_case					(   self     :   int	):
											import torch

											lowerCAmelCase__      			=						pipeline(model='''hf-internal-testing/tiny-random-bloom''' ,							device=0 ,							torch_dtype=torch.floataa	)
											pipe('''This is a test'''	)




							@require_torch
							@require_accelerate
							@require_torch_gpu
							def 						__snake_case					(   self     :   List[Any]	):
											import torch

											lowerCAmelCase__      			=						pipeline(model='''hf-internal-testing/tiny-random-bloom''' ,							device_map='''auto''' ,							torch_dtype=torch.floataa	)
											pipe('''This is a test''' ,							do_sample=SCREAMING_SNAKE_CASE_ ,							top_p=0.5	)




							def 						__snake_case					(   self     :   Dict	):
											lowerCAmelCase__      			=						'''Hello world'''
											lowerCAmelCase__      			=						pipeline('''text-generation''' ,							model='''hf-internal-testing/tiny-random-gpt2'''	)
											if text_generator.model.framework == "tf":
															lowerCAmelCase__      			=						logging.get_logger('''transformers.generation.tf_utils'''	)
											else:
															lowerCAmelCase__      			=						logging.get_logger('''transformers.generation.utils'''	)
											lowerCAmelCase__      			=						'''Both `max_new_tokens`'''  # The beggining of the message to be checked in this test

											# Both are set by the user -> log warning
											with CaptureLogger(SCREAMING_SNAKE_CASE_	) as cl:
															lowerCAmelCase__      			=						text_generator(SCREAMING_SNAKE_CASE_ ,							max_length=10 ,							max_new_tokens=1	)
											self.assertIn(SCREAMING_SNAKE_CASE_ ,							cl.out	)

											# The user only sets one -> no warning
											with CaptureLogger(SCREAMING_SNAKE_CASE_	) as cl:
															lowerCAmelCase__      			=						text_generator(SCREAMING_SNAKE_CASE_ ,							max_new_tokens=1	)
											self.assertNotIn(SCREAMING_SNAKE_CASE_ ,							cl.out	)

											with CaptureLogger(SCREAMING_SNAKE_CASE_	) as cl:
															lowerCAmelCase__      			=						text_generator(SCREAMING_SNAKE_CASE_ ,							max_length=10	)
											self.assertNotIn(SCREAMING_SNAKE_CASE_ ,							cl.out	)













import unittest

from transformers import BertGenerationConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import BertGenerationDecoder, BertGenerationEncoder
class  lowerCAmelCase_       :




							def __init__(   self     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   str ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any]=13 ,							SCREAMING_SNAKE_CASE_     :   Dict=7 ,							SCREAMING_SNAKE_CASE_     :   Dict=True ,							SCREAMING_SNAKE_CASE_     :   Optional[Any]=True ,							SCREAMING_SNAKE_CASE_     :   List[str]=99 ,							SCREAMING_SNAKE_CASE_     :   str=32 ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any]=5 ,							SCREAMING_SNAKE_CASE_     :   str=4 ,							SCREAMING_SNAKE_CASE_     :   Tuple=37 ,							SCREAMING_SNAKE_CASE_     :   List[Any]="gelu" ,							SCREAMING_SNAKE_CASE_     :   Tuple=0.1 ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any]=0.1 ,							SCREAMING_SNAKE_CASE_     :   Optional[Any]=50 ,							SCREAMING_SNAKE_CASE_     :   List[str]=0.02 ,							SCREAMING_SNAKE_CASE_     :   List[str]=True ,							SCREAMING_SNAKE_CASE_     :   Optional[Any]=None ,							):
											lowerCAmelCase__      			=						parent
											lowerCAmelCase__      			=						batch_size
											lowerCAmelCase__      			=						seq_length
											lowerCAmelCase__      			=						is_training
											lowerCAmelCase__      			=						use_input_mask
											lowerCAmelCase__      			=						vocab_size
											lowerCAmelCase__      			=						hidden_size
											lowerCAmelCase__      			=						num_hidden_layers
											lowerCAmelCase__      			=						num_attention_heads
											lowerCAmelCase__      			=						intermediate_size
											lowerCAmelCase__      			=						hidden_act
											lowerCAmelCase__      			=						hidden_dropout_prob
											lowerCAmelCase__      			=						attention_probs_dropout_prob
											lowerCAmelCase__      			=						max_position_embeddings
											lowerCAmelCase__      			=						initializer_range
											lowerCAmelCase__      			=						use_labels
											lowerCAmelCase__      			=						scope




							def 						__snake_case					(   self     :   Union[str, Any]	):
											lowerCAmelCase__      			=						ids_tensor([self.batch_size, self.seq_length] ,							self.vocab_size	)

											lowerCAmelCase__      			=						None
											if self.use_input_mask:
															lowerCAmelCase__      			=						random_attention_mask([self.batch_size, self.seq_length]	)

											if self.use_labels:
															lowerCAmelCase__      			=						ids_tensor([self.batch_size, self.seq_length] ,							self.vocab_size	)

											lowerCAmelCase__      			=						self.get_config()

											return config, input_ids, input_mask, token_labels




							def 						__snake_case					(   self     :   List[str]	):
											return BertGenerationConfig(
											    vocab_size=self.vocab_size ,							hidden_size=self.hidden_size ,							num_hidden_layers=self.num_hidden_layers ,							num_attention_heads=self.num_attention_heads ,							intermediate_size=self.intermediate_size ,							hidden_act=self.hidden_act ,							hidden_dropout_prob=self.hidden_dropout_prob ,							attention_probs_dropout_prob=self.attention_probs_dropout_prob ,							max_position_embeddings=self.max_position_embeddings ,							is_decoder=SCREAMING_SNAKE_CASE_ ,							initializer_range=self.initializer_range ,							)




							def 						__snake_case					(   self     :   str	):
											(
											    (
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		
											)      			=						self.prepare_config_and_inputs()

											lowerCAmelCase__      			=						True
											lowerCAmelCase__      			=						floats_tensor([self.batch_size, self.seq_length, self.hidden_size]	)
											lowerCAmelCase__      			=						ids_tensor([self.batch_size, self.seq_length] ,							vocab_size=2	)

											return (
											    config,
											    input_ids,
											    input_mask,
											    token_labels,
											    encoder_hidden_states,
											    encoder_attention_mask,
											)




							def 						__snake_case					(   self     :   Union[str, Any] ,							SCREAMING_SNAKE_CASE_     :   List[str] ,							SCREAMING_SNAKE_CASE_     :   str ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							**SCREAMING_SNAKE_CASE_     :   Union[str, Any] ,							):
											lowerCAmelCase__      			=						BertGenerationEncoder(config=SCREAMING_SNAKE_CASE_	)
											model.to(SCREAMING_SNAKE_CASE_	)
											model.eval()
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)
											self.parent.assertEqual(result.last_hidden_state.shape ,							(self.batch_size, self.seq_length, self.hidden_size)	)




							def 						__snake_case					(   self     :   Dict ,							SCREAMING_SNAKE_CASE_     :   Any ,							SCREAMING_SNAKE_CASE_     :   Any ,							SCREAMING_SNAKE_CASE_     :   Optional[int] ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any] ,							SCREAMING_SNAKE_CASE_     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   Optional[int] ,							**SCREAMING_SNAKE_CASE_     :   str ,							):
											lowerCAmelCase__      			=						True
											lowerCAmelCase__      			=						BertGenerationEncoder(config=SCREAMING_SNAKE_CASE_	)
											model.to(SCREAMING_SNAKE_CASE_	)
											model.eval()
											lowerCAmelCase__      			=						model(
											    SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_ ,							encoder_hidden_states=SCREAMING_SNAKE_CASE_ ,							encoder_attention_mask=SCREAMING_SNAKE_CASE_ ,							)
											lowerCAmelCase__      			=						model(
											    SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_ ,							encoder_hidden_states=SCREAMING_SNAKE_CASE_ ,							)
											self.parent.assertEqual(result.last_hidden_state.shape ,							(self.batch_size, self.seq_length, self.hidden_size)	)




							def 						__snake_case					(   self     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any] ,							SCREAMING_SNAKE_CASE_     :   Dict ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   Optional[int] ,							**SCREAMING_SNAKE_CASE_     :   Optional[Any] ,							):
											lowerCAmelCase__      			=						True
											lowerCAmelCase__      			=						True
											lowerCAmelCase__      			=						BertGenerationDecoder(config=SCREAMING_SNAKE_CASE_	).to(SCREAMING_SNAKE_CASE_	).eval()

											# first forward pass
											lowerCAmelCase__      			=						model(
											    SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_ ,							encoder_hidden_states=SCREAMING_SNAKE_CASE_ ,							encoder_attention_mask=SCREAMING_SNAKE_CASE_ ,							use_cache=SCREAMING_SNAKE_CASE_ ,							)
											lowerCAmelCase__      			=						outputs.past_key_values

											# create hypothetical multiple next token and extent to next_input_ids
											lowerCAmelCase__      			=						ids_tensor((self.batch_size, 3) ,							config.vocab_size	)
											lowerCAmelCase__      			=						ids_tensor((self.batch_size, 3) ,							vocab_size=2	)

											# append to next input_ids and
											lowerCAmelCase__      			=						torch.cat([input_ids, next_tokens] ,							dim=-1	)
											lowerCAmelCase__      			=						torch.cat([input_mask, next_mask] ,							dim=-1	)

											lowerCAmelCase__      			=						model(
											    SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_ ,							encoder_hidden_states=SCREAMING_SNAKE_CASE_ ,							encoder_attention_mask=SCREAMING_SNAKE_CASE_ ,							output_hidden_states=SCREAMING_SNAKE_CASE_ ,							)['''hidden_states'''][0]
											lowerCAmelCase__      			=						model(
											    SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_ ,							encoder_hidden_states=SCREAMING_SNAKE_CASE_ ,							encoder_attention_mask=SCREAMING_SNAKE_CASE_ ,							past_key_values=SCREAMING_SNAKE_CASE_ ,							output_hidden_states=SCREAMING_SNAKE_CASE_ ,							)['''hidden_states'''][0]

											# select random slice
											lowerCAmelCase__      			=						ids_tensor((1,) ,							output_from_past.shape[-1]	).item()
											lowerCAmelCase__      			=						output_from_no_past[:, -3:, random_slice_idx].detach()
											lowerCAmelCase__      			=						output_from_past[:, :, random_slice_idx].detach()

											self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]	)

											# test that outputs are equal for slice
											self.parent.assertTrue(torch.allclose(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							atol=1e-3	)	)




							def 						__snake_case					(   self     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   Any ,							SCREAMING_SNAKE_CASE_     :   int ,							SCREAMING_SNAKE_CASE_     :   Optional[Any] ,							SCREAMING_SNAKE_CASE_     :   Optional[Any] ,							*SCREAMING_SNAKE_CASE_     :   Tuple ,							):
											lowerCAmelCase__      			=						BertGenerationDecoder(SCREAMING_SNAKE_CASE_	)
											model.to(SCREAMING_SNAKE_CASE_	)
											model.eval()
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_ ,							labels=SCREAMING_SNAKE_CASE_	)
											self.parent.assertEqual(result.logits.shape ,							(self.batch_size, self.seq_length, self.vocab_size)	)




							def 						__snake_case					(   self     :   List[Any]	):
											lowerCAmelCase__			,		lowerCAmelCase__			,		lowerCAmelCase__			,		lowerCAmelCase__      			=						self.prepare_config_and_inputs()
											lowerCAmelCase__      			=						{'''input_ids''': input_ids, '''attention_mask''': input_mask}
											return config, inputs_dict



@require_torch
class  lowerCAmelCase_       (				snake_case__     ,    snake_case__     ,    snake_case__     ,    unittest.TestCase					):
							UpperCamelCase_		:Dict		      =    (BertGenerationEncoder, BertGenerationDecoder) if is_torch_available() else ()
							UpperCamelCase_		:str		      =    (BertGenerationDecoder,) if is_torch_available() else ()
							UpperCamelCase_		:List[str]		      =    (
							    {'feature-extraction': BertGenerationEncoder, 'text-generation': BertGenerationDecoder}
							    if is_torch_available()
							    else {}
							)




							def 						__snake_case					(   self     :   List[str]	):
											lowerCAmelCase__      			=						BertGenerationEncoderTester(self	)
											lowerCAmelCase__      			=						ConfigTester(self ,							config_class=SCREAMING_SNAKE_CASE_ ,							hidden_size=37	)




							def 						__snake_case					(   self     :   Optional[Any]	):
											self.config_tester.run_common_tests()




							def 						__snake_case					(   self     :   List[Any]	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   str	):
											lowerCAmelCase__			,		lowerCAmelCase__			,		lowerCAmelCase__			,		lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs()
											lowerCAmelCase__      			=						'''bert'''
											self.model_tester.create_and_check_model(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   int	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs_for_decoder()
											self.model_tester.create_and_check_model_as_decoder(*SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   List[Any]	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs_for_decoder()
											self.model_tester.create_and_check_decoder_model_past_large_inputs(*SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   Any	):
											# This regression test was failing with PyTorch < 1.3
											(
											    (
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		
											)      			=						self.model_tester.prepare_config_and_inputs_for_decoder()

											lowerCAmelCase__      			=						None

											self.model_tester.create_and_check_model_as_decoder(
											    SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							)




							def 						__snake_case					(   self     :   int	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs_for_decoder()
											self.model_tester.create_and_check_for_causal_lm(*SCREAMING_SNAKE_CASE_	)




							@slow
							def 						__snake_case					(   self     :   str	):
											lowerCAmelCase__      			=						BertGenerationEncoder.from_pretrained('''google/bert_for_seq_generation_L-24_bbc_encoder'''	)
											self.assertIsNotNone(SCREAMING_SNAKE_CASE_	)



@require_torch
class  lowerCAmelCase_       (				unittest.TestCase					):




							@slow
							def 						__snake_case					(   self     :   Tuple	):
											lowerCAmelCase__      			=						BertGenerationEncoder.from_pretrained('''google/bert_for_seq_generation_L-24_bbc_encoder'''	)
											lowerCAmelCase__      			=						torch.tensor([[101, 7_592, 1_010, 2_026, 3_899, 2_003, 10_140, 102]]	)
											with torch.no_grad():
															lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)[0]
											lowerCAmelCase__      			=						torch.Size([1, 8, 1_024]	)
											self.assertEqual(output.shape ,							SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						torch.tensor(
											    [[[0.1_775, 0.0_083, -0.0_321], [1.6_002, 0.1_287, 0.3_912], [2.1_473, 0.5_791, 0.6_066]]]	)
											self.assertTrue(torch.allclose(output[:, :3, :3] ,							SCREAMING_SNAKE_CASE_ ,							atol=1e-4	)	)



@require_torch
class  lowerCAmelCase_       (				unittest.TestCase					):




							@slow
							def 						__snake_case					(   self     :   Any	):
											lowerCAmelCase__      			=						BertGenerationDecoder.from_pretrained('''google/bert_for_seq_generation_L-24_bbc_encoder'''	)
											lowerCAmelCase__      			=						torch.tensor([[101, 7_592, 1_010, 2_026, 3_899, 2_003, 10_140, 102]]	)
											with torch.no_grad():
															lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)[0]
											lowerCAmelCase__      			=						torch.Size([1, 8, 50_358]	)
											self.assertEqual(output.shape ,							SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						torch.tensor(
											    [[[-0.5_788, -2.5_994, -3.7_054], [0.0_438, 4.7_997, 1.8_795], [1.5_862, 6.6_409, 4.4_638]]]	)
											self.assertTrue(torch.allclose(output[:, :3, :3] ,							SCREAMING_SNAKE_CASE_ ,							atol=1e-4	)	)





