



import unittest

from transformers import AutoTokenizer, NystromformerConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    NystromformerForMaskedLM,
				    NystromformerForMultipleChoice,
				    NystromformerForQuestionAnswering,
				    NystromformerForSequenceClassification,
				    NystromformerForTokenClassification,
				    NystromformerModel,
				)
				from transformers.models.nystromformer.modeling_nystromformer import NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST



class        lowerCAmelCase__					:


				def __init__(							self      :  Optional[int]						, __UpperCamelCase      :  Optional[int]						, __UpperCamelCase      :  int=13						, __UpperCamelCase      :  List[Any]=7						, __UpperCamelCase      :  Any=True						, __UpperCamelCase      :  Tuple=True						, __UpperCamelCase      :  int=True						, __UpperCamelCase      :  List[Any]=True						, __UpperCamelCase      :  Optional[int]=99						, __UpperCamelCase      :  str=32						, __UpperCamelCase      :  Tuple=5						, __UpperCamelCase      :  str=4						, __UpperCamelCase      :  Any=37						, __UpperCamelCase      :  Tuple="gelu"						, __UpperCamelCase      :  Optional[int]=0.1						, __UpperCamelCase      :  str=0.1						, __UpperCamelCase      :  Optional[Any]=512						, __UpperCamelCase      :  List[str]=16						, __UpperCamelCase      :  str=2						, __UpperCamelCase      :  Dict=0.0_2						, __UpperCamelCase      :  Optional[Any]=3						, __UpperCamelCase      :  int=4						, __UpperCamelCase      :  Any=None						, )		->     List[Any]:
											A      						=    parent
											A      						=    batch_size
											A      						=    seq_length
											A      						=    is_training
											A      						=    use_input_mask
											A      						=    use_token_type_ids
											A      						=    use_labels
											A      						=    vocab_size
											A      						=    hidden_size
											A      						=    num_hidden_layers
											A      						=    num_attention_heads
											A      						=    intermediate_size
											A      						=    hidden_act
											A      						=    hidden_dropout_prob
											A      						=    attention_probs_dropout_prob
											A      						=    max_position_embeddings
											A      						=    type_vocab_size
											A      						=    type_sequence_label_size
											A      						=    initializer_range
											A      						=    num_labels
											A      						=    num_choices
											A      						=    scope


				def __UpperCamelCase						(							self      :  int       )		->     int:
											A      						=    ids_tensor([self.batch_size, self.seq_length]						, self.vocab_size       )

											A      						=    None
											if self.use_input_mask:
																		A      						=    random_attention_mask([self.batch_size, self.seq_length]       )

											A      						=    None
											if self.use_token_type_ids:
																		A      						=    ids_tensor([self.batch_size, self.seq_length]						, self.type_vocab_size       )

											A      						=    None
											A      						=    None
											A      						=    None
											if self.use_labels:
																		A      						=    ids_tensor([self.batch_size]						, self.type_sequence_label_size       )
																		A      						=    ids_tensor([self.batch_size, self.seq_length]						, self.num_labels       )
																		A      						=    ids_tensor([self.batch_size]						, self.num_choices       )

											A      						=    self.get_config()

											return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


				def __UpperCamelCase						(							self      :  Optional[int]       )		->     Any:
											return NystromformerConfig(
											    vocab_size=self.vocab_size						, hidden_size=self.hidden_size						, num_hidden_layers=self.num_hidden_layers						, num_attention_heads=self.num_attention_heads						, intermediate_size=self.intermediate_size						, hidden_act=self.hidden_act						, hidden_dropout_prob=self.hidden_dropout_prob						, attention_probs_dropout_prob=self.attention_probs_dropout_prob						, max_position_embeddings=self.max_position_embeddings						, type_vocab_size=self.type_vocab_size						, is_decoder=__UpperCamelCase						, initializer_range=self.initializer_range						, )


				def __UpperCamelCase						(							self      :  Any						, __UpperCamelCase      :  Any						, __UpperCamelCase      :  List[str]						, __UpperCamelCase      :  Union[str, Any]						, __UpperCamelCase      :  Optional[Any]						, __UpperCamelCase      :  str						, __UpperCamelCase      :  Optional[int]						, __UpperCamelCase      :  Optional[Any]       )		->     str:
											A      						=    NystromformerModel(config=__UpperCamelCase       )
											model.to(__UpperCamelCase       )
											model.eval()
											A      						=    model(__UpperCamelCase						, attention_mask=__UpperCamelCase						, token_type_ids=__UpperCamelCase       )
											A      						=    model(__UpperCamelCase						, token_type_ids=__UpperCamelCase       )
											A      						=    model(__UpperCamelCase       )
											self.parent.assertEqual(result.last_hidden_state.shape						, (self.batch_size, self.seq_length, self.hidden_size)       )


				def __UpperCamelCase						(							self      :  List[Any]						, __UpperCamelCase      :  List[Any]						, __UpperCamelCase      :  Dict						, __UpperCamelCase      :  Optional[int]						, __UpperCamelCase      :  List[str]						, __UpperCamelCase      :  int						, __UpperCamelCase      :  List[Any]						, __UpperCamelCase      :  Union[str, Any]       )		->     Optional[Any]:
											A      						=    NystromformerForMaskedLM(config=__UpperCamelCase       )
											model.to(__UpperCamelCase       )
											model.eval()
											A      						=    model(__UpperCamelCase						, attention_mask=__UpperCamelCase						, token_type_ids=__UpperCamelCase						, labels=__UpperCamelCase       )
											self.parent.assertEqual(result.logits.shape						, (self.batch_size, self.seq_length, self.vocab_size)       )


				def __UpperCamelCase						(							self      :  int						, __UpperCamelCase      :  str						, __UpperCamelCase      :  Tuple						, __UpperCamelCase      :  Dict						, __UpperCamelCase      :  str						, __UpperCamelCase      :  Any						, __UpperCamelCase      :  int						, __UpperCamelCase      :  Dict       )		->     Optional[int]:
											A      						=    NystromformerForQuestionAnswering(config=__UpperCamelCase       )
											model.to(__UpperCamelCase       )
											model.eval()
											A      						=    model(
											    __UpperCamelCase						, attention_mask=__UpperCamelCase						, token_type_ids=__UpperCamelCase						, start_positions=__UpperCamelCase						, end_positions=__UpperCamelCase						, )
											self.parent.assertEqual(result.start_logits.shape						, (self.batch_size, self.seq_length)       )
											self.parent.assertEqual(result.end_logits.shape						, (self.batch_size, self.seq_length)       )


				def __UpperCamelCase						(							self      :  Any						, __UpperCamelCase      :  Any						, __UpperCamelCase      :  Any						, __UpperCamelCase      :  Optional[Any]						, __UpperCamelCase      :  List[str]						, __UpperCamelCase      :  List[Any]						, __UpperCamelCase      :  Optional[int]						, __UpperCamelCase      :  str       )		->     Tuple:
											A      						=    self.num_labels
											A      						=    NystromformerForSequenceClassification(__UpperCamelCase       )
											model.to(__UpperCamelCase       )
											model.eval()
											A      						=    model(__UpperCamelCase						, attention_mask=__UpperCamelCase						, token_type_ids=__UpperCamelCase						, labels=__UpperCamelCase       )
											self.parent.assertEqual(result.logits.shape						, (self.batch_size, self.num_labels)       )


				def __UpperCamelCase						(							self      :  List[Any]						, __UpperCamelCase      :  str						, __UpperCamelCase      :  Optional[Any]						, __UpperCamelCase      :  Dict						, __UpperCamelCase      :  Optional[int]						, __UpperCamelCase      :  List[Any]						, __UpperCamelCase      :  Optional[int]						, __UpperCamelCase      :  List[str]       )		->     Dict:
											A      						=    self.num_labels
											A      						=    NystromformerForTokenClassification(config=__UpperCamelCase       )
											model.to(__UpperCamelCase       )
											model.eval()
											A      						=    model(__UpperCamelCase						, attention_mask=__UpperCamelCase						, token_type_ids=__UpperCamelCase						, labels=__UpperCamelCase       )
											self.parent.assertEqual(result.logits.shape						, (self.batch_size, self.seq_length, self.num_labels)       )


				def __UpperCamelCase						(							self      :  Union[str, Any]						, __UpperCamelCase      :  List[str]						, __UpperCamelCase      :  Tuple						, __UpperCamelCase      :  Optional[Any]						, __UpperCamelCase      :  Union[str, Any]						, __UpperCamelCase      :  Optional[Any]						, __UpperCamelCase      :  str						, __UpperCamelCase      :  List[str]       )		->     str:
											A      						=    self.num_choices
											A      						=    NystromformerForMultipleChoice(config=__UpperCamelCase       )
											model.to(__UpperCamelCase       )
											model.eval()
											A      						=    input_ids.unsqueeze(1       ).expand(-1						, self.num_choices						, -1       ).contiguous()
											A      						=    token_type_ids.unsqueeze(1       ).expand(-1						, self.num_choices						, -1       ).contiguous()
											A      						=    input_mask.unsqueeze(1       ).expand(-1						, self.num_choices						, -1       ).contiguous()
											A      						=    model(
											    __UpperCamelCase						, attention_mask=__UpperCamelCase						, token_type_ids=__UpperCamelCase						, labels=__UpperCamelCase						, )
											self.parent.assertEqual(result.logits.shape						, (self.batch_size, self.num_choices)       )


				def __UpperCamelCase						(							self      :  Tuple       )		->     str:
											A      						=    self.prepare_config_and_inputs()
											(
											    (
											    A
											)						,   (
											    A
											)						,   (
											    A
											)						,   (
											    A
											)						,   (
											    A
											)						,   (
											    A
											)						,   (
											    A
											)						,   
											)      						=    config_and_inputs
											A      						=    {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
											return config, inputs_dict





@require_torch
class        lowerCAmelCase__					(  _lowerCamelCase ,				_lowerCamelCase ,				unittest.TestCase       ):
				A_   :     Optional[Any]								=     (
				    (
				        NystromformerModel,
				        NystromformerForMaskedLM,
				        NystromformerForMultipleChoice,
				        NystromformerForQuestionAnswering,
				        NystromformerForSequenceClassification,
				        NystromformerForTokenClassification,
				    )
				    if is_torch_available()
				    else ()
				)
				A_   :     Optional[Any]								=     (
				    {
				        'feature-extraction': NystromformerModel,
				        'fill-mask': NystromformerForMaskedLM,
				        'question-answering': NystromformerForQuestionAnswering,
				        'text-classification': NystromformerForSequenceClassification,
				        'token-classification': NystromformerForTokenClassification,
				        'zero-shot': NystromformerForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				A_   :     Dict								=     False
				A_   :     int								=     False


				def __UpperCamelCase						(							self      :  Dict       )		->     int:
											A      						=    NystromformerModelTester(self       )
											A      						=    ConfigTester(self						, config_class=__UpperCamelCase						, hidden_size=37       )


				def __UpperCamelCase						(							self      :  List[Any]       )		->     Tuple:
											self.config_tester.run_common_tests()


				def __UpperCamelCase						(							self      :  Tuple       )		->     str:
											A      						=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  str       )		->     Any:
											A      						=    self.model_tester.prepare_config_and_inputs()
											for type in ["absolute", "relative_key", "relative_key_query"]:
																		A      						=    type
																		self.model_tester.create_and_check_model(*__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Optional[Any]       )		->     Dict:
											A      						=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_masked_lm(*__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Optional[Any]       )		->     List[Any]:
											A      						=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_multiple_choice(*__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Dict       )		->     List[str]:
											A      						=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_question_answering(*__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Tuple       )		->     str:
											A      						=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_sequence_classification(*__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  List[str]       )		->     Optional[int]:
											A      						=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_token_classification(*__UpperCamelCase       )


				@slow
				def __UpperCamelCase						(							self      :  int       )		->     Optional[int]:
											for model_name in NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																		A      						=    NystromformerModel.from_pretrained(__UpperCamelCase       )
																		self.assertIsNotNone(__UpperCamelCase       )





@require_torch
class        lowerCAmelCase__					(  unittest.TestCase       ):


				@slow
				def __UpperCamelCase						(							self      :  int       )		->     Dict:
											A      						=    NystromformerModel.from_pretrained('uw-madison/nystromformer-512'       )
											A      						=    torch.tensor([[0, 1, 2, 3, 4, 5]]       )

											with torch.no_grad():
																		A      						=    model(__UpperCamelCase       )[0]

											A      						=    torch.Size((1, 6, 768)       )
											self.assertEqual(output.shape						, __UpperCamelCase       )

											A      						=    torch.tensor(
											    [[[-0.4_5_3_2, -0.0_9_3_6, 0.5_1_3_7], [-0.2_6_7_6, 0.0_6_2_8, 0.6_1_8_6], [-0.3_6_2_9, -0.1_7_2_6, 0.4_7_1_6]]]       )

											self.assertTrue(torch.allclose(output[:, :3, :3]						, __UpperCamelCase						, atol=1e-4       )       )


				@slow
				def __UpperCamelCase						(							self      :  List[Any]       )		->     int:
											A      						=    'the [MASK] of Belgium is Brussels'

											A      						=    AutoTokenizer.from_pretrained('uw-madison/nystromformer-512'       )
											A      						=    NystromformerForMaskedLM.from_pretrained('uw-madison/nystromformer-512'       )

											A      						=    tokenizer(__UpperCamelCase						, return_tensors='pt'       )

											with torch.no_grad():
																		A      						=    model(encoding.input_ids       ).logits

											A      						=    token_logits[:, 2, :].argmax(-1       )[0]

											self.assertEqual(tokenizer.decode(__UpperCamelCase       )						, 'capital'       )





import unittest

from transformers import BertGenerationConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import BertGenerationDecoder, BertGenerationEncoder


class 							UpperCamelCase_       :

    '''simple docstring'''






    def __init__(     self :      Tuple   ,					UpperCAmelCase__ :      List[Any]   ,					UpperCAmelCase__ :      Dict=13   ,					UpperCAmelCase__ :      Dict=7   ,					UpperCAmelCase__ :      Dict=True   ,					UpperCAmelCase__ :      List[Any]=True   ,					UpperCAmelCase__ :      str=99   ,					UpperCAmelCase__ :      Union[str, Any]=32   ,					UpperCAmelCase__ :      Tuple=5   ,					UpperCAmelCase__ :      Union[str, Any]=4   ,					UpperCAmelCase__ :      List[Any]=37   ,					UpperCAmelCase__ :      Union[str, Any]="gelu"   ,					UpperCAmelCase__ :      Optional[int]=0.1   ,					UpperCAmelCase__ :      Optional[Any]=0.1   ,					UpperCAmelCase__ :      Tuple=50   ,					UpperCAmelCase__ :      Optional[int]=0.02   ,					UpperCAmelCase__ :      List[str]=True   ,					UpperCAmelCase__ :      List[str]=None   ,					)		->Union[str, Any]:





           '''simple docstring'''

           A__ 					=       parent
           A__ 					=       batch_size
           A__ 					=       seq_length
           A__ 					=       is_training
           A__ 					=       use_input_mask
           A__ 					=       vocab_size
           A__ 					=       hidden_size
           A__ 					=       num_hidden_layers
           A__ 					=       num_attention_heads
           A__ 					=       intermediate_size
           A__ 					=       hidden_act
           A__ 					=       hidden_dropout_prob
           A__ 					=       attention_probs_dropout_prob
           A__ 					=       max_position_embeddings
           A__ 					=       initializer_range
           A__ 					=       use_labels
           A__ 					=       scope


    def 			SCREAMING_SNAKE_CASE				(     self :      int)		->Any:





           '''simple docstring'''

           A__ 					=       ids_tensor([self.batch_size, self.seq_length]   ,					self.vocab_size)

           A__ 					=       None
           if self.use_input_mask:
                  A__ 					=       random_attention_mask([self.batch_size, self.seq_length])

           if self.use_labels:
                  A__ 					=       ids_tensor([self.batch_size, self.seq_length]   ,					self.vocab_size)

           A__ 					=       self.get_config()

           return config, input_ids, input_mask, token_labels


    def 			SCREAMING_SNAKE_CASE				(     self :      int)		->int:





           '''simple docstring'''

           return BertGenerationConfig(
               vocab_size=self.vocab_size   ,					hidden_size=self.hidden_size   ,					num_hidden_layers=self.num_hidden_layers   ,					num_attention_heads=self.num_attention_heads   ,					intermediate_size=self.intermediate_size   ,					hidden_act=self.hidden_act   ,					hidden_dropout_prob=self.hidden_dropout_prob   ,					attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,					max_position_embeddings=self.max_position_embeddings   ,					is_decoder=UpperCAmelCase__   ,					initializer_range=self.initializer_range   ,					)


    def 			SCREAMING_SNAKE_CASE				(     self :      List[str])		->Union[str, Any]:





           '''simple docstring'''

           (
               (
               A__
           )					,				(
               A__
           )					,				(
               A__
           )					,				(
               A__
           )					,				
           ) 					=       self.prepare_config_and_inputs()

           A__ 					=       True
           A__ 					=       floats_tensor([self.batch_size, self.seq_length, self.hidden_size])
           A__ 					=       ids_tensor([self.batch_size, self.seq_length]   ,					vocab_size=2)

           return (
               config,
               input_ids,
               input_mask,
               token_labels,
               encoder_hidden_states,
               encoder_attention_mask,
           )


    def 			SCREAMING_SNAKE_CASE				(     self :      Any   ,					UpperCAmelCase__ :      Any   ,					UpperCAmelCase__ :      Optional[Any]   ,					UpperCAmelCase__ :      Optional[int]   ,					UpperCAmelCase__ :      List[str]   ,					**UpperCAmelCase__ :      List[Any]   ,					)		->Dict:





           '''simple docstring'''

           A__ 					=       BertGenerationEncoder(config=UpperCAmelCase__)
           model.to(UpperCAmelCase__)
           model.eval()
           A__ 					=       model(UpperCAmelCase__   ,					attention_mask=UpperCAmelCase__)
           A__ 					=       model(UpperCAmelCase__)
           self.parent.assertEqual(result.last_hidden_state.shape   ,					(self.batch_size, self.seq_length, self.hidden_size))


    def 			SCREAMING_SNAKE_CASE				(     self :      Dict   ,					UpperCAmelCase__ :      Union[str, Any]   ,					UpperCAmelCase__ :      Union[str, Any]   ,					UpperCAmelCase__ :      str   ,					UpperCAmelCase__ :      List[str]   ,					UpperCAmelCase__ :      Optional[int]   ,					UpperCAmelCase__ :      int   ,					**UpperCAmelCase__ :      Optional[Any]   ,					)		->Dict:





           '''simple docstring'''

           A__ 					=       True
           A__ 					=       BertGenerationEncoder(config=UpperCAmelCase__)
           model.to(UpperCAmelCase__)
           model.eval()
           A__ 					=       model(
               UpperCAmelCase__   ,					attention_mask=UpperCAmelCase__   ,					encoder_hidden_states=UpperCAmelCase__   ,					encoder_attention_mask=UpperCAmelCase__   ,					)
           A__ 					=       model(
               UpperCAmelCase__   ,					attention_mask=UpperCAmelCase__   ,					encoder_hidden_states=UpperCAmelCase__   ,					)
           self.parent.assertEqual(result.last_hidden_state.shape   ,					(self.batch_size, self.seq_length, self.hidden_size))


    def 			SCREAMING_SNAKE_CASE				(     self :      Optional[int]   ,					UpperCAmelCase__ :      str   ,					UpperCAmelCase__ :      Tuple   ,					UpperCAmelCase__ :      Optional[int]   ,					UpperCAmelCase__ :      Any   ,					UpperCAmelCase__ :      Optional[int]   ,					UpperCAmelCase__ :      Dict   ,					**UpperCAmelCase__ :      Optional[int]   ,					)		->Any:





           '''simple docstring'''

           A__ 					=       True
           A__ 					=       True
           A__ 					=       BertGenerationDecoder(config=UpperCAmelCase__).to(UpperCAmelCase__).eval()

           # first forward pass
           A__ 					=       model(
               UpperCAmelCase__   ,					attention_mask=UpperCAmelCase__   ,					encoder_hidden_states=UpperCAmelCase__   ,					encoder_attention_mask=UpperCAmelCase__   ,					use_cache=UpperCAmelCase__   ,					)
           A__ 					=       outputs.past_key_values

           # create hypothetical multiple next token and extent to next_input_ids
           A__ 					=       ids_tensor((self.batch_size, 3)   ,					config.vocab_size)
           A__ 					=       ids_tensor((self.batch_size, 3)   ,					vocab_size=2)

           # append to next input_ids and
           A__ 					=       torch.cat([input_ids, next_tokens]   ,					dim=-1)
           A__ 					=       torch.cat([input_mask, next_mask]   ,					dim=-1)

           A__ 					=       model(
               UpperCAmelCase__   ,					attention_mask=UpperCAmelCase__   ,					encoder_hidden_states=UpperCAmelCase__   ,					encoder_attention_mask=UpperCAmelCase__   ,					output_hidden_states=UpperCAmelCase__   ,					)['''hidden_states'''][0]
           A__ 					=       model(
               UpperCAmelCase__   ,					attention_mask=UpperCAmelCase__   ,					encoder_hidden_states=UpperCAmelCase__   ,					encoder_attention_mask=UpperCAmelCase__   ,					past_key_values=UpperCAmelCase__   ,					output_hidden_states=UpperCAmelCase__   ,					)['''hidden_states'''][0]

           # select random slice
           A__ 					=       ids_tensor((1,)   ,					output_from_past.shape[-1]).item()
           A__ 					=       output_from_no_past[:, -3:, random_slice_idx].detach()
           A__ 					=       output_from_past[:, :, random_slice_idx].detach()

           self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])

           # test that outputs are equal for slice
           self.parent.assertTrue(torch.allclose(UpperCAmelCase__   ,					UpperCAmelCase__   ,					atol=1e-3))


    def 			SCREAMING_SNAKE_CASE				(     self :      Tuple   ,					UpperCAmelCase__ :      int   ,					UpperCAmelCase__ :      Any   ,					UpperCAmelCase__ :      Optional[int]   ,					UpperCAmelCase__ :      List[str]   ,					*UpperCAmelCase__ :      List[str]   ,					)		->List[Any]:





           '''simple docstring'''

           A__ 					=       BertGenerationDecoder(UpperCAmelCase__)
           model.to(UpperCAmelCase__)
           model.eval()
           A__ 					=       model(UpperCAmelCase__   ,					attention_mask=UpperCAmelCase__   ,					labels=UpperCAmelCase__)
           self.parent.assertEqual(result.logits.shape   ,					(self.batch_size, self.seq_length, self.vocab_size))





    def 			SCREAMING_SNAKE_CASE				(     self :      Union[str, Any])		->List[str]:





           '''simple docstring'''

           A__					,				A__					,				A__					,				A__ 					=       self.prepare_config_and_inputs()
           A__ 					=       {'''input_ids''': input_ids, '''attention_mask''': input_mask}
           return config, inputs_dict


@require_torch
class 							UpperCamelCase_       (					UpperCAmelCase__						,   UpperCAmelCase__						,   UpperCAmelCase__						,   unittest.TestCase	):

    '''simple docstring'''






    UpperCAmelCase__				    = (BertGenerationEncoder, BertGenerationDecoder) if is_torch_available() else ()
    UpperCAmelCase__				    = (BertGenerationDecoder,) if is_torch_available() else ()
    UpperCAmelCase__				    = (
        {'''feature-extraction''': BertGenerationEncoder, '''text-generation''': BertGenerationDecoder}
        if is_torch_available()
        else {}
    )


    def 			SCREAMING_SNAKE_CASE				(     self :      Optional[int])		->Dict:





           '''simple docstring'''

           A__ 					=       BertGenerationEncoderTester(self)
           A__ 					=       ConfigTester(self   ,					config_class=UpperCAmelCase__   ,					hidden_size=37)


    def 			SCREAMING_SNAKE_CASE				(     self :      Optional[int])		->List[str]:





           '''simple docstring'''

           self.config_tester.run_common_tests()


    def 			SCREAMING_SNAKE_CASE				(     self :      List[Any])		->int:





           '''simple docstring'''

           A__ 					=       self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*UpperCAmelCase__)


    def 			SCREAMING_SNAKE_CASE				(     self :      Dict)		->Optional[Any]:





           '''simple docstring'''

           A__					,				A__					,				A__					,				A__ 					=       self.model_tester.prepare_config_and_inputs()
           A__ 					=       '''bert'''
           self.model_tester.create_and_check_model(UpperCAmelCase__   ,					UpperCAmelCase__   ,					UpperCAmelCase__   ,					UpperCAmelCase__)


    def 			SCREAMING_SNAKE_CASE				(     self :      int)		->Optional[int]:





           '''simple docstring'''

           A__ 					=       self.model_tester.prepare_config_and_inputs_for_decoder()
           self.model_tester.create_and_check_model_as_decoder(*UpperCAmelCase__)


    def 			SCREAMING_SNAKE_CASE				(     self :      Dict)		->Union[str, Any]:





           '''simple docstring'''

           A__ 					=       self.model_tester.prepare_config_and_inputs_for_decoder()
           self.model_tester.create_and_check_decoder_model_past_large_inputs(*UpperCAmelCase__)


    def 			SCREAMING_SNAKE_CASE				(     self :      Optional[int])		->Any:





           '''simple docstring'''

           (
               (
               A__
           )					,				(
               A__
           )					,				(
               A__
           )					,				(
               A__
           )					,				(
               A__
           )					,				(
               A__
           )					,				
           ) 					=       self.model_tester.prepare_config_and_inputs_for_decoder()

           A__ 					=       None

           self.model_tester.create_and_check_model_as_decoder(
               UpperCAmelCase__   ,					UpperCAmelCase__   ,					UpperCAmelCase__   ,					UpperCAmelCase__   ,					UpperCAmelCase__   ,					UpperCAmelCase__   ,					)


    def 			SCREAMING_SNAKE_CASE				(     self :      List[Any])		->List[Any]:





           '''simple docstring'''

           A__ 					=       self.model_tester.prepare_config_and_inputs_for_decoder()
           self.model_tester.create_and_check_for_causal_lm(*UpperCAmelCase__)





    @slow
    def 			SCREAMING_SNAKE_CASE				(     self :      Dict)		->List[Any]:





           '''simple docstring'''

           A__ 					=       BertGenerationEncoder.from_pretrained('''google/bert_for_seq_generation_L-24_bbc_encoder''')
           self.assertIsNotNone(UpperCAmelCase__)


@require_torch
class 							UpperCamelCase_       (					unittest.TestCase	):

    '''simple docstring'''






    @slow
    def 			SCREAMING_SNAKE_CASE				(     self :      Any)		->Union[str, Any]:





           '''simple docstring'''

           A__ 					=       BertGenerationEncoder.from_pretrained('''google/bert_for_seq_generation_L-24_bbc_encoder''')
           A__ 					=       torch.tensor([[101, 7_592, 1_010, 2_026, 3_899, 2_003, 10_140, 102]])
           with torch.no_grad():
                  A__ 					=       model(UpperCAmelCase__)[0]
           A__ 					=       torch.Size([1, 8, 1_024])
           self.assertEqual(output.shape   ,					UpperCAmelCase__)
           A__ 					=       torch.tensor(
               [[[0.1775, 0.0083, -0.0321], [1.6002, 0.1287, 0.3912], [2.1473, 0.5791, 0.6066]]])
           self.assertTrue(torch.allclose(output[:, :3, :3]   ,					UpperCAmelCase__   ,					atol=1e-4))


@require_torch
class 							UpperCamelCase_       (					unittest.TestCase	):

    '''simple docstring'''






    @slow
    def 			SCREAMING_SNAKE_CASE				(     self :      Optional[Any])		->Union[str, Any]:





           '''simple docstring'''

           A__ 					=       BertGenerationDecoder.from_pretrained('''google/bert_for_seq_generation_L-24_bbc_encoder''')
           A__ 					=       torch.tensor([[101, 7_592, 1_010, 2_026, 3_899, 2_003, 10_140, 102]])
           with torch.no_grad():
                  A__ 					=       model(UpperCAmelCase__)[0]
           A__ 					=       torch.Size([1, 8, 50_358])
           self.assertEqual(output.shape   ,					UpperCAmelCase__)
           A__ 					=       torch.tensor(
               [[[-0.5788, -2.5994, -3.7054], [0.0438, 4.7997, 1.8795], [1.5862, 6.6409, 4.4638]]])
           self.assertTrue(torch.allclose(output[:, :3, :3]   ,					UpperCAmelCase__   ,					atol=1e-4))


