






import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class  lowerCAmelCase_       (				snake_case__     ,    unittest.TestCase					):
							UpperCamelCase_		:Optional[int]		      =    RoCBertTokenizer
							UpperCamelCase_		:List[Any]		      =    None
							UpperCamelCase_		:Union[str, Any]		      =    False
							UpperCamelCase_		:Dict		      =    True
							UpperCamelCase_		:Union[str, Any]		      =    filter_non_english




							def 						__snake_case					(   self     :   Any	):
											super().setUp()

											lowerCAmelCase__      			=						['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
											lowerCAmelCase__      			=						{}
											lowerCAmelCase__      			=						{}
											for i, value in enumerate(SCREAMING_SNAKE_CASE_	):
															lowerCAmelCase__      			=						i
															lowerCAmelCase__      			=						i
											lowerCAmelCase__      			=						os.path.join(self.tmpdirname ,							VOCAB_FILES_NAMES['''vocab_file''']	)
											lowerCAmelCase__      			=						os.path.join(self.tmpdirname ,							VOCAB_FILES_NAMES['''word_shape_file''']	)
											lowerCAmelCase__      			=						os.path.join(self.tmpdirname ,							VOCAB_FILES_NAMES['''word_pronunciation_file''']	)
											with open(self.vocab_file ,							'''w''' ,							encoding='''utf-8'''	) as vocab_writer:
															vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]	)	)
											with open(self.word_shape_file ,							'''w''' ,							encoding='''utf-8'''	) as word_shape_writer:
															json.dump(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							ensure_ascii=SCREAMING_SNAKE_CASE_	)
											with open(self.word_pronunciation_file ,							'''w''' ,							encoding='''utf-8'''	) as word_pronunciation_writer:
															json.dump(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							ensure_ascii=SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   Any	):
											lowerCAmelCase__      			=						self.tokenizer_class(self.vocab_file ,							self.word_shape_file ,							self.word_pronunciation_file	)

											lowerCAmelCase__      			=						tokenizer.tokenize('''你好[SEP]你是谁'''	)
											self.assertListEqual(SCREAMING_SNAKE_CASE_ ,							['''你''', '''好''', '''[SEP]''', '''你''', '''是''', '''谁''']	)
											self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_	) ,							[5, 6, 2, 5, 7, 8]	)
											self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(SCREAMING_SNAKE_CASE_	) ,							[5, 6, 2, 5, 7, 8]	)
											self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(SCREAMING_SNAKE_CASE_	) ,							[5, 6, 2, 5, 7, 8]	)




							def 						__snake_case					(   self     :   Optional[int]	):
											lowerCAmelCase__      			=						RoCBertBasicTokenizer()

											self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''	) ,							['''ah''', '''\u535A''', '''\u63A8''', '''zz''']	)




							def 						__snake_case					(   self     :   List[Any]	):
											lowerCAmelCase__      			=						RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_	)

											self.assertListEqual(
											    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''	) ,							['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
											self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	) ,							['''hello''']	)




							def 						__snake_case					(   self     :   Optional[int]	):
											lowerCAmelCase__      			=						RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_ ,							strip_accents=SCREAMING_SNAKE_CASE_	)

											self.assertListEqual(
											    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	) ,							['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
											self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	) ,							['''h\u00E9llo''']	)




							def 						__snake_case					(   self     :   int	):
											lowerCAmelCase__      			=						RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_ ,							strip_accents=SCREAMING_SNAKE_CASE_	)

											self.assertListEqual(
											    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	) ,							['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
											self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	) ,							['''hello''']	)




							def 						__snake_case					(   self     :   Optional[Any]	):
											lowerCAmelCase__      			=						RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_	)

											self.assertListEqual(
											    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	) ,							['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
											self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	) ,							['''hello''']	)




							def 						__snake_case					(   self     :   Any	):
											lowerCAmelCase__      			=						RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_	)

											self.assertListEqual(
											    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''	) ,							['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)




							def 						__snake_case					(   self     :   int	):
											lowerCAmelCase__      			=						RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_ ,							strip_accents=SCREAMING_SNAKE_CASE_	)

											self.assertListEqual(
											    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	) ,							['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)




							def 						__snake_case					(   self     :   Tuple	):
											lowerCAmelCase__      			=						RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_ ,							strip_accents=SCREAMING_SNAKE_CASE_	)

											self.assertListEqual(
											    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	) ,							['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)




							def 						__snake_case					(   self     :   Optional[Any]	):
											lowerCAmelCase__      			=						RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_ ,							never_split=['''[UNK]''']	)

											self.assertListEqual(
											    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''	) ,							['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']	)




							def 						__snake_case					(   self     :   Optional[Any]	):
											lowerCAmelCase__      			=						['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

											lowerCAmelCase__      			=						{}
											for i, token in enumerate(SCREAMING_SNAKE_CASE_	):
															lowerCAmelCase__      			=						i
											lowerCAmelCase__      			=						RoCBertWordpieceTokenizer(vocab=SCREAMING_SNAKE_CASE_ ,							unk_token='''[UNK]'''	)

											self.assertListEqual(tokenizer.tokenize(''''''	) ,							[]	)

											self.assertListEqual(tokenizer.tokenize('''unwanted running'''	) ,							['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']	)

											self.assertListEqual(tokenizer.tokenize('''unwantedX running'''	) ,							['''[UNK]''', '''runn''', '''##ing''']	)




							def 						__snake_case					(   self     :   Optional[Any]	):
											self.assertTrue(_is_whitespace(''' '''	)	)
											self.assertTrue(_is_whitespace('''\t'''	)	)
											self.assertTrue(_is_whitespace('''\r'''	)	)
											self.assertTrue(_is_whitespace('''\n'''	)	)
											self.assertTrue(_is_whitespace('''\u00A0'''	)	)

											self.assertFalse(_is_whitespace('''A'''	)	)
											self.assertFalse(_is_whitespace('''-'''	)	)




							def 						__snake_case					(   self     :   Tuple	):
											self.assertTrue(_is_control('''\u0005'''	)	)

											self.assertFalse(_is_control('''A'''	)	)
											self.assertFalse(_is_control(''' '''	)	)
											self.assertFalse(_is_control('''\t'''	)	)
											self.assertFalse(_is_control('''\r'''	)	)




							def 						__snake_case					(   self     :   int	):
											self.assertTrue(_is_punctuation('''-'''	)	)
											self.assertTrue(_is_punctuation('''$'''	)	)
											self.assertTrue(_is_punctuation('''`'''	)	)
											self.assertTrue(_is_punctuation('''.'''	)	)

											self.assertFalse(_is_punctuation('''A'''	)	)
											self.assertFalse(_is_punctuation(''' '''	)	)




							def 						__snake_case					(   self     :   str	):
											lowerCAmelCase__      			=						self.get_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(SCREAMING_SNAKE_CASE_	) for t in ['''Test''', '''\xad''', '''test''']] ,							[['''[UNK]'''], [], ['''[UNK]''']]	)

											if self.test_rust_tokenizer:
															lowerCAmelCase__      			=						self.get_rust_tokenizer()
															self.assertListEqual(
															    [rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_	) for t in ['''Test''', '''\xad''', '''test''']] ,							[['''[UNK]'''], [], ['''[UNK]''']]	)




							def 						__snake_case					(   self     :   Dict	):
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'	):
																			lowerCAmelCase__      			=						self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_ ,							**SCREAMING_SNAKE_CASE_	)

																			lowerCAmelCase__      			=						f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
																			lowerCAmelCase__      			=						tokenizer_r.encode_plus(
																			    SCREAMING_SNAKE_CASE_ ,							return_attention_mask=SCREAMING_SNAKE_CASE_ ,							return_token_type_ids=SCREAMING_SNAKE_CASE_ ,							return_offsets_mapping=SCREAMING_SNAKE_CASE_ ,							add_special_tokens=SCREAMING_SNAKE_CASE_ ,							)

																			lowerCAmelCase__      			=						tokenizer_r.do_lower_case if hasattr(SCREAMING_SNAKE_CASE_ ,							'''do_lower_case'''	) else False
																			lowerCAmelCase__      			=						(
																			    [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), '''A'''),
																			        ((1, 2), ''','''),
																			        ((3, 5), '''na'''),
																			        ((5, 6), '''##ï'''),
																			        ((6, 8), '''##ve'''),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), '''Allen'''),
																			        ((21, 23), '''##NL'''),
																			        ((23, 24), '''##P'''),
																			        ((25, 33), '''sentence'''),
																			        ((33, 34), '''.'''),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			    if not do_lower_case
																			    else [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), '''a'''),
																			        ((1, 2), ''','''),
																			        ((3, 8), '''naive'''),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), '''allen'''),
																			        ((21, 23), '''##nl'''),
																			        ((23, 24), '''##p'''),
																			        ((25, 33), '''sentence'''),
																			        ((33, 34), '''.'''),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			)

																			self.assertEqual(
																			    [e[1] for e in expected_results] ,							tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']	)	)
																			self.assertEqual([e[0] for e in expected_results] ,							tokens['''offset_mapping''']	)




							def 						__snake_case					(   self     :   Optional[Any]	):
											lowerCAmelCase__      			=						['''的''', '''人''', '''有''']
											lowerCAmelCase__      			=						''''''.join(SCREAMING_SNAKE_CASE_	)
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'	):
																			lowerCAmelCase__      			=						True
																			lowerCAmelCase__      			=						self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_ ,							**SCREAMING_SNAKE_CASE_	)
																			lowerCAmelCase__      			=						self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_ ,							**SCREAMING_SNAKE_CASE_	)

																			lowerCAmelCase__      			=						tokenizer_p.encode(SCREAMING_SNAKE_CASE_ ,							add_special_tokens=SCREAMING_SNAKE_CASE_	)
																			lowerCAmelCase__      			=						tokenizer_r.encode(SCREAMING_SNAKE_CASE_ ,							add_special_tokens=SCREAMING_SNAKE_CASE_	)

																			lowerCAmelCase__      			=						tokenizer_r.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_	)
																			lowerCAmelCase__      			=						tokenizer_p.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_	)

																			# it is expected that each Chinese character is not preceded by "##"
																			self.assertListEqual(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_	)
																			self.assertListEqual(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_	)

																			lowerCAmelCase__      			=						False
																			lowerCAmelCase__      			=						self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_ ,							**SCREAMING_SNAKE_CASE_	)
																			lowerCAmelCase__      			=						self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_ ,							**SCREAMING_SNAKE_CASE_	)

																			lowerCAmelCase__      			=						tokenizer_r.encode(SCREAMING_SNAKE_CASE_ ,							add_special_tokens=SCREAMING_SNAKE_CASE_	)
																			lowerCAmelCase__      			=						tokenizer_p.encode(SCREAMING_SNAKE_CASE_ ,							add_special_tokens=SCREAMING_SNAKE_CASE_	)

																			lowerCAmelCase__      			=						tokenizer_r.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_	)
																			lowerCAmelCase__      			=						tokenizer_p.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_	)

																			# it is expected that only the first Chinese character is not preceded by "##".
																			lowerCAmelCase__      			=						[
																			    f'##{token}' if idx != 0 else token for idx, token in enumerate(SCREAMING_SNAKE_CASE_	)
																			]
																			self.assertListEqual(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_	)
																			self.assertListEqual(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_	)




							@slow
							def 						__snake_case					(   self     :   Tuple	):
											lowerCAmelCase__      			=						self.tokenizer_class(self.vocab_file ,							self.word_shape_file ,							self.word_pronunciation_file	)

											lowerCAmelCase__      			=						tokenizer.encode('''你好''' ,							add_special_tokens=SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						tokenizer.encode('''你是谁''' ,							add_special_tokens=SCREAMING_SNAKE_CASE_	)

											lowerCAmelCase__      			=						tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_	)

											assert encoded_sentence == [1] + text + [2]
											assert encoded_pair == [1] + text + [2] + text_a + [2]




							def 						__snake_case					(   self     :   Union[str, Any]	):
											lowerCAmelCase__      			=						self.get_tokenizers(do_lower_case=SCREAMING_SNAKE_CASE_	)
											for tokenizer in tokenizers:
															with self.subTest(f'{tokenizer.__class__.__name__}'	):
																			lowerCAmelCase__      			=						'''你好，你是谁'''
																			lowerCAmelCase__      			=						tokenizer.tokenize(SCREAMING_SNAKE_CASE_	)
																			lowerCAmelCase__      			=						tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_	)
																			lowerCAmelCase__      			=						tokenizer.convert_tokens_to_shape_ids(SCREAMING_SNAKE_CASE_	)
																			lowerCAmelCase__      			=						tokenizer.convert_tokens_to_pronunciation_ids(SCREAMING_SNAKE_CASE_	)
																			lowerCAmelCase__      			=						tokenizer.prepare_for_model(
																			    SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							add_special_tokens=SCREAMING_SNAKE_CASE_	)

																			lowerCAmelCase__      			=						tokenizer.encode_plus(SCREAMING_SNAKE_CASE_ ,							add_special_tokens=SCREAMING_SNAKE_CASE_	)

																			self.assertEqual(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_	)













import unittest

from transformers import BertGenerationConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import BertGenerationDecoder, BertGenerationEncoder
class  lowerCAmelCase_       :




							def __init__(   self     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   str ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any]=13 ,							SCREAMING_SNAKE_CASE_     :   Dict=7 ,							SCREAMING_SNAKE_CASE_     :   Dict=True ,							SCREAMING_SNAKE_CASE_     :   Optional[Any]=True ,							SCREAMING_SNAKE_CASE_     :   List[str]=99 ,							SCREAMING_SNAKE_CASE_     :   str=32 ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any]=5 ,							SCREAMING_SNAKE_CASE_     :   str=4 ,							SCREAMING_SNAKE_CASE_     :   Tuple=37 ,							SCREAMING_SNAKE_CASE_     :   List[Any]="gelu" ,							SCREAMING_SNAKE_CASE_     :   Tuple=0.1 ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any]=0.1 ,							SCREAMING_SNAKE_CASE_     :   Optional[Any]=50 ,							SCREAMING_SNAKE_CASE_     :   List[str]=0.02 ,							SCREAMING_SNAKE_CASE_     :   List[str]=True ,							SCREAMING_SNAKE_CASE_     :   Optional[Any]=None ,							):
											lowerCAmelCase__      			=						parent
											lowerCAmelCase__      			=						batch_size
											lowerCAmelCase__      			=						seq_length
											lowerCAmelCase__      			=						is_training
											lowerCAmelCase__      			=						use_input_mask
											lowerCAmelCase__      			=						vocab_size
											lowerCAmelCase__      			=						hidden_size
											lowerCAmelCase__      			=						num_hidden_layers
											lowerCAmelCase__      			=						num_attention_heads
											lowerCAmelCase__      			=						intermediate_size
											lowerCAmelCase__      			=						hidden_act
											lowerCAmelCase__      			=						hidden_dropout_prob
											lowerCAmelCase__      			=						attention_probs_dropout_prob
											lowerCAmelCase__      			=						max_position_embeddings
											lowerCAmelCase__      			=						initializer_range
											lowerCAmelCase__      			=						use_labels
											lowerCAmelCase__      			=						scope




							def 						__snake_case					(   self     :   Union[str, Any]	):
											lowerCAmelCase__      			=						ids_tensor([self.batch_size, self.seq_length] ,							self.vocab_size	)

											lowerCAmelCase__      			=						None
											if self.use_input_mask:
															lowerCAmelCase__      			=						random_attention_mask([self.batch_size, self.seq_length]	)

											if self.use_labels:
															lowerCAmelCase__      			=						ids_tensor([self.batch_size, self.seq_length] ,							self.vocab_size	)

											lowerCAmelCase__      			=						self.get_config()

											return config, input_ids, input_mask, token_labels




							def 						__snake_case					(   self     :   List[str]	):
											return BertGenerationConfig(
											    vocab_size=self.vocab_size ,							hidden_size=self.hidden_size ,							num_hidden_layers=self.num_hidden_layers ,							num_attention_heads=self.num_attention_heads ,							intermediate_size=self.intermediate_size ,							hidden_act=self.hidden_act ,							hidden_dropout_prob=self.hidden_dropout_prob ,							attention_probs_dropout_prob=self.attention_probs_dropout_prob ,							max_position_embeddings=self.max_position_embeddings ,							is_decoder=SCREAMING_SNAKE_CASE_ ,							initializer_range=self.initializer_range ,							)




							def 						__snake_case					(   self     :   str	):
											(
											    (
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		
											)      			=						self.prepare_config_and_inputs()

											lowerCAmelCase__      			=						True
											lowerCAmelCase__      			=						floats_tensor([self.batch_size, self.seq_length, self.hidden_size]	)
											lowerCAmelCase__      			=						ids_tensor([self.batch_size, self.seq_length] ,							vocab_size=2	)

											return (
											    config,
											    input_ids,
											    input_mask,
											    token_labels,
											    encoder_hidden_states,
											    encoder_attention_mask,
											)




							def 						__snake_case					(   self     :   Union[str, Any] ,							SCREAMING_SNAKE_CASE_     :   List[str] ,							SCREAMING_SNAKE_CASE_     :   str ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							**SCREAMING_SNAKE_CASE_     :   Union[str, Any] ,							):
											lowerCAmelCase__      			=						BertGenerationEncoder(config=SCREAMING_SNAKE_CASE_	)
											model.to(SCREAMING_SNAKE_CASE_	)
											model.eval()
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)
											self.parent.assertEqual(result.last_hidden_state.shape ,							(self.batch_size, self.seq_length, self.hidden_size)	)




							def 						__snake_case					(   self     :   Dict ,							SCREAMING_SNAKE_CASE_     :   Any ,							SCREAMING_SNAKE_CASE_     :   Any ,							SCREAMING_SNAKE_CASE_     :   Optional[int] ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any] ,							SCREAMING_SNAKE_CASE_     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   Optional[int] ,							**SCREAMING_SNAKE_CASE_     :   str ,							):
											lowerCAmelCase__      			=						True
											lowerCAmelCase__      			=						BertGenerationEncoder(config=SCREAMING_SNAKE_CASE_	)
											model.to(SCREAMING_SNAKE_CASE_	)
											model.eval()
											lowerCAmelCase__      			=						model(
											    SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_ ,							encoder_hidden_states=SCREAMING_SNAKE_CASE_ ,							encoder_attention_mask=SCREAMING_SNAKE_CASE_ ,							)
											lowerCAmelCase__      			=						model(
											    SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_ ,							encoder_hidden_states=SCREAMING_SNAKE_CASE_ ,							)
											self.parent.assertEqual(result.last_hidden_state.shape ,							(self.batch_size, self.seq_length, self.hidden_size)	)




							def 						__snake_case					(   self     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any] ,							SCREAMING_SNAKE_CASE_     :   Dict ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   Tuple ,							SCREAMING_SNAKE_CASE_     :   Optional[int] ,							**SCREAMING_SNAKE_CASE_     :   Optional[Any] ,							):
											lowerCAmelCase__      			=						True
											lowerCAmelCase__      			=						True
											lowerCAmelCase__      			=						BertGenerationDecoder(config=SCREAMING_SNAKE_CASE_	).to(SCREAMING_SNAKE_CASE_	).eval()

											# first forward pass
											lowerCAmelCase__      			=						model(
											    SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_ ,							encoder_hidden_states=SCREAMING_SNAKE_CASE_ ,							encoder_attention_mask=SCREAMING_SNAKE_CASE_ ,							use_cache=SCREAMING_SNAKE_CASE_ ,							)
											lowerCAmelCase__      			=						outputs.past_key_values

											# create hypothetical multiple next token and extent to next_input_ids
											lowerCAmelCase__      			=						ids_tensor((self.batch_size, 3) ,							config.vocab_size	)
											lowerCAmelCase__      			=						ids_tensor((self.batch_size, 3) ,							vocab_size=2	)

											# append to next input_ids and
											lowerCAmelCase__      			=						torch.cat([input_ids, next_tokens] ,							dim=-1	)
											lowerCAmelCase__      			=						torch.cat([input_mask, next_mask] ,							dim=-1	)

											lowerCAmelCase__      			=						model(
											    SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_ ,							encoder_hidden_states=SCREAMING_SNAKE_CASE_ ,							encoder_attention_mask=SCREAMING_SNAKE_CASE_ ,							output_hidden_states=SCREAMING_SNAKE_CASE_ ,							)['''hidden_states'''][0]
											lowerCAmelCase__      			=						model(
											    SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_ ,							encoder_hidden_states=SCREAMING_SNAKE_CASE_ ,							encoder_attention_mask=SCREAMING_SNAKE_CASE_ ,							past_key_values=SCREAMING_SNAKE_CASE_ ,							output_hidden_states=SCREAMING_SNAKE_CASE_ ,							)['''hidden_states'''][0]

											# select random slice
											lowerCAmelCase__      			=						ids_tensor((1,) ,							output_from_past.shape[-1]	).item()
											lowerCAmelCase__      			=						output_from_no_past[:, -3:, random_slice_idx].detach()
											lowerCAmelCase__      			=						output_from_past[:, :, random_slice_idx].detach()

											self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]	)

											# test that outputs are equal for slice
											self.parent.assertTrue(torch.allclose(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							atol=1e-3	)	)




							def 						__snake_case					(   self     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   Any ,							SCREAMING_SNAKE_CASE_     :   int ,							SCREAMING_SNAKE_CASE_     :   Optional[Any] ,							SCREAMING_SNAKE_CASE_     :   Optional[Any] ,							*SCREAMING_SNAKE_CASE_     :   Tuple ,							):
											lowerCAmelCase__      			=						BertGenerationDecoder(SCREAMING_SNAKE_CASE_	)
											model.to(SCREAMING_SNAKE_CASE_	)
											model.eval()
											lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_ ,							attention_mask=SCREAMING_SNAKE_CASE_ ,							labels=SCREAMING_SNAKE_CASE_	)
											self.parent.assertEqual(result.logits.shape ,							(self.batch_size, self.seq_length, self.vocab_size)	)




							def 						__snake_case					(   self     :   List[Any]	):
											lowerCAmelCase__			,		lowerCAmelCase__			,		lowerCAmelCase__			,		lowerCAmelCase__      			=						self.prepare_config_and_inputs()
											lowerCAmelCase__      			=						{'''input_ids''': input_ids, '''attention_mask''': input_mask}
											return config, inputs_dict



@require_torch
class  lowerCAmelCase_       (				snake_case__     ,    snake_case__     ,    snake_case__     ,    unittest.TestCase					):
							UpperCamelCase_		:Dict		      =    (BertGenerationEncoder, BertGenerationDecoder) if is_torch_available() else ()
							UpperCamelCase_		:str		      =    (BertGenerationDecoder,) if is_torch_available() else ()
							UpperCamelCase_		:List[str]		      =    (
							    {'feature-extraction': BertGenerationEncoder, 'text-generation': BertGenerationDecoder}
							    if is_torch_available()
							    else {}
							)




							def 						__snake_case					(   self     :   List[str]	):
											lowerCAmelCase__      			=						BertGenerationEncoderTester(self	)
											lowerCAmelCase__      			=						ConfigTester(self ,							config_class=SCREAMING_SNAKE_CASE_ ,							hidden_size=37	)




							def 						__snake_case					(   self     :   Optional[Any]	):
											self.config_tester.run_common_tests()




							def 						__snake_case					(   self     :   List[Any]	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   str	):
											lowerCAmelCase__			,		lowerCAmelCase__			,		lowerCAmelCase__			,		lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs()
											lowerCAmelCase__      			=						'''bert'''
											self.model_tester.create_and_check_model(SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   int	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs_for_decoder()
											self.model_tester.create_and_check_model_as_decoder(*SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   List[Any]	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs_for_decoder()
											self.model_tester.create_and_check_decoder_model_past_large_inputs(*SCREAMING_SNAKE_CASE_	)




							def 						__snake_case					(   self     :   Any	):
											# This regression test was failing with PyTorch < 1.3
											(
											    (
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		(
											    lowerCAmelCase__
											)			,		
											)      			=						self.model_tester.prepare_config_and_inputs_for_decoder()

											lowerCAmelCase__      			=						None

											self.model_tester.create_and_check_model_as_decoder(
											    SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							SCREAMING_SNAKE_CASE_ ,							)




							def 						__snake_case					(   self     :   int	):
											lowerCAmelCase__      			=						self.model_tester.prepare_config_and_inputs_for_decoder()
											self.model_tester.create_and_check_for_causal_lm(*SCREAMING_SNAKE_CASE_	)




							@slow
							def 						__snake_case					(   self     :   str	):
											lowerCAmelCase__      			=						BertGenerationEncoder.from_pretrained('''google/bert_for_seq_generation_L-24_bbc_encoder'''	)
											self.assertIsNotNone(SCREAMING_SNAKE_CASE_	)



@require_torch
class  lowerCAmelCase_       (				unittest.TestCase					):




							@slow
							def 						__snake_case					(   self     :   Tuple	):
											lowerCAmelCase__      			=						BertGenerationEncoder.from_pretrained('''google/bert_for_seq_generation_L-24_bbc_encoder'''	)
											lowerCAmelCase__      			=						torch.tensor([[101, 7_592, 1_010, 2_026, 3_899, 2_003, 10_140, 102]]	)
											with torch.no_grad():
															lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)[0]
											lowerCAmelCase__      			=						torch.Size([1, 8, 1_024]	)
											self.assertEqual(output.shape ,							SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						torch.tensor(
											    [[[0.1_775, 0.0_083, -0.0_321], [1.6_002, 0.1_287, 0.3_912], [2.1_473, 0.5_791, 0.6_066]]]	)
											self.assertTrue(torch.allclose(output[:, :3, :3] ,							SCREAMING_SNAKE_CASE_ ,							atol=1e-4	)	)



@require_torch
class  lowerCAmelCase_       (				unittest.TestCase					):




							@slow
							def 						__snake_case					(   self     :   Any	):
											lowerCAmelCase__      			=						BertGenerationDecoder.from_pretrained('''google/bert_for_seq_generation_L-24_bbc_encoder'''	)
											lowerCAmelCase__      			=						torch.tensor([[101, 7_592, 1_010, 2_026, 3_899, 2_003, 10_140, 102]]	)
											with torch.no_grad():
															lowerCAmelCase__      			=						model(SCREAMING_SNAKE_CASE_	)[0]
											lowerCAmelCase__      			=						torch.Size([1, 8, 50_358]	)
											self.assertEqual(output.shape ,							SCREAMING_SNAKE_CASE_	)
											lowerCAmelCase__      			=						torch.tensor(
											    [[[-0.5_788, -2.5_994, -3.7_054], [0.0_438, 4.7_997, 1.8_795], [1.5_862, 6.6_409, 4.4_638]]]	)
											self.assertTrue(torch.allclose(output[:, :3, :3] ,							SCREAMING_SNAKE_CASE_ ,							atol=1e-4	)	)





