




import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 				lowerCAmelCase_       (					lowerCamelCase_						,     unittest.TestCase							):
				__a     :       Any						=     BertTokenizer
				__a     :       Tuple						=     BertTokenizerFast
				__a     :       Union[str, Any]						=     True
				__a     :       int						=     True
				__a     :       Union[str, Any]						=     filter_non_english






				def 		snake_case	(			self ):
							super().setUp()

							SCREAMING_SNAKE_CASE_   : Any					  =      [
							    '[UNK]',
							    '[CLS]',
							    '[SEP]',
							    '[PAD]',
							    '[MASK]',
							    'want',
							    '##want',
							    '##ed',
							    'wa',
							    'un',
							    'runn',
							    '##ing',
							    ',',
							    'low',
							    'lowest',
							]
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      os.path.join(self.tmpdirname      ,VOCAB_FILES_NAMES['vocab_file'] )
							with open(self.vocab_file      ,'w'      ,encoding='utf-8' ) as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in vocab_tokens] ) )






				def 		snake_case	(			self      ,snake_case__ ):
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      'UNwant\u00E9d,running'
							SCREAMING_SNAKE_CASE_   : List[str]					  =      'unwanted, running'
							return input_text, output_text






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : str					  =      self.tokenizer_class(self.vocab_file )

							SCREAMING_SNAKE_CASE_   : List[str]					  =      tokenizer.tokenize('UNwant\u00E9d,running' )
							self.assertListEqual(snake_case__      ,['un', '##want', '##ed', ',', 'runn', '##ing'] )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__ )      ,[9, 6, 7, 12, 10, 11] )






				def 		snake_case	(			self ):
							if not self.test_rust_tokenizer:
										return

							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      self.get_tokenizer()
							SCREAMING_SNAKE_CASE_   : int					  =      self.get_rust_tokenizer()

							SCREAMING_SNAKE_CASE_   : str					  =      'UNwant\u00E9d,running'

							SCREAMING_SNAKE_CASE_   : str					  =      tokenizer.tokenize(snake_case__ )
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      rust_tokenizer.tokenize(snake_case__ )
							self.assertListEqual(snake_case__      ,snake_case__ )

							SCREAMING_SNAKE_CASE_   : List[Any]					  =      tokenizer.encode(snake_case__      ,add_special_tokens=snake_case__ )
							SCREAMING_SNAKE_CASE_   : int					  =      rust_tokenizer.encode(snake_case__      ,add_special_tokens=snake_case__ )
							self.assertListEqual(snake_case__      ,snake_case__ )

							SCREAMING_SNAKE_CASE_   : Dict					  =      self.get_rust_tokenizer()
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      tokenizer.encode(snake_case__ )
							SCREAMING_SNAKE_CASE_   : Any					  =      rust_tokenizer.encode(snake_case__ )
							self.assertListEqual(snake_case__      ,snake_case__ )

							# With lower casing
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      self.get_tokenizer(do_lower_case=snake_case__ )
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      self.get_rust_tokenizer(do_lower_case=snake_case__ )

							SCREAMING_SNAKE_CASE_   : int					  =      'UNwant\u00E9d,running'

							SCREAMING_SNAKE_CASE_   : Any					  =      tokenizer.tokenize(snake_case__ )
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      rust_tokenizer.tokenize(snake_case__ )
							self.assertListEqual(snake_case__      ,snake_case__ )

							SCREAMING_SNAKE_CASE_   : str					  =      tokenizer.encode(snake_case__      ,add_special_tokens=snake_case__ )
							SCREAMING_SNAKE_CASE_   : str					  =      rust_tokenizer.encode(snake_case__      ,add_special_tokens=snake_case__ )
							self.assertListEqual(snake_case__      ,snake_case__ )

							SCREAMING_SNAKE_CASE_   : Dict					  =      self.get_rust_tokenizer()
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      tokenizer.encode(snake_case__ )
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      rust_tokenizer.encode(snake_case__ )
							self.assertListEqual(snake_case__      ,snake_case__ )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz' )      ,['ah', '\u535A', '\u63A8', 'zz'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : int					  =      BasicTokenizer(do_lower_case=snake_case__ )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ' )      ,['hello', '!', 'how', 'are', 'you', '?'] )
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )      ,['hello'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Any					  =      BasicTokenizer(do_lower_case=snake_case__      ,strip_accents=snake_case__ )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )      ,['hällo', '!', 'how', 'are', 'you', '?'] )
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )      ,['h\u00E9llo'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : str					  =      BasicTokenizer(do_lower_case=snake_case__      ,strip_accents=snake_case__ )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )      ,['hallo', '!', 'how', 'are', 'you', '?'] )
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )      ,['hello'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Any					  =      BasicTokenizer(do_lower_case=snake_case__ )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )      ,['hallo', '!', 'how', 'are', 'you', '?'] )
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )      ,['hello'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : str					  =      BasicTokenizer(do_lower_case=snake_case__ )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ' )      ,['HeLLo', '!', 'how', 'Are', 'yoU', '?'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Tuple					  =      BasicTokenizer(do_lower_case=snake_case__      ,strip_accents=snake_case__ )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )      ,['HäLLo', '!', 'how', 'Are', 'yoU', '?'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Any					  =      BasicTokenizer(do_lower_case=snake_case__      ,strip_accents=snake_case__ )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )      ,['HaLLo', '!', 'how', 'Are', 'yoU', '?'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : List[str]					  =      BasicTokenizer(do_lower_case=snake_case__      ,never_split=['[UNK]'] )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]' )      ,['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : List[str]					  =      BasicTokenizer()
							SCREAMING_SNAKE_CASE_   : Any					  =      'a\n\'ll !!to?\'d of, can\'t.'
							SCREAMING_SNAKE_CASE_   : Tuple					  =      ['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
							self.assertListEqual(tokenizer.tokenize(snake_case__ )      ,snake_case__ )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

							SCREAMING_SNAKE_CASE_   : List[str]					  =      {}
							for i, token in enumerate(snake_case__ ):
										SCREAMING_SNAKE_CASE_   : Tuple					  =      i
							SCREAMING_SNAKE_CASE_   : List[str]					  =      WordpieceTokenizer(vocab=snake_case__      ,unk_token='[UNK]' )

							self.assertListEqual(tokenizer.tokenize('' )      ,[] )

							self.assertListEqual(tokenizer.tokenize('unwanted running' )      ,['un', '##want', '##ed', 'runn', '##ing'] )

							self.assertListEqual(tokenizer.tokenize('unwantedX running' )      ,['[UNK]', 'runn', '##ing'] )






				def 		snake_case	(			self ):
							self.assertTrue(_is_whitespace(' ' ) )
							self.assertTrue(_is_whitespace('\t' ) )
							self.assertTrue(_is_whitespace('\r' ) )
							self.assertTrue(_is_whitespace('\n' ) )
							self.assertTrue(_is_whitespace('\u00A0' ) )

							self.assertFalse(_is_whitespace('A' ) )
							self.assertFalse(_is_whitespace('-' ) )






				def 		snake_case	(			self ):
							self.assertTrue(_is_control('\u0005' ) )

							self.assertFalse(_is_control('A' ) )
							self.assertFalse(_is_control(' ' ) )
							self.assertFalse(_is_control('\t' ) )
							self.assertFalse(_is_control('\r' ) )






				def 		snake_case	(			self ):
							self.assertTrue(_is_punctuation('-' ) )
							self.assertTrue(_is_punctuation('$' ) )
							self.assertTrue(_is_punctuation('`' ) )
							self.assertTrue(_is_punctuation('.' ) )

							self.assertFalse(_is_punctuation('A' ) )
							self.assertFalse(_is_punctuation(' ' ) )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      self.get_tokenizer()
							SCREAMING_SNAKE_CASE_   : Dict					  =      self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(snake_case__ ) for t in ['Test', '\xad', 'test']]      ,[['[UNK]'], [], ['[UNK]']] )

							self.assertListEqual(
							    [rust_tokenizer.tokenize(snake_case__ ) for t in ['Test', '\xad', 'test']]      ,[['[UNK]'], [], ['[UNK]']] )






				@slow
				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      self.tokenizer_class.from_pretrained('bert-base-uncased' )

							SCREAMING_SNAKE_CASE_   : str					  =      tokenizer.encode('sequence builders'      ,add_special_tokens=snake_case__ )
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      tokenizer.encode('multi-sequence build'      ,add_special_tokens=snake_case__ )

							SCREAMING_SNAKE_CASE_   : int					  =      tokenizer.build_inputs_with_special_tokens(snake_case__ )
							SCREAMING_SNAKE_CASE_   : str					  =      tokenizer.build_inputs_with_special_tokens(snake_case__      ,snake_case__ )

							assert encoded_sentence == [101] + text + [102]
							assert encoded_pair == [101] + text + [102] + text_a + [102]






				def 		snake_case	(			self ):
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})' ):
													SCREAMING_SNAKE_CASE_   : int					  =      self.rust_tokenizer_class.from_pretrained(snake_case__      ,**snake_case__ )

													SCREAMING_SNAKE_CASE_   : List[Any]					  =      F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
													SCREAMING_SNAKE_CASE_   : List[Any]					  =      tokenizer_r.encode_plus(
													    snake_case__      ,return_attention_mask=snake_case__      ,return_token_type_ids=snake_case__      ,return_offsets_mapping=snake_case__      ,add_special_tokens=snake_case__      ,)

													SCREAMING_SNAKE_CASE_   : Any					  =      tokenizer_r.do_lower_case if hasattr(snake_case__      ,'do_lower_case' ) else False
													SCREAMING_SNAKE_CASE_   : Any					  =      (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), 'A'),
													        ((1, 2), ','),
													        ((3, 5), 'na'),
													        ((5, 6), '##ï'),
													        ((6, 8), '##ve'),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), 'Allen'),
													        ((21, 23), '##NL'),
													        ((23, 24), '##P'),
													        ((25, 33), 'sentence'),
													        ((33, 34), '.'),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), 'a'),
													        ((1, 2), ','),
													        ((3, 8), 'naive'),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), 'allen'),
													        ((21, 23), '##nl'),
													        ((23, 24), '##p'),
													        ((25, 33), 'sentence'),
													        ((33, 34), '.'),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]      ,tokenizer_r.convert_ids_to_tokens(tokens['input_ids'] ) )
													self.assertEqual([e[0] for e in expected_results]      ,tokens['offset_mapping'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Dict					  =      ['的', '人', '有']
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      ''.join(snake_case__ )
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})' ):
													SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      True
													SCREAMING_SNAKE_CASE_   : Dict					  =      self.tokenizer_class.from_pretrained(snake_case__      ,**snake_case__ )
													SCREAMING_SNAKE_CASE_   : str					  =      self.rust_tokenizer_class.from_pretrained(snake_case__      ,**snake_case__ )

													SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      tokenizer_p.encode(snake_case__      ,add_special_tokens=snake_case__ )
													SCREAMING_SNAKE_CASE_   : Optional[int]					  =      tokenizer_r.encode(snake_case__      ,add_special_tokens=snake_case__ )

													SCREAMING_SNAKE_CASE_   : Optional[int]					  =      tokenizer_r.convert_ids_to_tokens(snake_case__ )
													SCREAMING_SNAKE_CASE_   : Tuple					  =      tokenizer_p.convert_ids_to_tokens(snake_case__ )

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(snake_case__      ,snake_case__ )
													self.assertListEqual(snake_case__      ,snake_case__ )

													SCREAMING_SNAKE_CASE_   : Optional[int]					  =      False
													SCREAMING_SNAKE_CASE_   : Dict					  =      self.rust_tokenizer_class.from_pretrained(snake_case__      ,**snake_case__ )
													SCREAMING_SNAKE_CASE_   : List[Any]					  =      self.tokenizer_class.from_pretrained(snake_case__      ,**snake_case__ )

													SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      tokenizer_r.encode(snake_case__      ,add_special_tokens=snake_case__ )
													SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      tokenizer_p.encode(snake_case__      ,add_special_tokens=snake_case__ )

													SCREAMING_SNAKE_CASE_   : int					  =      tokenizer_r.convert_ids_to_tokens(snake_case__ )
													SCREAMING_SNAKE_CASE_   : Optional[int]					  =      tokenizer_p.convert_ids_to_tokens(snake_case__ )

													# it is expected that only the first Chinese character is not preceded by "##".
													SCREAMING_SNAKE_CASE_   : List[Any]					  =      [
													    F'##{token}' if idx != 0 else token for idx, token in enumerate(snake_case__ )
													]
													self.assertListEqual(snake_case__      ,snake_case__ )
													self.assertListEqual(snake_case__      ,snake_case__ )








import tempfile
import unittest

from transformers import TaConfig, is_torch_available
from transformers.testing_utils import (
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
    torch_device,
)

from ...generation.test_utils import GenerationTesterMixin
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import AutoTokenizer, UMTaForConditionalGeneration, UMTaForQuestionAnswering, UMTaModel



class 			__lowerCAmelCase  :




 def __init__(    self						,					__UpperCAmelCase						,					__UpperCAmelCase=99						,					__UpperCAmelCase=13						,					__UpperCAmelCase=7						,					__UpperCAmelCase=9						,					__UpperCAmelCase=True						,					__UpperCAmelCase=True						,					__UpperCAmelCase=False						,					__UpperCAmelCase=32						,					__UpperCAmelCase=5						,					__UpperCAmelCase=4						,					__UpperCAmelCase=37						,					__UpperCAmelCase=8						,					__UpperCAmelCase=0.1						,					__UpperCAmelCase=0.002						,					__UpperCAmelCase=1						,					__UpperCAmelCase=0						,					__UpperCAmelCase=0						,					__UpperCAmelCase=None						,					__UpperCAmelCase=None						,					):



    '''simple docstring'''
    __lowerCamelCase    				=     parent
    __lowerCamelCase    				=     batch_size
    __lowerCamelCase    				=     encoder_seq_length
    __lowerCamelCase    				=     decoder_seq_length
    # For common tests
    __lowerCamelCase    				=     self.decoder_seq_length
    __lowerCamelCase    				=     is_training
    __lowerCamelCase    				=     use_attention_mask
    __lowerCamelCase    				=     use_labels
    __lowerCamelCase    				=     vocab_size
    __lowerCamelCase    				=     hidden_size
    __lowerCamelCase    				=     num_hidden_layers
    __lowerCamelCase    				=     num_attention_heads
    __lowerCamelCase    				=     d_ff
    __lowerCamelCase    				=     relative_attention_num_buckets
    __lowerCamelCase    				=     dropout_rate
    __lowerCamelCase    				=     initializer_factor
    __lowerCamelCase    				=     eos_token_id
    __lowerCamelCase    				=     pad_token_id
    __lowerCamelCase    				=     decoder_start_token_id
    __lowerCamelCase    				=     None
    __lowerCamelCase    				=     decoder_layers




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    return TaConfig.from_pretrained('''google/umt5-base'''					)




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase						,					__UpperCAmelCase						,					__UpperCAmelCase						,					__UpperCAmelCase=None						,					__UpperCAmelCase=None						,					__UpperCAmelCase=None						,					__UpperCAmelCase=None						,					__UpperCAmelCase=None						,					):



    '''simple docstring'''
    if attention_mask is None:
       __lowerCamelCase    				=     input_ids.ne(config.pad_token_id					)
    if decoder_attention_mask is None:
       __lowerCamelCase    				=     decoder_input_ids.ne(config.pad_token_id					)
    if head_mask is None:
       __lowerCamelCase    				=     torch.ones(config.num_hidden_layers						,					config.num_attention_heads						,					device=__UpperCAmelCase					)
    if decoder_head_mask is None:
       __lowerCamelCase    				=     torch.ones(config.num_decoder_layers						,					config.num_attention_heads						,					device=__UpperCAmelCase					)
    if cross_attn_head_mask is None:
       __lowerCamelCase    				=     torch.ones(
           config.num_decoder_layers						,					config.num_attention_heads						,					device=__UpperCAmelCase					)
    return {
        "input_ids": input_ids,
        "decoder_input_ids": decoder_input_ids,
        "attention_mask": attention_mask,
        "decoder_attention_mask": decoder_attention_mask,
        "head_mask": head_mask,
        "decoder_head_mask": decoder_head_mask,
        "cross_attn_head_mask": cross_attn_head_mask,
    }




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     ids_tensor([self.batch_size, self.encoder_seq_length]						,					self.vocab_size					)
    __lowerCamelCase    				=     ids_tensor([self.batch_size, self.decoder_seq_length]						,					self.vocab_size					)

    # we need to clamp the input ids here to avoid having pad token in between
    # this is because for NllbMoe the position_ids are prepared such that
    # all pad tokens have pos id = 2 and rest are between 2..seq_length
    # and the seq_length here is seq_length - num_pad_tokens
    # but when using past, there is no way of knowing if the past input ids had
    # pad tokens in them, which results in incorrect seq_lenth and which in turn results in
    # position_ids being off by num_pad_tokens in past input
    __lowerCamelCase    				=     input_ids.clamp(self.pad_token_id + 1					)
    __lowerCamelCase    				=     decoder_input_ids.clamp(self.pad_token_id + 1					)

    __lowerCamelCase    				=     self.get_config()
    __lowerCamelCase    				=     config.num_attention_heads
    __lowerCamelCase    				=     self.prepare_inputs_dict(__UpperCAmelCase						,					__UpperCAmelCase						,					__UpperCAmelCase					)
    return config, input_dict




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase				,__lowerCamelCase    				=     self.prepare_config_and_inputs()
    return config, inputs_dict




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    return TaConfig(
        vocab_size=166						,					d_model=self.hidden_size						,					d_ff=self.d_ff						,					d_kv=self.hidden_size // self.num_attention_heads						,					num_layers=self.num_hidden_layers						,					num_decoder_layers=self.decoder_layers						,					num_heads=self.num_attention_heads						,					relative_attention_num_buckets=self.relative_attention_num_buckets						,					dropout_rate=self.dropout_rate						,					initializer_factor=self.initializer_factor						,					eos_token_id=self.eos_token_id						,					bos_token_id=self.pad_token_id						,					pad_token_id=self.pad_token_id						,					decoder_start_token_id=self.decoder_start_token_id						,					)




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    return TaConfig(
        vocab_size=self.vocab_size						,					d_model=self.hidden_size						,					d_ff=self.d_ff						,					d_kv=self.hidden_size // self.num_attention_heads						,					num_layers=self.num_hidden_layers						,					num_decoder_layers=self.decoder_layers						,					num_heads=self.num_attention_heads						,					relative_attention_num_buckets=self.relative_attention_num_buckets						,					dropout_rate=self.dropout_rate						,					initializer_factor=self.initializer_factor						,					eos_token_id=self.eos_token_id						,					bos_token_id=self.pad_token_id						,					pad_token_id=self.pad_token_id						,					decoder_start_token_id=self.decoder_start_token_id						,					)




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase						,					__UpperCAmelCase						,					__UpperCAmelCase						,					__UpperCAmelCase						,					__UpperCAmelCase						,					__UpperCAmelCase						,					):



    '''simple docstring'''
    __lowerCamelCase    				=     UMTaModel(config=__UpperCAmelCase					)
    model.to(__UpperCAmelCase					)
    model.eval()
    __lowerCamelCase    				=     model(
        input_ids=__UpperCAmelCase						,					decoder_input_ids=__UpperCAmelCase						,					attention_mask=__UpperCAmelCase						,					decoder_attention_mask=__UpperCAmelCase						,					)
    __lowerCamelCase    				=     model(input_ids=__UpperCAmelCase						,					decoder_input_ids=__UpperCAmelCase					)
    __lowerCamelCase    				=     result.last_hidden_state
    __lowerCamelCase    				=     result.past_key_values
    __lowerCamelCase    				=     result.encoder_last_hidden_state

    self.parent.assertEqual(encoder_output.size()						,					(self.batch_size, self.encoder_seq_length, self.hidden_size)					)
    self.parent.assertEqual(decoder_output.size()						,					(self.batch_size, self.decoder_seq_length, self.hidden_size)					)
    # There should be `num_layers` key value embeddings stored in decoder_past
    self.parent.assertEqual(len(__UpperCAmelCase					)						,					config.num_layers					)
    # There should be a self attn key, a self attn value, a cross attn key and a cross attn value stored in each decoder_past tuple
    self.parent.assertEqual(len(decoder_past[0]					)						,					4					)




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase						,					__UpperCAmelCase						,					__UpperCAmelCase						,					__UpperCAmelCase						,					__UpperCAmelCase						,					__UpperCAmelCase						,					):



    '''simple docstring'''
    __lowerCamelCase    				=     UMTaModel(config=__UpperCAmelCase					).get_decoder().to(__UpperCAmelCase					).eval()
    # first forward pass
    __lowerCamelCase    				=     model(__UpperCAmelCase						,					use_cache=__UpperCAmelCase					)
    __lowerCamelCase    				=     model(__UpperCAmelCase					)
    __lowerCamelCase    				=     model(__UpperCAmelCase						,					use_cache=__UpperCAmelCase					)

    self.parent.assertTrue(len(__UpperCAmelCase					) == len(__UpperCAmelCase					)					)
    self.parent.assertTrue(len(__UpperCAmelCase					) == len(__UpperCAmelCase					) + 1					)

    __lowerCamelCase				,__lowerCamelCase    				=     outputs.to_tuple()

    # create hypothetical next token and extent to next_input_ids
    __lowerCamelCase    				=     ids_tensor((self.batch_size, 1)						,					config.vocab_size					)

    # append to next input_ids and
    __lowerCamelCase    				=     torch.cat([input_ids, next_tokens]						,					dim=-1					)

    __lowerCamelCase    				=     model(__UpperCAmelCase					)['''last_hidden_state''']
    __lowerCamelCase    				=     model(__UpperCAmelCase						,					past_key_values=__UpperCAmelCase					)['''last_hidden_state''']

    # select random slice
    __lowerCamelCase    				=     ids_tensor((1,)						,					output_from_past.shape[-1]					).item()
    __lowerCamelCase    				=     output_from_no_past[:, -1, random_slice_idx].detach()
    __lowerCamelCase    				=     output_from_past[:, 0, random_slice_idx].detach()

    # test that outputs are equal for slice
    self.parent.assertTrue(torch.allclose(__UpperCAmelCase						,					__UpperCAmelCase						,					atol=1E-3					)					)




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase						,					__UpperCAmelCase						,					):



    '''simple docstring'''
    __lowerCamelCase    				=     UMTaModel(config=__UpperCAmelCase					).to(__UpperCAmelCase					).half().eval()
    __lowerCamelCase    				=     model(**__UpperCAmelCase					)['''last_hidden_state''']
    self.parent.assertFalse(torch.isnan(__UpperCAmelCase					).any().item()					)



@require_torch
class 			__lowerCAmelCase  (						lowerCAmelCase__	,						lowerCAmelCase__	,						lowerCAmelCase__	,						unittest.TestCase ):
 lowerCAmelCase__							       =	(
     (UMTaModel, UMTaForConditionalGeneration, UMTaForQuestionAnswering) if is_torch_available() else ()
 )
 lowerCAmelCase__							       =	(UMTaForConditionalGeneration,) if is_torch_available() else ()
 lowerCAmelCase__							       =	(
     {
         """conversational""": UMTaForConditionalGeneration,
         """feature-extraction""": UMTaModel,
         """summarization""": UMTaForConditionalGeneration,
         """text2text-generation""": UMTaForConditionalGeneration,
         """translation""": UMTaForConditionalGeneration,
         """question-answering""": UMTaForQuestionAnswering,
     }
     if is_torch_available()
     else {}
 )
 lowerCAmelCase__							       =	True
 lowerCAmelCase__							       =	False
 lowerCAmelCase__							       =	False
 lowerCAmelCase__							       =	True
 lowerCAmelCase__							       =	True
 # The small UMT5 model needs higher percentages for CPU/MP tests
 lowerCAmelCase__							       =	[0.8, 0.9]




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     UMTaModelTester(self					)




 @unittest.skip('''Test has a segmentation fault on torch 1.8.0'''					)
 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.model_tester.prepare_config_and_inputs()
    __lowerCamelCase    				=     UMTaModel(config_and_inputs[0]					).to(__UpperCAmelCase					)
    with tempfile.TemporaryDirectory() as tmpdirname:
       torch.onnx.export(
           __UpperCAmelCase						,					(config_and_inputs[1], config_and_inputs[3], config_and_inputs[2])						,					F"""{tmpdirname}/t5_test.onnx"""						,					export_params=__UpperCAmelCase						,					opset_version=9						,					input_names=['''input_ids''', '''decoder_input_ids''']						,					)




 @unittest.skipIf(torch_device == '''cpu'''						,					'''Cant do half precision'''					)
 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_model_fpaa_forward(*__UpperCAmelCase					)




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     ['''encoder_attentions''', '''decoder_attentions''', '''cross_attentions''']
    __lowerCamelCase    				=     self.model_tester.prepare_config_and_inputs()
    __lowerCamelCase    				=     config_and_inputs[0]
    __lowerCamelCase    				=     UMTaForConditionalGeneration(__UpperCAmelCase					).eval()
    model.to(__UpperCAmelCase					)

    __lowerCamelCase    				=     {
        '''head_mask''': torch.zeros(config.num_layers						,					config.num_heads						,					device=__UpperCAmelCase					),
        '''decoder_head_mask''': torch.zeros(config.num_decoder_layers						,					config.num_heads						,					device=__UpperCAmelCase					),
        '''cross_attn_head_mask''': torch.zeros(config.num_decoder_layers						,					config.num_heads						,					device=__UpperCAmelCase					),
    }

    for attn_name, (name, mask) in zip(__UpperCAmelCase						,					head_masking.items()					):
       __lowerCamelCase    				=     {name: mask}
       # Explicitly pass decoder_head_mask as it is required from T5 model when head_mask specified
       if name == "head_mask":
          __lowerCamelCase    				=     torch.ones(
              config.num_decoder_layers						,					config.num_heads						,					device=__UpperCAmelCase					)

       __lowerCamelCase    				=     model.generate(
           config_and_inputs[1]['''input_ids''']						,					num_beams=1						,					max_length=3						,					output_attentions=__UpperCAmelCase						,					return_dict_in_generate=__UpperCAmelCase						,					**__UpperCAmelCase						,					)
       # We check the state of decoder_attentions and cross_attentions just from the last step
       __lowerCamelCase    				=     out[attn_name] if attn_name == attention_names[0] else out[attn_name][-1]
       self.assertEqual(sum([w.sum().item() for w in attn_weights]					)						,					0.0					)




 @unittest.skip('''Does not work on the tiny model as we keep hitting edge cases.'''					)
 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    pass



@require_torch
@require_sentencepiece
@require_tokenizers
class 			__lowerCAmelCase  (						unittest.TestCase ):




 @slow
 @unittest.skip(
     '''Unless we stop stripping left and right by default for all special tokens, the expected ids obtained here will not match the original ones. Wait for https://github.com/huggingface/transformers/pull/23909 to be merged'''					)
 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     UMTaForConditionalGeneration.from_pretrained('''google/umt5-small'''						,					return_dict=__UpperCAmelCase					).to(__UpperCAmelCase					)
    __lowerCamelCase    				=     AutoTokenizer.from_pretrained('''google/umt5-small'''						,					use_fast=__UpperCAmelCase						,					legacy=__UpperCAmelCase					)
    __lowerCamelCase    				=     [
        '''Bonjour monsieur <extra_id_0> bien <extra_id_1>.''',
        '''No se como puedo <extra_id_0>.''',
        '''This is the reason why we <extra_id_0> them.''',
        '''The <extra_id_0> walks in <extra_id_1>, seats''',
        '''A <extra_id_0> walks into a bar and orders a <extra_id_1> with <extra_id_2> pinch of <extra_id_3>.''',
    ]
    __lowerCamelCase    				=     tokenizer(__UpperCAmelCase						,					return_tensors='''pt'''						,					padding=__UpperCAmelCase					).input_ids
    # fmt: off
    __lowerCamelCase    				=     torch.tensor(
        [
            [ 38530, 210703, 256299, 1410, 256298, 274, 1, 0,0, 0, 0, 0, 0, 0, 0, 0,0, 0],
            [   826, 321, 671, 25922, 256299, 274, 1, 0,0, 0, 0, 0, 0, 0, 0, 0,0, 0],
            [  1460, 339, 312, 19014, 10620, 758, 256299, 2355,274, 1, 0, 0, 0, 0, 0, 0,0, 0],
            [   517, 256299, 14869, 281, 301, 256298, 275, 119983,1, 0, 0, 0, 0, 0, 0, 0,0, 0],
            [   320, 256299, 14869, 281, 2234, 289, 2275, 333,61391, 289, 256298, 543, 256297, 168714, 329, 256296,274, 1],
        ]					)
    # fmt: on
    torch.testing.assert_allclose(__UpperCAmelCase						,					__UpperCAmelCase					)

    __lowerCamelCase    				=     model.generate(input_ids.to(__UpperCAmelCase					)					)
    __lowerCamelCase    				=     [
        '''<pad><extra_id_0> et<extra_id_1> [eod] <extra_id_2><extra_id_55>.. [eod] 💐 💐 💐 💐 💐 💐 💐 💐 💐 💐 💐 <extra_id_56>ajšietosto<extra_id_56>lleux<extra_id_19><extra_id_6>ajšie</s>''',
        '''<pad><extra_id_0>.<extra_id_1>.,<0x0A>...spech <0x0A><extra_id_20> <extra_id_21></s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>''',
        '''<pad><extra_id_0> are not going to be a part of the world. We are not going to be a part of<extra_id_1> and<extra_id_2><0x0A><extra_id_48>.<extra_id_48></s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>''',
        '''<pad><extra_id_0> door<extra_id_1>, the door<extra_id_2> 피해[/</s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>''',
        '''<pad><extra_id_0>nyone who<extra_id_1> drink<extra_id_2> a<extra_id_3> alcohol<extra_id_4> A<extra_id_5> A. This<extra_id_6> I<extra_id_7><extra_id_52><extra_id_53></s><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>''',
    ]
    __lowerCamelCase    				=     tokenizer.batch_decode(__UpperCAmelCase					)
    self.assertEqual(__UpperCAmelCase						,					__UpperCAmelCase					)






