



"""simple docstring"""







import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class      UpperCAmelCase__		(      UpperCamelCase   ,unittest.TestCase     ):
		lowerCAmelCase_       :	int 							=     MobileBertTokenizer
		lowerCAmelCase_       :	List[Any] 							=     MobileBertTokenizerFast
		lowerCAmelCase_       :	Any 							=     True
		lowerCAmelCase_       :	Union[str, Any] 							=     True
		lowerCAmelCase_       :	Optional[Any] 							=     filter_non_english
		lowerCAmelCase_       :	Any 							=     """google/mobilebert-uncased"""



		def 					A_		( self  :    Union[str, Any]				)			->						Optional[Any]:






									'''simple docstring'''

									super().setUp()

									A         =					[
									    '[UNK]',
									    '[CLS]',
									    '[SEP]',
									    '[PAD]',
									    '[MASK]',
									    'want',
									    '##want',
									    '##ed',
									    'wa',
									    'un',
									    'runn',
									    '##ing',
									    ',',
									    'low',
									    'lowest',
									]
									A         =					os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES['vocab_file']				)
									with open(self.vocab_file      ,							'w'      ,							encoding='utf-8'				) as vocab_writer:
																vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]				)				)

									A         =					[
									    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
									    for tokenizer_def in self.tokenizers_list
									]



		def 					A_		( self  :    List[Any]      ,							snake_case  :    Optional[Any]				)			->						Dict:






									'''simple docstring'''

									A         =					'UNwant\u00E9d,running'
									A         =					'unwanted, running'
									return input_text, output_text



		def 					A_		( self  :    Tuple				)			->						List[Any]:






									'''simple docstring'''

									A         =					self.tokenizer_class(self.vocab_file				)

									A         =					tokenizer.tokenize('UNwant\u00E9d,running'				)
									self.assertListEqual(snake_case      ,							['un', '##want', '##ed', ',', 'runn', '##ing']				)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case				)      ,							[9, 6, 7, 12, 10, 11]				)



		def 					A_		( self  :    List[Any]				)			->						Any:






									'''simple docstring'''

									if not self.test_rust_tokenizer:
																return

									A         =					self.get_tokenizer()
									A         =					self.get_rust_tokenizer()

									A         =					'UNwant\u00E9d,running'

									A         =					tokenizer.tokenize(snake_case				)
									A         =					rust_tokenizer.tokenize(snake_case				)
									self.assertListEqual(snake_case      ,							snake_case				)

									A         =					tokenizer.encode(snake_case      ,							add_special_tokens=snake_case				)
									A         =					rust_tokenizer.encode(snake_case      ,							add_special_tokens=snake_case				)
									self.assertListEqual(snake_case      ,							snake_case				)

									A         =					self.get_rust_tokenizer()
									A         =					tokenizer.encode(snake_case				)
									A         =					rust_tokenizer.encode(snake_case				)
									self.assertListEqual(snake_case      ,							snake_case				)

									# With lower casing
									A         =					self.get_tokenizer(do_lower_case=snake_case				)
									A         =					self.get_rust_tokenizer(do_lower_case=snake_case				)

									A         =					'UNwant\u00E9d,running'

									A         =					tokenizer.tokenize(snake_case				)
									A         =					rust_tokenizer.tokenize(snake_case				)
									self.assertListEqual(snake_case      ,							snake_case				)

									A         =					tokenizer.encode(snake_case      ,							add_special_tokens=snake_case				)
									A         =					rust_tokenizer.encode(snake_case      ,							add_special_tokens=snake_case				)
									self.assertListEqual(snake_case      ,							snake_case				)

									A         =					self.get_rust_tokenizer()
									A         =					tokenizer.encode(snake_case				)
									A         =					rust_tokenizer.encode(snake_case				)
									self.assertListEqual(snake_case      ,							snake_case				)



		def 					A_		( self  :    Dict				)			->						List[Any]:






									'''simple docstring'''

									A         =					BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'				)      ,							['ah', '\u535A', '\u63A8', 'zz']				)



		def 					A_		( self  :    Union[str, Any]				)			->						Optional[int]:






									'''simple docstring'''

									A         =					BasicTokenizer(do_lower_case=snake_case				)

									self.assertListEqual(
									    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '				)      ,							['hello', '!', 'how', 'are', 'you', '?']				)
									self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				)      ,							['hello']				)



		def 					A_		( self  :    Dict				)			->						Dict:






									'''simple docstring'''

									A         =					BasicTokenizer(do_lower_case=snake_case      ,							strip_accents=snake_case				)

									self.assertListEqual(
									    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				)      ,							['hällo', '!', 'how', 'are', 'you', '?']				)
									self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				)      ,							['h\u00E9llo']				)



		def 					A_		( self  :    Tuple				)			->						int:






									'''simple docstring'''

									A         =					BasicTokenizer(do_lower_case=snake_case      ,							strip_accents=snake_case				)

									self.assertListEqual(
									    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				)      ,							['hallo', '!', 'how', 'are', 'you', '?']				)
									self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				)      ,							['hello']				)



		def 					A_		( self  :    str				)			->						Tuple:






									'''simple docstring'''

									A         =					BasicTokenizer(do_lower_case=snake_case				)

									self.assertListEqual(
									    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				)      ,							['hallo', '!', 'how', 'are', 'you', '?']				)
									self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				)      ,							['hello']				)



		def 					A_		( self  :    str				)			->						Union[str, Any]:






									'''simple docstring'''

									A         =					BasicTokenizer(do_lower_case=snake_case				)

									self.assertListEqual(
									    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '				)      ,							['HeLLo', '!', 'how', 'Are', 'yoU', '?']				)



		def 					A_		( self  :    List[str]				)			->						List[Any]:






									'''simple docstring'''

									A         =					BasicTokenizer(do_lower_case=snake_case      ,							strip_accents=snake_case				)

									self.assertListEqual(
									    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				)      ,							['HäLLo', '!', 'how', 'Are', 'yoU', '?']				)



		def 					A_		( self  :    Tuple				)			->						Any:






									'''simple docstring'''

									A         =					BasicTokenizer(do_lower_case=snake_case      ,							strip_accents=snake_case				)

									self.assertListEqual(
									    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				)      ,							['HaLLo', '!', 'how', 'Are', 'yoU', '?']				)



		def 					A_		( self  :    Dict				)			->						Optional[int]:






									'''simple docstring'''

									A         =					BasicTokenizer(do_lower_case=snake_case      ,							never_split=['[UNK]']				)

									self.assertListEqual(
									    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'				)      ,							['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']				)



		def 					A_		( self  :    Optional[Any]				)			->						Optional[Any]:






									'''simple docstring'''

									A         =					['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

									A         =					{}
									for i, token in enumerate(snake_case				):
																A         =					i
									A         =					WordpieceTokenizer(vocab=snake_case      ,							unk_token='[UNK]'				)

									self.assertListEqual(tokenizer.tokenize(''				)      ,							[]				)

									self.assertListEqual(tokenizer.tokenize('unwanted running'				)      ,							['un', '##want', '##ed', 'runn', '##ing']				)

									self.assertListEqual(tokenizer.tokenize('unwantedX running'				)      ,							['[UNK]', 'runn', '##ing']				)



		def 					A_		( self  :    Any				)			->						int:






									'''simple docstring'''

									self.assertTrue(_is_whitespace(' '				)				)
									self.assertTrue(_is_whitespace('\t'				)				)
									self.assertTrue(_is_whitespace('\r'				)				)
									self.assertTrue(_is_whitespace('\n'				)				)
									self.assertTrue(_is_whitespace('\u00A0'				)				)

									self.assertFalse(_is_whitespace('A'				)				)
									self.assertFalse(_is_whitespace('-'				)				)



		def 					A_		( self  :    Optional[int]				)			->						Union[str, Any]:






									'''simple docstring'''

									self.assertTrue(_is_control('\u0005'				)				)

									self.assertFalse(_is_control('A'				)				)
									self.assertFalse(_is_control(' '				)				)
									self.assertFalse(_is_control('\t'				)				)
									self.assertFalse(_is_control('\r'				)				)



		def 					A_		( self  :    Any				)			->						Dict:






									'''simple docstring'''

									self.assertTrue(_is_punctuation('-'				)				)
									self.assertTrue(_is_punctuation('$'				)				)
									self.assertTrue(_is_punctuation('`'				)				)
									self.assertTrue(_is_punctuation('.'				)				)

									self.assertFalse(_is_punctuation('A'				)				)
									self.assertFalse(_is_punctuation(' '				)				)



		def 					A_		( self  :    Optional[int]				)			->						Optional[Any]:






									'''simple docstring'''

									A         =					self.get_tokenizer()
									A         =					self.get_rust_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(snake_case				) for t in ['Test', '\xad', 'test']]      ,							[['[UNK]'], [], ['[UNK]']]				)

									self.assertListEqual(
									    [rust_tokenizer.tokenize(snake_case				) for t in ['Test', '\xad', 'test']]      ,							[['[UNK]'], [], ['[UNK]']]				)



		@slow
		def 					A_		( self  :    str				)			->						Any:






									'''simple docstring'''

									A         =					self.tokenizer_class.from_pretrained('google/mobilebert-uncased'				)

									A         =					tokenizer.encode('sequence builders'      ,							add_special_tokens=snake_case				)
									A         =					tokenizer.encode('multi-sequence build'      ,							add_special_tokens=snake_case				)

									A         =					tokenizer.build_inputs_with_special_tokens(snake_case				)
									A         =					tokenizer.build_inputs_with_special_tokens(snake_case      ,							snake_case				)

									assert encoded_sentence == [101] + text + [102]
									assert encoded_pair == [101] + text + [102] + text_a + [102]



		def 					A_		( self  :    Optional[Any]				)			->						List[Any]:






									'''simple docstring'''

									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
																							A         =					self.rust_tokenizer_class.from_pretrained(snake_case      ,							**snake_case				)

																							A         =					f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																							A         =					tokenizer_r.encode_plus(
																							    snake_case      ,							return_attention_mask=snake_case      ,							return_token_type_ids=snake_case      ,							return_offsets_mapping=snake_case      ,							add_special_tokens=snake_case      ,							)

																							A         =					tokenizer_r.do_lower_case if hasattr(snake_case      ,							'do_lower_case'				) else False
																							A         =					(
																							    [
																							        ((0, 0), tokenizer_r.cls_token),
																							        ((0, 1), 'A'),
																							        ((1, 2), ','),
																							        ((3, 5), 'na'),
																							        ((5, 6), '##ï'),
																							        ((6, 8), '##ve'),
																							        ((9, 15), tokenizer_r.mask_token),
																							        ((16, 21), 'Allen'),
																							        ((21, 23), '##NL'),
																							        ((23, 24), '##P'),
																							        ((25, 33), 'sentence'),
																							        ((33, 34), '.'),
																							        ((0, 0), tokenizer_r.sep_token),
																							    ]
																							    if not do_lower_case
																							    else [
																							        ((0, 0), tokenizer_r.cls_token),
																							        ((0, 1), 'a'),
																							        ((1, 2), ','),
																							        ((3, 8), 'naive'),
																							        ((9, 15), tokenizer_r.mask_token),
																							        ((16, 21), 'allen'),
																							        ((21, 23), '##nl'),
																							        ((23, 24), '##p'),
																							        ((25, 33), 'sentence'),
																							        ((33, 34), '.'),
																							        ((0, 0), tokenizer_r.sep_token),
																							    ]
																							)

																							self.assertEqual(
																							    [e[1] for e in expected_results]      ,							tokenizer_r.convert_ids_to_tokens(tokens['input_ids']				)				)
																							self.assertEqual([e[0] for e in expected_results]      ,							tokens['offset_mapping']				)



		def 					A_		( self  :    Dict				)			->						int:






									'''simple docstring'''

									A         =					['的', '人', '有']
									A         =					''.join(snake_case				)
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
																							A         =					True
																							A         =					self.tokenizer_class.from_pretrained(snake_case      ,							**snake_case				)
																							A         =					self.rust_tokenizer_class.from_pretrained(snake_case      ,							**snake_case				)

																							A         =					tokenizer_p.encode(snake_case      ,							add_special_tokens=snake_case				)
																							A         =					tokenizer_r.encode(snake_case      ,							add_special_tokens=snake_case				)

																							A         =					tokenizer_r.convert_ids_to_tokens(snake_case				)
																							A         =					tokenizer_p.convert_ids_to_tokens(snake_case				)

																							# it is expected that each Chinese character is not preceded by "##"
																							self.assertListEqual(snake_case      ,							snake_case				)
																							self.assertListEqual(snake_case      ,							snake_case				)

																							A         =					False
																							A         =					self.rust_tokenizer_class.from_pretrained(snake_case      ,							**snake_case				)
																							A         =					self.tokenizer_class.from_pretrained(snake_case      ,							**snake_case				)

																							A         =					tokenizer_r.encode(snake_case      ,							add_special_tokens=snake_case				)
																							A         =					tokenizer_p.encode(snake_case      ,							add_special_tokens=snake_case				)

																							A         =					tokenizer_r.convert_ids_to_tokens(snake_case				)
																							A         =					tokenizer_p.convert_ids_to_tokens(snake_case				)

																							# it is expected that only the first Chinese character is not preceded by "##".
																							A         =					[
																							    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(snake_case				)
																							]
																							self.assertListEqual(snake_case      ,							snake_case				)
																							self.assertListEqual(snake_case      ,							snake_case				)








"""simple docstring"""







import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY






@is_pipeline_test
class      UpperCAmelCase__		(      unittest.TestCase     ):
		lowerCAmelCase_       :	int 							=     MODEL_FOR_MASKED_LM_MAPPING
		lowerCAmelCase_       :	Tuple 							=     TF_MODEL_FOR_MASKED_LM_MAPPING



		def 					A_		( self  :    Tuple				)			->						int:






									'''simple docstring'''

									super().tearDown()
									# clean-up as much as possible GPU memory occupied by PyTorch
									gc.collect()
									if is_torch_available():
																import torch

																torch.cuda.empty_cache()



		@require_tf
		def 					A_		( self  :    int				)			->						Optional[int]:






									'''simple docstring'''

									A         =					pipeline(task='fill-mask'      ,							model='sshleifer/tiny-distilroberta-base'      ,							top_k=2      ,							framework='tf'				)
									A         =					unmasker('My name is <mask>'				)
									self.assertEqual(
									    nested_simplify(snake_case      ,							decimals=6				)      ,							[
									        {'sequence': 'My name is grouped', 'score': 2.1E-0_5, 'token': 38_015, 'token_str': ' grouped'},
									        {'sequence': 'My name is accuser', 'score': 2.1E-0_5, 'token': 25_506, 'token_str': ' accuser'},
									    ]      ,							)

									A         =					unmasker('The largest city in France is <mask>'				)
									self.assertEqual(
									    nested_simplify(snake_case      ,							decimals=6				)      ,							[
									        {
									            'sequence': 'The largest city in France is grouped',
									            'score': 2.1E-0_5,
									            'token': 38_015,
									            'token_str': ' grouped',
									        },
									        {
									            'sequence': 'The largest city in France is accuser',
									            'score': 2.1E-0_5,
									            'token': 25_506,
									            'token_str': ' accuser',
									        },
									    ]      ,							)

									A         =					unmasker('My name is <mask>'      ,							targets=[' Patrick', ' Clara', ' Teven']      ,							top_k=3				)
									self.assertEqual(
									    nested_simplify(snake_case      ,							decimals=6				)      ,							[
									        {'sequence': 'My name is Clara', 'score': 2E-0_5, 'token': 13_606, 'token_str': ' Clara'},
									        {'sequence': 'My name is Patrick', 'score': 2E-0_5, 'token': 3_499, 'token_str': ' Patrick'},
									        {'sequence': 'My name is Te', 'score': 1.9E-0_5, 'token': 2_941, 'token_str': ' Te'},
									    ]      ,							)



		@require_torch
		def 					A_		( self  :    str				)			->						int:






									'''simple docstring'''

									A         =					pipeline(task='fill-mask'      ,							model='sshleifer/tiny-distilroberta-base'      ,							top_k=2      ,							framework='pt'				)

									A         =					unmasker('My name is <mask>'				)
									self.assertEqual(
									    nested_simplify(snake_case      ,							decimals=6				)      ,							[
									        {'sequence': 'My name is Maul', 'score': 2.2E-0_5, 'token': 35_676, 'token_str': ' Maul'},
									        {'sequence': 'My name isELS', 'score': 2.2E-0_5, 'token': 16_416, 'token_str': 'ELS'},
									    ]      ,							)

									A         =					unmasker('The largest city in France is <mask>'				)
									self.assertEqual(
									    nested_simplify(snake_case      ,							decimals=6				)      ,							[
									        {
									            'sequence': 'The largest city in France is Maul',
									            'score': 2.2E-0_5,
									            'token': 35_676,
									            'token_str': ' Maul',
									        },
									        {'sequence': 'The largest city in France isELS', 'score': 2.2E-0_5, 'token': 16_416, 'token_str': 'ELS'},
									    ]      ,							)

									A         =					unmasker('My name is <mask>'      ,							targets=[' Patrick', ' Clara', ' Teven']      ,							top_k=3				)
									self.assertEqual(
									    nested_simplify(snake_case      ,							decimals=6				)      ,							[
									        {'sequence': 'My name is Patrick', 'score': 2.1E-0_5, 'token': 3_499, 'token_str': ' Patrick'},
									        {'sequence': 'My name is Te', 'score': 2E-0_5, 'token': 2_941, 'token_str': ' Te'},
									        {'sequence': 'My name is Clara', 'score': 2E-0_5, 'token': 13_606, 'token_str': ' Clara'},
									    ]      ,							)

									A         =					unmasker('My name is <mask> <mask>'      ,							top_k=2				)

									self.assertEqual(
									    nested_simplify(snake_case      ,							decimals=6				)      ,							[
									        [
									            {
									                'score': 2.2E-0_5,
									                'token': 35_676,
									                'token_str': ' Maul',
									                'sequence': '<s>My name is Maul<mask></s>',
									            },
									            {'score': 2.2E-0_5, 'token': 16_416, 'token_str': 'ELS', 'sequence': '<s>My name isELS<mask></s>'},
									        ],
									        [
									            {
									                'score': 2.2E-0_5,
									                'token': 35_676,
									                'token_str': ' Maul',
									                'sequence': '<s>My name is<mask> Maul</s>',
									            },
									            {'score': 2.2E-0_5, 'token': 16_416, 'token_str': 'ELS', 'sequence': '<s>My name is<mask>ELS</s>'},
									        ],
									    ]      ,							)



		@require_torch_gpu
		def 					A_		( self  :    Tuple				)			->						List[str]:






									'''simple docstring'''

									A         =					pipeline('fill-mask'      ,							model='hf-internal-testing/tiny-random-distilbert'      ,							device=0      ,							framework='pt'				)

									# convert model to fp16
									pipe.model.half()

									A         =					pipe('Paris is the [MASK] of France.'				)
									# We actually don't care about the result, we just want to make sure
									# it works, meaning the float16 tensor got casted back to float32
									# for postprocessing.
									self.assertIsInstance(snake_case      ,							snake_case				)



		@slow
		@require_torch
		def 					A_		( self  :    List[Any]				)			->						int:






									'''simple docstring'''

									A         =					pipeline(task='fill-mask'      ,							model='distilroberta-base'      ,							top_k=2      ,							framework='pt'				)
									self.run_large_test(snake_case				)



		@slow
		@require_tf
		def 					A_		( self  :    Tuple				)			->						Optional[int]:






									'''simple docstring'''

									A         =					pipeline(task='fill-mask'      ,							model='distilroberta-base'      ,							top_k=2      ,							framework='tf'				)
									self.run_large_test(snake_case				)



		def 					A_		( self  :    Dict      ,							snake_case  :    List[Any]				)			->						Optional[int]:






									'''simple docstring'''

									A         =					unmasker('My name is <mask>'				)
									self.assertEqual(
									    nested_simplify(snake_case				)      ,							[
									        {'sequence': 'My name is John', 'score': 0.008, 'token': 610, 'token_str': ' John'},
									        {'sequence': 'My name is Chris', 'score': 0.007, 'token': 1_573, 'token_str': ' Chris'},
									    ]      ,							)
									A         =					unmasker('The largest city in France is <mask>'				)
									self.assertEqual(
									    nested_simplify(snake_case				)      ,							[
									        {
									            'sequence': 'The largest city in France is Paris',
									            'score': 0.251,
									            'token': 2_201,
									            'token_str': ' Paris',
									        },
									        {
									            'sequence': 'The largest city in France is Lyon',
									            'score': 0.214,
									            'token': 12_790,
									            'token_str': ' Lyon',
									        },
									    ]      ,							)

									A         =					unmasker('My name is <mask>'      ,							targets=[' Patrick', ' Clara', ' Teven']      ,							top_k=3				)
									self.assertEqual(
									    nested_simplify(snake_case				)      ,							[
									        {'sequence': 'My name is Patrick', 'score': 0.005, 'token': 3_499, 'token_str': ' Patrick'},
									        {'sequence': 'My name is Clara', 'score': 0.000, 'token': 13_606, 'token_str': ' Clara'},
									        {'sequence': 'My name is Te', 'score': 0.000, 'token': 2_941, 'token_str': ' Te'},
									    ]      ,							)



		@require_torch
		def 					A_		( self  :    List[str]				)			->						Any:






									'''simple docstring'''

									A         =					pipeline(task='fill-mask'      ,							model='sshleifer/tiny-distilroberta-base'      ,							framework='pt'				)
									A         =					None
									A         =					None
									self.run_pipeline_test(snake_case      ,							[]				)



		@require_tf
		def 					A_		( self  :    Tuple				)			->						Dict:






									'''simple docstring'''

									A         =					pipeline(task='fill-mask'      ,							model='sshleifer/tiny-distilroberta-base'      ,							framework='tf'				)
									A         =					None
									A         =					None
									self.run_pipeline_test(snake_case      ,							[]				)



		def 					A_		( self  :    str      ,							snake_case  :    Union[str, Any]      ,							snake_case  :    Optional[int]      ,							snake_case  :    Optional[Any]				)			->						Union[str, Any]:






									'''simple docstring'''

									if tokenizer is None or tokenizer.mask_token_id is None:
																self.skipTest('The provided tokenizer has no mask token, (probably reformer or wav2vec2)'				)

									A         =					FillMaskPipeline(model=snake_case      ,							tokenizer=snake_case				)
									A         =					[
									    f"""This is another {tokenizer.mask_token} test""",
									]
									return fill_masker, examples



		def 					A_		( self  :    Any      ,							snake_case  :    Any      ,							snake_case  :    Optional[Any]				)			->						Optional[int]:






									'''simple docstring'''

									A         =					fill_masker.tokenizer
									A         =					fill_masker.model

									A         =					fill_masker(
									    f"""This is a {tokenizer.mask_token}"""      ,							)
									self.assertEqual(
									    snake_case      ,							[
									        {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									        {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									        {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									        {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									        {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									    ]      ,							)

									A         =					fill_masker([f"""This is a {tokenizer.mask_token}"""]				)
									self.assertEqual(
									    snake_case      ,							[
									        {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									        {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									        {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									        {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									        {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									    ]      ,							)

									A         =					fill_masker([f"""This is a {tokenizer.mask_token}""", f"""Another {tokenizer.mask_token} great test."""]				)
									self.assertEqual(
									    snake_case      ,							[
									        [
									            {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									            {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									            {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									            {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									            {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									        ],
									        [
									            {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									            {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									            {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									            {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									            {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									        ],
									    ]      ,							)

									with self.assertRaises(snake_case				):
																fill_masker([None]				)
									# No mask_token is not supported
									with self.assertRaises(snake_case				):
																fill_masker('This is'				)

									self.run_test_top_k(snake_case      ,							snake_case				)
									self.run_test_targets(snake_case      ,							snake_case				)
									self.run_test_top_k_targets(snake_case      ,							snake_case				)
									self.fill_mask_with_duplicate_targets_and_top_k(snake_case      ,							snake_case				)
									self.fill_mask_with_multiple_masks(snake_case      ,							snake_case				)



		def 					A_		( self  :    str      ,							snake_case  :    Any      ,							snake_case  :    Optional[int]				)			->						str:






									'''simple docstring'''

									A         =					tokenizer.get_vocab()
									A         =					sorted(vocab.keys()				)[:2]
									# Pipeline argument
									A         =					FillMaskPipeline(model=snake_case      ,							tokenizer=snake_case      ,							targets=snake_case				)
									A         =					fill_masker(f"""This is a {tokenizer.mask_token}"""				)
									self.assertEqual(
									    snake_case      ,							[
									        {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									        {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									    ]      ,							)
									A         =					{vocab[el] for el in targets}
									self.assertEqual({el['token'] for el in outputs}      ,							snake_case				)
									A         =					[tokenizer.decode([x]				) for x in target_ids]
									self.assertEqual({el['token_str'] for el in outputs}      ,							set(snake_case				)				)

									# Call argument
									A         =					FillMaskPipeline(model=snake_case      ,							tokenizer=snake_case				)
									A         =					fill_masker(f"""This is a {tokenizer.mask_token}"""      ,							targets=snake_case				)
									self.assertEqual(
									    snake_case      ,							[
									        {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									        {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									    ]      ,							)
									A         =					{vocab[el] for el in targets}
									self.assertEqual({el['token'] for el in outputs}      ,							snake_case				)
									A         =					[tokenizer.decode([x]				) for x in target_ids]
									self.assertEqual({el['token_str'] for el in outputs}      ,							set(snake_case				)				)

									# Score equivalence
									A         =					fill_masker(f"""This is a {tokenizer.mask_token}"""      ,							targets=snake_case				)
									A         =					[top_mask['token_str'] for top_mask in outputs]
									A         =					[top_mask['score'] for top_mask in outputs]

									# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
									if set(snake_case				) == set(snake_case				):
																A         =					fill_masker(f"""This is a {tokenizer.mask_token}"""      ,							targets=snake_case				)
																A         =					[top_mask['score'] for top_mask in unmasked_targets]
																self.assertEqual(nested_simplify(snake_case				)      ,							nested_simplify(snake_case				)				)

									# Raises with invalid
									with self.assertRaises(snake_case				):
																A         =					fill_masker(f"""This is a {tokenizer.mask_token}"""      ,							targets=[]				)
									# For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
									if "" not in tokenizer.get_vocab():
																with self.assertRaises(snake_case				):
																							A         =					fill_masker(f"""This is a {tokenizer.mask_token}"""      ,							targets=['']				)
																with self.assertRaises(snake_case				):
																							A         =					fill_masker(f"""This is a {tokenizer.mask_token}"""      ,							targets=''				)



		def 					A_		( self  :    Any      ,							snake_case  :    Optional[Any]      ,							snake_case  :    int				)			->						List[Any]:






									'''simple docstring'''

									A         =					FillMaskPipeline(model=snake_case      ,							tokenizer=snake_case      ,							top_k=2				)
									A         =					fill_masker(f"""This is a {tokenizer.mask_token}"""				)
									self.assertEqual(
									    snake_case      ,							[
									        {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									        {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									    ]      ,							)

									A         =					FillMaskPipeline(model=snake_case      ,							tokenizer=snake_case				)
									A         =					fill_masker(f"""This is a {tokenizer.mask_token}"""      ,							top_k=2				)
									self.assertEqual(
									    snake_case      ,							[
									        {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									        {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									    ]      ,							)
									self.assertEqual(nested_simplify(snake_case				)      ,							nested_simplify(snake_case				)				)



		def 					A_		( self  :    str      ,							snake_case  :    List[str]      ,							snake_case  :    Dict				)			->						Tuple:






									'''simple docstring'''

									A         =					tokenizer.get_vocab()
									A         =					FillMaskPipeline(model=snake_case      ,							tokenizer=snake_case				)

									# top_k=2, ntargets=3
									A         =					sorted(vocab.keys()				)[:3]
									A         =					fill_masker(f"""This is a {tokenizer.mask_token}"""      ,							top_k=2      ,							targets=snake_case				)

									# If we use the most probably targets, and filter differently, we should still
									# have the same results
									A         =					[el['token_str'] for el in sorted(snake_case      ,							key=lambda snake_case				: x["score"]      ,							reverse=snake_case				)]
									# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
									if set(snake_case				).issubset(snake_case				):
																A         =					fill_masker(f"""This is a {tokenizer.mask_token}"""      ,							top_k=3      ,							targets=snake_case				)
																# They should yield exactly the same result
																self.assertEqual(nested_simplify(snake_case				)      ,							nested_simplify(snake_case				)				)



		def 					A_		( self  :    Union[str, Any]      ,							snake_case  :    Union[str, Any]      ,							snake_case  :    Dict				)			->						int:






									'''simple docstring'''

									A         =					FillMaskPipeline(model=snake_case      ,							tokenizer=snake_case				)
									A         =					tokenizer.get_vocab()
									# String duplicates + id duplicates
									A         =					sorted(vocab.keys()				)[:3]
									A         =					[targets[0], targets[1], targets[0], targets[2], targets[1]]
									A         =					fill_masker(f"""My name is {tokenizer.mask_token}"""      ,							targets=snake_case      ,							top_k=10				)

									# The target list contains duplicates, so we can't output more
									# than them
									self.assertEqual(len(snake_case				)      ,							3				)



		def 					A_		( self  :    str      ,							snake_case  :    List[Any]      ,							snake_case  :    Union[str, Any]				)			->						Union[str, Any]:






									'''simple docstring'''

									A         =					FillMaskPipeline(model=snake_case      ,							tokenizer=snake_case				)

									A         =					fill_masker(
									    f"""This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}"""      ,							top_k=2				)
									self.assertEqual(
									    snake_case      ,							[
									        [
									            {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									            {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									        ],
									        [
									            {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									            {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									        ],
									        [
									            {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									            {'sequence': ANY(snake_case				), 'score': ANY(snake_case				), 'token': ANY(snake_case				), 'token_str': ANY(snake_case				)},
									        ],
									    ]      ,							)



