






import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY
@is_pipeline_test
class 						lowerCamelCase_				(      unittest.TestCase ):
				a__					       =  MODEL_FOR_MASKED_LM_MAPPING
				a__					       =  TF_MODEL_FOR_MASKED_LM_MAPPING







				def 			A	(						self     ):
								"""simple docstring"""
								super().tearDown()
								# clean-up as much as possible GPU memory occupied by PyTorch
								gc.collect()
								if is_torch_available():
												import torch

												torch.cuda.empty_cache()







				@require_tf
				def 			A	(						self     ):
								"""simple docstring"""
								__magic_name__		:Dict        =      pipeline(task='''fill-mask'''       ,					model='''sshleifer/tiny-distilroberta-base'''       ,					top_k=2       ,					framework='''tf'''     )
								__magic_name__		:Tuple        =      unmasker('''My name is <mask>'''     )
								self.assertEqual(
								    nested_simplify(__lowerCAmelCase       ,					decimals=6     )       ,					[
								        {'''sequence''': '''My name is grouped''', '''score''': 2.1E-05, '''token''': 3_8_0_1_5, '''token_str''': ''' grouped'''},
								        {'''sequence''': '''My name is accuser''', '''score''': 2.1E-05, '''token''': 2_5_5_0_6, '''token_str''': ''' accuser'''},
								    ]       ,					)

								__magic_name__		:List[str]        =      unmasker('''The largest city in France is <mask>'''     )
								self.assertEqual(
								    nested_simplify(__lowerCAmelCase       ,					decimals=6     )       ,					[
								        {
								            '''sequence''': '''The largest city in France is grouped''',
								            '''score''': 2.1E-05,
								            '''token''': 3_8_0_1_5,
								            '''token_str''': ''' grouped''',
								        },
								        {
								            '''sequence''': '''The largest city in France is accuser''',
								            '''score''': 2.1E-05,
								            '''token''': 2_5_5_0_6,
								            '''token_str''': ''' accuser''',
								        },
								    ]       ,					)

								__magic_name__		:List[str]        =      unmasker('''My name is <mask>'''       ,					targets=[''' Patrick''', ''' Clara''', ''' Teven''']       ,					top_k=3     )
								self.assertEqual(
								    nested_simplify(__lowerCAmelCase       ,					decimals=6     )       ,					[
								        {'''sequence''': '''My name is Clara''', '''score''': 2E-05, '''token''': 1_3_6_0_6, '''token_str''': ''' Clara'''},
								        {'''sequence''': '''My name is Patrick''', '''score''': 2E-05, '''token''': 3_4_9_9, '''token_str''': ''' Patrick'''},
								        {'''sequence''': '''My name is Te''', '''score''': 1.9E-05, '''token''': 2_9_4_1, '''token_str''': ''' Te'''},
								    ]       ,					)







				@require_torch
				def 			A	(						self     ):
								"""simple docstring"""
								__magic_name__		:Union[str, Any]        =      pipeline(task='''fill-mask'''       ,					model='''sshleifer/tiny-distilroberta-base'''       ,					top_k=2       ,					framework='''pt'''     )

								__magic_name__		:int        =      unmasker('''My name is <mask>'''     )
								self.assertEqual(
								    nested_simplify(__lowerCAmelCase       ,					decimals=6     )       ,					[
								        {'''sequence''': '''My name is Maul''', '''score''': 2.2E-05, '''token''': 3_5_6_7_6, '''token_str''': ''' Maul'''},
								        {'''sequence''': '''My name isELS''', '''score''': 2.2E-05, '''token''': 1_6_4_1_6, '''token_str''': '''ELS'''},
								    ]       ,					)

								__magic_name__		:Optional[Any]        =      unmasker('''The largest city in France is <mask>'''     )
								self.assertEqual(
								    nested_simplify(__lowerCAmelCase       ,					decimals=6     )       ,					[
								        {
								            '''sequence''': '''The largest city in France is Maul''',
								            '''score''': 2.2E-05,
								            '''token''': 3_5_6_7_6,
								            '''token_str''': ''' Maul''',
								        },
								        {'''sequence''': '''The largest city in France isELS''', '''score''': 2.2E-05, '''token''': 1_6_4_1_6, '''token_str''': '''ELS'''},
								    ]       ,					)

								__magic_name__		:int        =      unmasker('''My name is <mask>'''       ,					targets=[''' Patrick''', ''' Clara''', ''' Teven''']       ,					top_k=3     )
								self.assertEqual(
								    nested_simplify(__lowerCAmelCase       ,					decimals=6     )       ,					[
								        {'''sequence''': '''My name is Patrick''', '''score''': 2.1E-05, '''token''': 3_4_9_9, '''token_str''': ''' Patrick'''},
								        {'''sequence''': '''My name is Te''', '''score''': 2E-05, '''token''': 2_9_4_1, '''token_str''': ''' Te'''},
								        {'''sequence''': '''My name is Clara''', '''score''': 2E-05, '''token''': 1_3_6_0_6, '''token_str''': ''' Clara'''},
								    ]       ,					)

								__magic_name__		:Optional[int]        =      unmasker('''My name is <mask> <mask>'''       ,					top_k=2     )

								self.assertEqual(
								    nested_simplify(__lowerCAmelCase       ,					decimals=6     )       ,					[
								        [
								            {
								                '''score''': 2.2E-05,
								                '''token''': 3_5_6_7_6,
								                '''token_str''': ''' Maul''',
								                '''sequence''': '''<s>My name is Maul<mask></s>''',
								            },
								            {'''score''': 2.2E-05, '''token''': 1_6_4_1_6, '''token_str''': '''ELS''', '''sequence''': '''<s>My name isELS<mask></s>'''},
								        ],
								        [
								            {
								                '''score''': 2.2E-05,
								                '''token''': 3_5_6_7_6,
								                '''token_str''': ''' Maul''',
								                '''sequence''': '''<s>My name is<mask> Maul</s>''',
								            },
								            {'''score''': 2.2E-05, '''token''': 1_6_4_1_6, '''token_str''': '''ELS''', '''sequence''': '''<s>My name is<mask>ELS</s>'''},
								        ],
								    ]       ,					)







				@require_torch_gpu
				def 			A	(						self     ):
								"""simple docstring"""
								__magic_name__		:List[str]        =      pipeline('''fill-mask'''       ,					model='''hf-internal-testing/tiny-random-distilbert'''       ,					device=0       ,					framework='''pt'''     )

								# convert model to fp16
								pipe.model.half()

								__magic_name__		:List[Any]        =      pipe('''Paris is the [MASK] of France.'''     )
								# We actually don't care about the result, we just want to make sure
								# it works, meaning the float16 tensor got casted back to float32
								# for postprocessing.
								self.assertIsInstance(__lowerCAmelCase       ,					__lowerCAmelCase     )







				@slow
				@require_torch
				def 			A	(						self     ):
								"""simple docstring"""
								__magic_name__		:int        =      pipeline(task='''fill-mask'''       ,					model='''distilroberta-base'''       ,					top_k=2       ,					framework='''pt'''     )
								self.run_large_test(__lowerCAmelCase     )







				@slow
				@require_tf
				def 			A	(						self     ):
								"""simple docstring"""
								__magic_name__		:Dict        =      pipeline(task='''fill-mask'''       ,					model='''distilroberta-base'''       ,					top_k=2       ,					framework='''tf'''     )
								self.run_large_test(__lowerCAmelCase     )







				def 			A	(						self       ,					__lowerCAmelCase     ):
								"""simple docstring"""
								__magic_name__		:str        =      unmasker('''My name is <mask>'''     )
								self.assertEqual(
								    nested_simplify(__lowerCAmelCase     )       ,					[
								        {'''sequence''': '''My name is John''', '''score''': 0.008, '''token''': 6_1_0, '''token_str''': ''' John'''},
								        {'''sequence''': '''My name is Chris''', '''score''': 0.007, '''token''': 1_5_7_3, '''token_str''': ''' Chris'''},
								    ]       ,					)
								__magic_name__		:List[str]        =      unmasker('''The largest city in France is <mask>'''     )
								self.assertEqual(
								    nested_simplify(__lowerCAmelCase     )       ,					[
								        {
								            '''sequence''': '''The largest city in France is Paris''',
								            '''score''': 0.251,
								            '''token''': 2_2_0_1,
								            '''token_str''': ''' Paris''',
								        },
								        {
								            '''sequence''': '''The largest city in France is Lyon''',
								            '''score''': 0.214,
								            '''token''': 1_2_7_9_0,
								            '''token_str''': ''' Lyon''',
								        },
								    ]       ,					)

								__magic_name__		:Dict        =      unmasker('''My name is <mask>'''       ,					targets=[''' Patrick''', ''' Clara''', ''' Teven''']       ,					top_k=3     )
								self.assertEqual(
								    nested_simplify(__lowerCAmelCase     )       ,					[
								        {'''sequence''': '''My name is Patrick''', '''score''': 0.005, '''token''': 3_4_9_9, '''token_str''': ''' Patrick'''},
								        {'''sequence''': '''My name is Clara''', '''score''': 0.000, '''token''': 1_3_6_0_6, '''token_str''': ''' Clara'''},
								        {'''sequence''': '''My name is Te''', '''score''': 0.000, '''token''': 2_9_4_1, '''token_str''': ''' Te'''},
								    ]       ,					)







				@require_torch
				def 			A	(						self     ):
								"""simple docstring"""
								__magic_name__		:Union[str, Any]        =      pipeline(task='''fill-mask'''       ,					model='''sshleifer/tiny-distilroberta-base'''       ,					framework='''pt'''     )
								__magic_name__		:Optional[Any]        =      None
								__magic_name__		:Tuple        =      None
								self.run_pipeline_test(__lowerCAmelCase       ,					[]     )







				@require_tf
				def 			A	(						self     ):
								"""simple docstring"""
								__magic_name__		:List[Any]        =      pipeline(task='''fill-mask'''       ,					model='''sshleifer/tiny-distilroberta-base'''       ,					framework='''tf'''     )
								__magic_name__		:Optional[Any]        =      None
								__magic_name__		:Dict        =      None
								self.run_pipeline_test(__lowerCAmelCase       ,					[]     )







				def 			A	(						self       ,					__lowerCAmelCase       ,					__lowerCAmelCase       ,					__lowerCAmelCase     ):
								"""simple docstring"""
								if tokenizer is None or tokenizer.mask_token_id is None:
												self.skipTest('''The provided tokenizer has no mask token, (probably reformer or wav2vec2)'''     )

								__magic_name__		:Any        =      FillMaskPipeline(model=__lowerCAmelCase       ,					tokenizer=__lowerCAmelCase     )
								__magic_name__		:int        =      [
								    F'''This is another {tokenizer.mask_token} test''',
								]
								return fill_masker, examples







				def 			A	(						self       ,					__lowerCAmelCase       ,					__lowerCAmelCase     ):
								"""simple docstring"""
								__magic_name__		:List[str]        =      fill_masker.tokenizer
								__magic_name__		:str        =      fill_masker.model

								__magic_name__		:List[str]        =      fill_masker(
								    F'''This is a {tokenizer.mask_token}'''       ,					)
								self.assertEqual(
								    __lowerCAmelCase       ,					[
								        {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								        {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								        {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								        {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								        {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								    ]       ,					)

								__magic_name__		:List[Any]        =      fill_masker([F'''This is a {tokenizer.mask_token}''']     )
								self.assertEqual(
								    __lowerCAmelCase       ,					[
								        {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								        {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								        {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								        {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								        {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								    ]       ,					)

								__magic_name__		:Optional[int]        =      fill_masker([F'''This is a {tokenizer.mask_token}''', F'''Another {tokenizer.mask_token} great test.''']     )
								self.assertEqual(
								    __lowerCAmelCase       ,					[
								        [
								            {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								            {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								            {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								            {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								            {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								        ],
								        [
								            {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								            {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								            {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								            {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								            {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								        ],
								    ]       ,					)

								with self.assertRaises(__lowerCAmelCase     ):
												fill_masker([None]     )
								# No mask_token is not supported
								with self.assertRaises(__lowerCAmelCase     ):
												fill_masker('''This is'''     )

								self.run_test_top_k(__lowerCAmelCase       ,					__lowerCAmelCase     )
								self.run_test_targets(__lowerCAmelCase       ,					__lowerCAmelCase     )
								self.run_test_top_k_targets(__lowerCAmelCase       ,					__lowerCAmelCase     )
								self.fill_mask_with_duplicate_targets_and_top_k(__lowerCAmelCase       ,					__lowerCAmelCase     )
								self.fill_mask_with_multiple_masks(__lowerCAmelCase       ,					__lowerCAmelCase     )







				def 			A	(						self       ,					__lowerCAmelCase       ,					__lowerCAmelCase     ):
								"""simple docstring"""
								__magic_name__		:Optional[int]        =      tokenizer.get_vocab()
								__magic_name__		:str        =      sorted(vocab.keys()     )[:2]
								# Pipeline argument
								__magic_name__		:Union[str, Any]        =      FillMaskPipeline(model=__lowerCAmelCase       ,					tokenizer=__lowerCAmelCase       ,					targets=__lowerCAmelCase     )
								__magic_name__		:Dict        =      fill_masker(F'''This is a {tokenizer.mask_token}'''     )
								self.assertEqual(
								    __lowerCAmelCase       ,					[
								        {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								        {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								    ]       ,					)
								__magic_name__		:Any        =      {vocab[el] for el in targets}
								self.assertEqual({el['''token'''] for el in outputs}       ,					__lowerCAmelCase     )
								__magic_name__		:Dict        =      [tokenizer.decode([x]     ) for x in target_ids]
								self.assertEqual({el['''token_str'''] for el in outputs}       ,					set(__lowerCAmelCase     )     )

								# Call argument
								__magic_name__		:Dict        =      FillMaskPipeline(model=__lowerCAmelCase       ,					tokenizer=__lowerCAmelCase     )
								__magic_name__		:Optional[int]        =      fill_masker(F'''This is a {tokenizer.mask_token}'''       ,					targets=__lowerCAmelCase     )
								self.assertEqual(
								    __lowerCAmelCase       ,					[
								        {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								        {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								    ]       ,					)
								__magic_name__		:Tuple        =      {vocab[el] for el in targets}
								self.assertEqual({el['''token'''] for el in outputs}       ,					__lowerCAmelCase     )
								__magic_name__		:Optional[int]        =      [tokenizer.decode([x]     ) for x in target_ids]
								self.assertEqual({el['''token_str'''] for el in outputs}       ,					set(__lowerCAmelCase     )     )

								# Score equivalence
								__magic_name__		:List[str]        =      fill_masker(F'''This is a {tokenizer.mask_token}'''       ,					targets=__lowerCAmelCase     )
								__magic_name__		:int        =      [top_mask['''token_str'''] for top_mask in outputs]
								__magic_name__		:Tuple        =      [top_mask['''score'''] for top_mask in outputs]

								# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
								if set(__lowerCAmelCase     ) == set(__lowerCAmelCase     ):
												__magic_name__		:Dict        =      fill_masker(F'''This is a {tokenizer.mask_token}'''       ,					targets=__lowerCAmelCase     )
												__magic_name__		:Any        =      [top_mask['''score'''] for top_mask in unmasked_targets]
												self.assertEqual(nested_simplify(__lowerCAmelCase     )       ,					nested_simplify(__lowerCAmelCase     )     )

								# Raises with invalid
								with self.assertRaises(__lowerCAmelCase     ):
												__magic_name__		:Any        =      fill_masker(F'''This is a {tokenizer.mask_token}'''       ,					targets=[]     )
								# For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
								if "" not in tokenizer.get_vocab():
												with self.assertRaises(__lowerCAmelCase     ):
																__magic_name__		:int        =      fill_masker(F'''This is a {tokenizer.mask_token}'''       ,					targets=['''''']     )
												with self.assertRaises(__lowerCAmelCase     ):
																__magic_name__		:Optional[int]        =      fill_masker(F'''This is a {tokenizer.mask_token}'''       ,					targets=''''''     )







				def 			A	(						self       ,					__lowerCAmelCase       ,					__lowerCAmelCase     ):
								"""simple docstring"""
								__magic_name__		:List[str]        =      FillMaskPipeline(model=__lowerCAmelCase       ,					tokenizer=__lowerCAmelCase       ,					top_k=2     )
								__magic_name__		:Union[str, Any]        =      fill_masker(F'''This is a {tokenizer.mask_token}'''     )
								self.assertEqual(
								    __lowerCAmelCase       ,					[
								        {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								        {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								    ]       ,					)

								__magic_name__		:str        =      FillMaskPipeline(model=__lowerCAmelCase       ,					tokenizer=__lowerCAmelCase     )
								__magic_name__		:Any        =      fill_masker(F'''This is a {tokenizer.mask_token}'''       ,					top_k=2     )
								self.assertEqual(
								    __lowerCAmelCase       ,					[
								        {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								        {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								    ]       ,					)
								self.assertEqual(nested_simplify(__lowerCAmelCase     )       ,					nested_simplify(__lowerCAmelCase     )     )







				def 			A	(						self       ,					__lowerCAmelCase       ,					__lowerCAmelCase     ):
								"""simple docstring"""
								__magic_name__		:Tuple        =      tokenizer.get_vocab()
								__magic_name__		:int        =      FillMaskPipeline(model=__lowerCAmelCase       ,					tokenizer=__lowerCAmelCase     )

								# top_k=2, ntargets=3
								__magic_name__		:Optional[int]        =      sorted(vocab.keys()     )[:3]
								__magic_name__		:Optional[int]        =      fill_masker(F'''This is a {tokenizer.mask_token}'''       ,					top_k=2       ,					targets=__lowerCAmelCase     )

								# If we use the most probably targets, and filter differently, we should still
								# have the same results
								__magic_name__		:Dict        =      [el['''token_str'''] for el in sorted(__lowerCAmelCase       ,					key=lambda __lowerCAmelCase     : x["score"]       ,					reverse=__lowerCAmelCase     )]
								# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
								if set(__lowerCAmelCase     ).issubset(__lowerCAmelCase     ):
												__magic_name__		:str        =      fill_masker(F'''This is a {tokenizer.mask_token}'''       ,					top_k=3       ,					targets=__lowerCAmelCase     )
												# They should yield exactly the same result
												self.assertEqual(nested_simplify(__lowerCAmelCase     )       ,					nested_simplify(__lowerCAmelCase     )     )







				def 			A	(						self       ,					__lowerCAmelCase       ,					__lowerCAmelCase     ):
								"""simple docstring"""
								__magic_name__		:Optional[int]        =      FillMaskPipeline(model=__lowerCAmelCase       ,					tokenizer=__lowerCAmelCase     )
								__magic_name__		:int        =      tokenizer.get_vocab()
								# String duplicates + id duplicates
								__magic_name__		:Any        =      sorted(vocab.keys()     )[:3]
								__magic_name__		:Optional[int]        =      [targets[0], targets[1], targets[0], targets[2], targets[1]]
								__magic_name__		:str        =      fill_masker(F'''My name is {tokenizer.mask_token}'''       ,					targets=__lowerCAmelCase       ,					top_k=1_0     )

								# The target list contains duplicates, so we can't output more
								# than them
								self.assertEqual(len(__lowerCAmelCase     )       ,					3     )







				def 			A	(						self       ,					__lowerCAmelCase       ,					__lowerCAmelCase     ):
								"""simple docstring"""
								__magic_name__		:Union[str, Any]        =      FillMaskPipeline(model=__lowerCAmelCase       ,					tokenizer=__lowerCAmelCase     )

								__magic_name__		:Union[str, Any]        =      fill_masker(
								    F'''This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}'''       ,					top_k=2     )
								self.assertEqual(
								    __lowerCAmelCase       ,					[
								        [
								            {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								            {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								        ],
								        [
								            {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								            {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								        ],
								        [
								            {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								            {'''sequence''': ANY(__lowerCAmelCase     ), '''score''': ANY(__lowerCAmelCase     ), '''token''': ANY(__lowerCAmelCase     ), '''token_str''': ANY(__lowerCAmelCase     )},
								        ],
								    ]       ,					)














from ...processing_utils import ProcessorMixin
class 						lowerCamelCase_				(      lowerCamelCase ):
				a__					       =  '''WhisperFeatureExtractor'''
				a__					       =  '''WhisperTokenizer'''







				def __init__(						self       ,					__lowerCAmelCase       ,					__lowerCAmelCase     ):
								"""simple docstring"""
								super().__init__(__lowerCAmelCase       ,					__lowerCAmelCase     )
								__magic_name__		:int        =      self.feature_extractor
								__magic_name__		:int        =      False







				def 			A	(						self       ,					__lowerCAmelCase=None       ,					__lowerCAmelCase=None       ,					__lowerCAmelCase=True     ):
								"""simple docstring"""
								return self.tokenizer.get_decoder_prompt_ids(task=__lowerCAmelCase       ,					language=__lowerCAmelCase       ,					no_timestamps=__lowerCAmelCase     )







				def __call__(						self       ,					*__lowerCAmelCase       ,					**__lowerCAmelCase     ):
								"""simple docstring"""
								# For backward compatibility
								if self._in_target_context_manager:
												return self.current_processor(*__lowerCAmelCase       ,					**__lowerCAmelCase     )

								__magic_name__		:List[Any]        =      kwargs.pop('''audio'''       ,					__lowerCAmelCase     )
								__magic_name__		:Any        =      kwargs.pop('''sampling_rate'''       ,					__lowerCAmelCase     )
								__magic_name__		:str        =      kwargs.pop('''text'''       ,					__lowerCAmelCase     )
								if len(__lowerCAmelCase     ) > 0:
												__magic_name__		:Any        =      args[0]
												__magic_name__		:Union[str, Any]        =      args[1:]

								if audio is None and text is None:
												raise ValueError('''You need to specify either an `audio` or `text` input to process.'''     )

								if audio is not None:
												__magic_name__		:Optional[int]        =      self.feature_extractor(__lowerCAmelCase       ,					*__lowerCAmelCase       ,					sampling_rate=__lowerCAmelCase       ,					**__lowerCAmelCase     )
								if text is not None:
												__magic_name__		:Optional[int]        =      self.tokenizer(__lowerCAmelCase       ,					**__lowerCAmelCase     )

								if text is None:
												return inputs

								elif audio is None:
												return encodings
								else:
												__magic_name__		:str        =      encodings['''input_ids''']
												return inputs







				def 			A	(						self       ,					*__lowerCAmelCase       ,					**__lowerCAmelCase     ):
								"""simple docstring"""
								return self.tokenizer.batch_decode(*__lowerCAmelCase       ,					**__lowerCAmelCase     )







				def 			A	(						self       ,					*__lowerCAmelCase       ,					**__lowerCAmelCase     ):
								"""simple docstring"""
								return self.tokenizer.decode(*__lowerCAmelCase       ,					**__lowerCAmelCase     )







				def 			A	(						self       ,					__lowerCAmelCase       ,					__lowerCAmelCase="np"     ):
								"""simple docstring"""
								return self.tokenizer.get_prompt_ids(__lowerCAmelCase       ,					return_tensors=__lowerCAmelCase     )






