

import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY

@is_pipeline_test
class 				__UpperCamelCase				(					unittest.TestCase			):
	_lowercase  :   Union[str, Any]       				=  MODEL_FOR_MASKED_LM_MAPPING
	_lowercase  :   Any       				=  TF_MODEL_FOR_MASKED_LM_MAPPING







	def 	_SCREAMING_SNAKE_CASE							(   self: str      ):

						'''simple docstring'''





						super().tearDown()
						# clean-up as much as possible GPU memory occupied by PyTorch
						gc.collect()
						if is_torch_available():
											import torch

											torch.cuda.empty_cache()







	@require_tf
	def 	_SCREAMING_SNAKE_CASE							(   self: Dict      ):

						'''simple docstring'''





						__magic_name__							=    pipeline(task='fill-mask'  , model='sshleifer/tiny-distilroberta-base'  , top_k=2  , framework='tf'      )
						__magic_name__							=    unmasker('My name is <mask>'      )
						self.assertEqual(
						    nested_simplify(__UpperCamelCase  , decimals=6      )  , [
						        {'sequence': 'My name is grouped', 'score': 2.1E-05, 'token': 3_80_15, 'token_str': ' grouped'},
						        {'sequence': 'My name is accuser', 'score': 2.1E-05, 'token': 2_55_06, 'token_str': ' accuser'},
						    ]  , )

						__magic_name__							=    unmasker('The largest city in France is <mask>'      )
						self.assertEqual(
						    nested_simplify(__UpperCamelCase  , decimals=6      )  , [
						        {
						            'sequence': 'The largest city in France is grouped',
						            'score': 2.1E-05,
						            'token': 3_80_15,
						            'token_str': ' grouped',
						        },
						        {
						            'sequence': 'The largest city in France is accuser',
						            'score': 2.1E-05,
						            'token': 2_55_06,
						            'token_str': ' accuser',
						        },
						    ]  , )

						__magic_name__							=    unmasker('My name is <mask>'  , targets=[' Patrick', ' Clara', ' Teven']  , top_k=3      )
						self.assertEqual(
						    nested_simplify(__UpperCamelCase  , decimals=6      )  , [
						        {'sequence': 'My name is Clara', 'score': 2E-05, 'token': 1_36_06, 'token_str': ' Clara'},
						        {'sequence': 'My name is Patrick', 'score': 2E-05, 'token': 34_99, 'token_str': ' Patrick'},
						        {'sequence': 'My name is Te', 'score': 1.9E-05, 'token': 29_41, 'token_str': ' Te'},
						    ]  , )







	@require_torch
	def 	_SCREAMING_SNAKE_CASE							(   self: List[Any]      ):

						'''simple docstring'''





						__magic_name__							=    pipeline(task='fill-mask'  , model='sshleifer/tiny-distilroberta-base'  , top_k=2  , framework='pt'      )

						__magic_name__							=    unmasker('My name is <mask>'      )
						self.assertEqual(
						    nested_simplify(__UpperCamelCase  , decimals=6      )  , [
						        {'sequence': 'My name is Maul', 'score': 2.2E-05, 'token': 3_56_76, 'token_str': ' Maul'},
						        {'sequence': 'My name isELS', 'score': 2.2E-05, 'token': 1_64_16, 'token_str': 'ELS'},
						    ]  , )

						__magic_name__							=    unmasker('The largest city in France is <mask>'      )
						self.assertEqual(
						    nested_simplify(__UpperCamelCase  , decimals=6      )  , [
						        {
						            'sequence': 'The largest city in France is Maul',
						            'score': 2.2E-05,
						            'token': 3_56_76,
						            'token_str': ' Maul',
						        },
						        {'sequence': 'The largest city in France isELS', 'score': 2.2E-05, 'token': 1_64_16, 'token_str': 'ELS'},
						    ]  , )

						__magic_name__							=    unmasker('My name is <mask>'  , targets=[' Patrick', ' Clara', ' Teven']  , top_k=3      )
						self.assertEqual(
						    nested_simplify(__UpperCamelCase  , decimals=6      )  , [
						        {'sequence': 'My name is Patrick', 'score': 2.1E-05, 'token': 34_99, 'token_str': ' Patrick'},
						        {'sequence': 'My name is Te', 'score': 2E-05, 'token': 29_41, 'token_str': ' Te'},
						        {'sequence': 'My name is Clara', 'score': 2E-05, 'token': 1_36_06, 'token_str': ' Clara'},
						    ]  , )

						__magic_name__							=    unmasker('My name is <mask> <mask>'  , top_k=2      )

						self.assertEqual(
						    nested_simplify(__UpperCamelCase  , decimals=6      )  , [
						        [
						            {
						                'score': 2.2E-05,
						                'token': 3_56_76,
						                'token_str': ' Maul',
						                'sequence': '<s>My name is Maul<mask></s>',
						            },
						            {'score': 2.2E-05, 'token': 1_64_16, 'token_str': 'ELS', 'sequence': '<s>My name isELS<mask></s>'},
						        ],
						        [
						            {
						                'score': 2.2E-05,
						                'token': 3_56_76,
						                'token_str': ' Maul',
						                'sequence': '<s>My name is<mask> Maul</s>',
						            },
						            {'score': 2.2E-05, 'token': 1_64_16, 'token_str': 'ELS', 'sequence': '<s>My name is<mask>ELS</s>'},
						        ],
						    ]  , )







	@require_torch_gpu
	def 	_SCREAMING_SNAKE_CASE							(   self: Optional[int]      ):

						'''simple docstring'''





						__magic_name__							=    pipeline('fill-mask'  , model='hf-internal-testing/tiny-random-distilbert'  , device=0  , framework='pt'      )

						# convert model to fp16
						pipe.model.half()

						__magic_name__							=    pipe('Paris is the [MASK] of France.'      )
						# We actually don't care about the result, we just want to make sure
						# it works, meaning the float16 tensor got casted back to float32
						# for postprocessing.
						self.assertIsInstance(__UpperCamelCase  , __UpperCamelCase      )







	@slow
	@require_torch
	def 	_SCREAMING_SNAKE_CASE							(   self: Optional[Any]      ):

						'''simple docstring'''





						__magic_name__							=    pipeline(task='fill-mask'  , model='distilroberta-base'  , top_k=2  , framework='pt'      )
						self.run_large_test(__UpperCamelCase      )







	@slow
	@require_tf
	def 	_SCREAMING_SNAKE_CASE							(   self: Optional[int]      ):

						'''simple docstring'''





						__magic_name__							=    pipeline(task='fill-mask'  , model='distilroberta-base'  , top_k=2  , framework='tf'      )
						self.run_large_test(__UpperCamelCase      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Tuple  , __UpperCamelCase: Optional[int]      ):

						'''simple docstring'''





						__magic_name__							=    unmasker('My name is <mask>'      )
						self.assertEqual(
						    nested_simplify(__UpperCamelCase      )  , [
						        {'sequence': 'My name is John', 'score': 0.008, 'token': 6_10, 'token_str': ' John'},
						        {'sequence': 'My name is Chris', 'score': 0.007, 'token': 15_73, 'token_str': ' Chris'},
						    ]  , )
						__magic_name__							=    unmasker('The largest city in France is <mask>'      )
						self.assertEqual(
						    nested_simplify(__UpperCamelCase      )  , [
						        {
						            'sequence': 'The largest city in France is Paris',
						            'score': 0.251,
						            'token': 22_01,
						            'token_str': ' Paris',
						        },
						        {
						            'sequence': 'The largest city in France is Lyon',
						            'score': 0.214,
						            'token': 1_27_90,
						            'token_str': ' Lyon',
						        },
						    ]  , )

						__magic_name__							=    unmasker('My name is <mask>'  , targets=[' Patrick', ' Clara', ' Teven']  , top_k=3      )
						self.assertEqual(
						    nested_simplify(__UpperCamelCase      )  , [
						        {'sequence': 'My name is Patrick', 'score': 0.005, 'token': 34_99, 'token_str': ' Patrick'},
						        {'sequence': 'My name is Clara', 'score': 0.000, 'token': 1_36_06, 'token_str': ' Clara'},
						        {'sequence': 'My name is Te', 'score': 0.000, 'token': 29_41, 'token_str': ' Te'},
						    ]  , )







	@require_torch
	def 	_SCREAMING_SNAKE_CASE							(   self: List[Any]      ):

						'''simple docstring'''





						__magic_name__							=    pipeline(task='fill-mask'  , model='sshleifer/tiny-distilroberta-base'  , framework='pt'      )
						__magic_name__							=    None
						__magic_name__							=    None
						self.run_pipeline_test(__UpperCamelCase  , []      )







	@require_tf
	def 	_SCREAMING_SNAKE_CASE							(   self: Union[str, Any]      ):

						'''simple docstring'''





						__magic_name__							=    pipeline(task='fill-mask'  , model='sshleifer/tiny-distilroberta-base'  , framework='tf'      )
						__magic_name__							=    None
						__magic_name__							=    None
						self.run_pipeline_test(__UpperCamelCase  , []      )







	def 	_SCREAMING_SNAKE_CASE							(   self: List[Any]  , __UpperCamelCase: Optional[int]  , __UpperCamelCase: Union[str, Any]  , __UpperCamelCase: str      ):

						'''simple docstring'''





						if tokenizer is None or tokenizer.mask_token_id is None:
											self.skipTest('The provided tokenizer has no mask token, (probably reformer or wav2vec2)'      )

						__magic_name__							=    FillMaskPipeline(model=__UpperCamelCase  , tokenizer=__UpperCamelCase      )
						__magic_name__							=    [
						    F'This is another {tokenizer.mask_token} test',
						]
						return fill_masker, examples







	def 	_SCREAMING_SNAKE_CASE							(   self: Union[str, Any]  , __UpperCamelCase: Dict  , __UpperCamelCase: Optional[int]      ):

						'''simple docstring'''





						__magic_name__							=    fill_masker.tokenizer
						__magic_name__							=    fill_masker.model

						__magic_name__							=    fill_masker(
						    F'This is a {tokenizer.mask_token}'  , )
						self.assertEqual(
						    __UpperCamelCase  , [
						        {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						        {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						        {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						        {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						        {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						    ]  , )

						__magic_name__							=    fill_masker([F'This is a {tokenizer.mask_token}']      )
						self.assertEqual(
						    __UpperCamelCase  , [
						        {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						        {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						        {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						        {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						        {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						    ]  , )

						__magic_name__							=    fill_masker([F'This is a {tokenizer.mask_token}', F'Another {tokenizer.mask_token} great test.']      )
						self.assertEqual(
						    __UpperCamelCase  , [
						        [
						            {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						            {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						            {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						            {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						            {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						        ],
						        [
						            {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						            {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						            {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						            {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						            {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						        ],
						    ]  , )

						with self.assertRaises(__UpperCamelCase      ):
											fill_masker([None]      )
						# No mask_token is not supported
						with self.assertRaises(__UpperCamelCase      ):
											fill_masker('This is'      )

						self.run_test_top_k(__UpperCamelCase  , __UpperCamelCase      )
						self.run_test_targets(__UpperCamelCase  , __UpperCamelCase      )
						self.run_test_top_k_targets(__UpperCamelCase  , __UpperCamelCase      )
						self.fill_mask_with_duplicate_targets_and_top_k(__UpperCamelCase  , __UpperCamelCase      )
						self.fill_mask_with_multiple_masks(__UpperCamelCase  , __UpperCamelCase      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Tuple  , __UpperCamelCase: Optional[int]  , __UpperCamelCase: Union[str, Any]      ):

						'''simple docstring'''





						__magic_name__							=    tokenizer.get_vocab()
						__magic_name__							=    sorted(vocab.keys()      )[:2]
						# Pipeline argument
						__magic_name__							=    FillMaskPipeline(model=__UpperCamelCase  , tokenizer=__UpperCamelCase  , targets=__UpperCamelCase      )
						__magic_name__							=    fill_masker(F'This is a {tokenizer.mask_token}'      )
						self.assertEqual(
						    __UpperCamelCase  , [
						        {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						        {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						    ]  , )
						__magic_name__							=    {vocab[el] for el in targets}
						self.assertEqual({el['token'] for el in outputs}  , __UpperCamelCase      )
						__magic_name__							=    [tokenizer.decode([x]      ) for x in target_ids]
						self.assertEqual({el['token_str'] for el in outputs}  , set(__UpperCamelCase      )      )

						# Call argument
						__magic_name__							=    FillMaskPipeline(model=__UpperCamelCase  , tokenizer=__UpperCamelCase      )
						__magic_name__							=    fill_masker(F'This is a {tokenizer.mask_token}'  , targets=__UpperCamelCase      )
						self.assertEqual(
						    __UpperCamelCase  , [
						        {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						        {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						    ]  , )
						__magic_name__							=    {vocab[el] for el in targets}
						self.assertEqual({el['token'] for el in outputs}  , __UpperCamelCase      )
						__magic_name__							=    [tokenizer.decode([x]      ) for x in target_ids]
						self.assertEqual({el['token_str'] for el in outputs}  , set(__UpperCamelCase      )      )

						# Score equivalence
						__magic_name__							=    fill_masker(F'This is a {tokenizer.mask_token}'  , targets=__UpperCamelCase      )
						__magic_name__							=    [top_mask['token_str'] for top_mask in outputs]
						__magic_name__							=    [top_mask['score'] for top_mask in outputs]

						# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
						if set(__UpperCamelCase      ) == set(__UpperCamelCase      ):
											__magic_name__							=    fill_masker(F'This is a {tokenizer.mask_token}'  , targets=__UpperCamelCase      )
											__magic_name__							=    [top_mask['score'] for top_mask in unmasked_targets]
											self.assertEqual(nested_simplify(__UpperCamelCase      )  , nested_simplify(__UpperCamelCase      )      )

						# Raises with invalid
						with self.assertRaises(__UpperCamelCase      ):
											__magic_name__							=    fill_masker(F'This is a {tokenizer.mask_token}'  , targets=[]      )
						# For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
						if "" not in tokenizer.get_vocab():
											with self.assertRaises(__UpperCamelCase      ):
																__magic_name__							=    fill_masker(F'This is a {tokenizer.mask_token}'  , targets=['']      )
											with self.assertRaises(__UpperCamelCase      ):
																__magic_name__							=    fill_masker(F'This is a {tokenizer.mask_token}'  , targets=''      )







	def 	_SCREAMING_SNAKE_CASE							(   self: List[Any]  , __UpperCamelCase: List[Any]  , __UpperCamelCase: int      ):

						'''simple docstring'''





						__magic_name__							=    FillMaskPipeline(model=__UpperCamelCase  , tokenizer=__UpperCamelCase  , top_k=2      )
						__magic_name__							=    fill_masker(F'This is a {tokenizer.mask_token}'      )
						self.assertEqual(
						    __UpperCamelCase  , [
						        {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						        {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						    ]  , )

						__magic_name__							=    FillMaskPipeline(model=__UpperCamelCase  , tokenizer=__UpperCamelCase      )
						__magic_name__							=    fill_masker(F'This is a {tokenizer.mask_token}'  , top_k=2      )
						self.assertEqual(
						    __UpperCamelCase  , [
						        {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						        {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						    ]  , )
						self.assertEqual(nested_simplify(__UpperCamelCase      )  , nested_simplify(__UpperCamelCase      )      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Dict  , __UpperCamelCase: List[Any]  , __UpperCamelCase: str      ):

						'''simple docstring'''





						__magic_name__							=    tokenizer.get_vocab()
						__magic_name__							=    FillMaskPipeline(model=__UpperCamelCase  , tokenizer=__UpperCamelCase      )

						# top_k=2, ntargets=3
						__magic_name__							=    sorted(vocab.keys()      )[:3]
						__magic_name__							=    fill_masker(F'This is a {tokenizer.mask_token}'  , top_k=2  , targets=__UpperCamelCase      )

						# If we use the most probably targets, and filter differently, we should still
						# have the same results
						__magic_name__							=    [el['token_str'] for el in sorted(__UpperCamelCase  , key=lambda __UpperCamelCase      : x["score"]  , reverse=__UpperCamelCase      )]
						# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
						if set(__UpperCamelCase      ).issubset(__UpperCamelCase      ):
											__magic_name__							=    fill_masker(F'This is a {tokenizer.mask_token}'  , top_k=3  , targets=__UpperCamelCase      )
											# They should yield exactly the same result
											self.assertEqual(nested_simplify(__UpperCamelCase      )  , nested_simplify(__UpperCamelCase      )      )







	def 	_SCREAMING_SNAKE_CASE							(   self: int  , __UpperCamelCase: Optional[Any]  , __UpperCamelCase: List[Any]      ):

						'''simple docstring'''





						__magic_name__							=    FillMaskPipeline(model=__UpperCamelCase  , tokenizer=__UpperCamelCase      )
						__magic_name__							=    tokenizer.get_vocab()
						# String duplicates + id duplicates
						__magic_name__							=    sorted(vocab.keys()      )[:3]
						__magic_name__							=    [targets[0], targets[1], targets[0], targets[2], targets[1]]
						__magic_name__							=    fill_masker(F'My name is {tokenizer.mask_token}'  , targets=__UpperCamelCase  , top_k=10      )

						# The target list contains duplicates, so we can't output more
						# than them
						self.assertEqual(len(__UpperCamelCase      )  , 3      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Tuple  , __UpperCamelCase: int  , __UpperCamelCase: Optional[int]      ):

						'''simple docstring'''





						__magic_name__							=    FillMaskPipeline(model=__UpperCamelCase  , tokenizer=__UpperCamelCase      )

						__magic_name__							=    fill_masker(
						    F'This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}'  , top_k=2      )
						self.assertEqual(
						    __UpperCamelCase  , [
						        [
						            {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						            {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						        ],
						        [
						            {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						            {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						        ],
						        [
						            {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						            {'sequence': ANY(__UpperCamelCase      ), 'score': ANY(__UpperCamelCase      ), 'token': ANY(__UpperCamelCase      ), 'token_str': ANY(__UpperCamelCase      )},
						        ],
						    ]  , )







import shutil
import tempfile
import unittest

from transformers import ClapFeatureExtractor, ClapProcessor, RobertaTokenizer, RobertaTokenizerFast
from transformers.testing_utils import require_sentencepiece, require_torchaudio

from .test_feature_extraction_clap import floats_list

@require_torchaudio
@require_sentencepiece
class 				__UpperCamelCase				(					unittest.TestCase			):







	def 	_SCREAMING_SNAKE_CASE							(   self: Optional[Any]      ):

						'''simple docstring'''





						__magic_name__							=    'laion/clap-htsat-unfused'
						__magic_name__							=    tempfile.mkdtemp()







	def 	_SCREAMING_SNAKE_CASE							(   self: Union[str, Any]  , **__UpperCamelCase: List[Any]      ):

						'''simple docstring'''





						return RobertaTokenizer.from_pretrained(self.checkpoint  , **__UpperCamelCase      )







	def 	_SCREAMING_SNAKE_CASE							(   self: int  , **__UpperCamelCase: Dict      ):

						'''simple docstring'''





						return ClapFeatureExtractor.from_pretrained(self.checkpoint  , **__UpperCamelCase      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Optional[int]      ):

						'''simple docstring'''





						shutil.rmtree(self.tmpdirname      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Optional[Any]      ):

						'''simple docstring'''





						__magic_name__							=    self.get_tokenizer()
						__magic_name__							=    self.get_feature_extractor()

						__magic_name__							=    ClapProcessor(tokenizer=__UpperCamelCase  , feature_extractor=__UpperCamelCase      )

						processor.save_pretrained(self.tmpdirname      )
						__magic_name__							=    ClapProcessor.from_pretrained(self.tmpdirname      )

						self.assertEqual(processor.tokenizer.get_vocab()  , tokenizer.get_vocab()      )
						self.assertIsInstance(processor.tokenizer  , __UpperCamelCase      )

						self.assertEqual(processor.feature_extractor.to_json_string()  , feature_extractor.to_json_string()      )
						self.assertIsInstance(processor.feature_extractor  , __UpperCamelCase      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Union[str, Any]      ):

						'''simple docstring'''





						__magic_name__							=    ClapProcessor(tokenizer=self.get_tokenizer()  , feature_extractor=self.get_feature_extractor()      )
						processor.save_pretrained(self.tmpdirname      )

						__magic_name__							=    self.get_tokenizer(bos_token='(BOS)'  , eos_token='(EOS)'      )
						__magic_name__							=    self.get_feature_extractor(do_normalize=__UpperCamelCase  , padding_value=1.0      )

						__magic_name__							=    ClapProcessor.from_pretrained(
						    self.tmpdirname  , bos_token='(BOS)'  , eos_token='(EOS)'  , do_normalize=__UpperCamelCase  , padding_value=1.0      )

						self.assertEqual(processor.tokenizer.get_vocab()  , tokenizer_add_kwargs.get_vocab()      )
						self.assertIsInstance(processor.tokenizer  , __UpperCamelCase      )

						self.assertEqual(processor.feature_extractor.to_json_string()  , feature_extractor_add_kwargs.to_json_string()      )
						self.assertIsInstance(processor.feature_extractor  , __UpperCamelCase      )







	def 	_SCREAMING_SNAKE_CASE							(   self: List[str]      ):

						'''simple docstring'''





						__magic_name__							=    self.get_feature_extractor()
						__magic_name__							=    self.get_tokenizer()

						__magic_name__							=    ClapProcessor(tokenizer=__UpperCamelCase  , feature_extractor=__UpperCamelCase      )

						__magic_name__							=    floats_list((3, 10_00)      )

						__magic_name__							=    feature_extractor(__UpperCamelCase  , return_tensors='np'      )
						__magic_name__							=    processor(audios=__UpperCamelCase  , return_tensors='np'      )

						for key in input_feat_extract.keys():
											self.assertAlmostEqual(input_feat_extract[key].sum()  , input_processor[key].sum()  , delta=1E-2      )







	def 	_SCREAMING_SNAKE_CASE							(   self: List[Any]      ):

						'''simple docstring'''





						__magic_name__							=    self.get_feature_extractor()
						__magic_name__							=    self.get_tokenizer()

						__magic_name__							=    ClapProcessor(tokenizer=__UpperCamelCase  , feature_extractor=__UpperCamelCase      )

						__magic_name__							=    'This is a test string'

						__magic_name__							=    processor(text=__UpperCamelCase      )

						__magic_name__							=    tokenizer(__UpperCamelCase      )

						for key in encoded_tok.keys():
											self.assertListEqual(encoded_tok[key]  , encoded_processor[key]      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Optional[Any]      ):

						'''simple docstring'''





						__magic_name__							=    self.get_feature_extractor()
						__magic_name__							=    self.get_tokenizer()

						__magic_name__							=    ClapProcessor(tokenizer=__UpperCamelCase  , feature_extractor=__UpperCamelCase      )

						__magic_name__							=    [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

						__magic_name__							=    processor.batch_decode(__UpperCamelCase      )
						__magic_name__							=    tokenizer.batch_decode(__UpperCamelCase      )

						self.assertListEqual(__UpperCamelCase  , __UpperCamelCase      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Any      ):

						'''simple docstring'''





						__magic_name__							=    self.get_feature_extractor()
						__magic_name__							=    self.get_tokenizer()

						__magic_name__							=    ClapProcessor(tokenizer=__UpperCamelCase  , feature_extractor=__UpperCamelCase      )

						self.assertListEqual(
						    processor.model_input_names[2:]  , feature_extractor.model_input_names  , msg='`processor` and `feature_extractor` model input names do not match'  , )




