





import os
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen, xsplitext
from ..table import array_cast
from ..utils.py_utils import no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
					from .features import FeatureType


UpperCamelCase__					, UpperCamelCase__					, UpperCamelCase__											= False, False, False

@dataclass
class        __SCREAMING_SNAKE_CASE					:
							snake_case     :      Optional[int]						     =       None
							snake_case     :      bool						     =       True
							snake_case     :      bool						     =       True
							snake_case     :      Optional[str]						     =       None
							# Automatically constructed
							snake_case     :      ClassVar[str]						     =       "dict"
							snake_case     :      ClassVar[Any]						     =       pa.struct({"""bytes""": pa.binary(), """path""": pa.string()}       )
							snake_case     :      str						     =       field(default="""Audio"""   ,		init=_a   ,		repr=_a       )






							def __call__(						self  ):
														return self.pa_type






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														try:
																					import soundfile as sf  # soundfile is a dependency of librosa, needed to decode audio files.
														except ImportError as err:
																					raise ImportError("""To support encoding audio data, please install 'soundfile'."""  ) from err
														if isinstance(__lowerCAmelCase  , __lowerCAmelCase  ):
																					return {"bytes": None, "path": value}
														elif isinstance(__lowerCAmelCase  , __lowerCAmelCase  ):
																					return {"bytes": value, "path": None}
														elif "array" in value:
																					# convert the audio array to wav bytes
																					UpperCamelCase__				     =			BytesIO()
																					sf.write(__lowerCAmelCase  , value["""array"""]  , value["""sampling_rate"""]  , format="""wav"""  )
																					return {"bytes": buffer.getvalue(), "path": None}
														elif value.get("""path"""  ) is not None and os.path.isfile(value["""path"""]  ):
																					# we set "bytes": None to not duplicate the data if they're already available locally
																					if value["path"].endswith("""pcm"""  ):
																												# "PCM" only has raw audio bytes
																												if value.get("""sampling_rate"""  ) is None:
																																			# At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate
																																			raise KeyError("""To use PCM files, please specify a 'sampling_rate' in Audio object"""  )
																												if value.get("""bytes"""  ):
																																			# If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!)
																																			UpperCamelCase__				     =			np.frombuffer(value["""bytes"""]  , dtype=np.intaa  ).astype(np.floataa  ) / 32767
																												else:
																																			UpperCamelCase__				     =			np.memmap(value["""path"""]  , dtype="""h"""  , mode="""r"""  ).astype(np.floataa  ) / 32767

																												UpperCamelCase__				     =			BytesIO(bytes()  )
																												sf.write(__lowerCAmelCase  , __lowerCAmelCase  , value["""sampling_rate"""]  , format="""wav"""  )
																												return {"bytes": buffer.getvalue(), "path": None}
																					else:
																												return {"bytes": None, "path": value.get("""path"""  )}
														elif value.get("""bytes"""  ) is not None or value.get("""path"""  ) is not None:
																					# store the audio bytes, and path is used to infer the audio format using the file extension
																					return {"bytes": value.get("""bytes"""  ), "path": value.get("""path"""  )}
														else:
																					raise ValueError(
																					    f"""An audio sample should have one of 'path' or 'bytes' but they are missing or None in {value}."""  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase = None  ):
														if not self.decode:
																					raise RuntimeError("""Decoding is disabled for this feature. Please use Audio(decode=True) instead."""  )

														UpperCamelCase__     ,     UpperCamelCase__				     =			(value["""path"""], BytesIO(value["""bytes"""]  )) if value["""bytes"""] is not None else (value["""path"""], None)
														if path is None and file is None:
																					raise ValueError(f"""An audio sample should have one of 'path' or 'bytes' but both are None in {value}."""  )

														try:
																					import librosa
																					import soundfile as sf
														except ImportError as err:
																					raise ImportError("""To support decoding audio files, please install 'librosa' and 'soundfile'."""  ) from err

														UpperCamelCase__				     =			xsplitext(__lowerCAmelCase  )[1][1:].lower() if path is not None else None
														if not config.IS_OPUS_SUPPORTED and audio_format == "opus":
																					raise RuntimeError(
																					    """Decoding 'opus' files requires system library 'libsndfile'>=1.0.31, """
																					    """You can try to update `soundfile` python library: `pip install \"soundfile>=0.12.1\"`. """  )
														elif not config.IS_MP3_SUPPORTED and audio_format == "mp3":
																					raise RuntimeError(
																					    """Decoding 'mp3' files requires system library 'libsndfile'>=1.1.0, """
																					    """You can try to update `soundfile` python library: `pip install \"soundfile>=0.12.1\"`. """  )

														if file is None:
																					UpperCamelCase__				     =			token_per_repo_id or {}
																					UpperCamelCase__				     =			path.split("""::"""  )[-1]
																					try:
																												UpperCamelCase__				     =			string_to_dict(__lowerCAmelCase  , config.HUB_DATASETS_URL  )["""repo_id"""]
																												UpperCamelCase__				     =			token_per_repo_id[repo_id]
																					except (ValueError, KeyError):
																												UpperCamelCase__				     =			None

																					with xopen(__lowerCAmelCase  , """rb"""  , use_auth_token=__lowerCAmelCase  ) as f:
																												UpperCamelCase__     ,     UpperCamelCase__				     =			sf.read(__lowerCAmelCase  )

														else:
																					UpperCamelCase__     ,     UpperCamelCase__				     =			sf.read(__lowerCAmelCase  )

														UpperCamelCase__				     =			array.T
														if self.mono:
																					UpperCamelCase__				     =			librosa.to_mono(__lowerCAmelCase  )
														if self.sampling_rate and self.sampling_rate != sampling_rate:
																					UpperCamelCase__				     =			librosa.resample(__lowerCAmelCase  , orig_sr=__lowerCAmelCase  , target_sr=self.sampling_rate  )
																					UpperCamelCase__				     =			self.sampling_rate

														return {"path": path, "array": array, "sampling_rate": sampling_rate}






							def 				_lowerCamelCase		(						self  ):
														from .features import Value

														if self.decode:
																					raise ValueError("""Cannot flatten a decoded Audio feature."""  )
														return {
														    "bytes": Value("""binary"""  ),
														    "path": Value("""string"""  ),
														}






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														if pa.types.is_string(storage.type  ):
																					UpperCamelCase__				     =			pa.array([None] * len(__lowerCAmelCase  )  , type=pa.binary()  )
																					UpperCamelCase__				     =			pa.StructArray.from_arrays([bytes_array, storage]  , ["""bytes""", """path"""]  , mask=storage.is_null()  )
														elif pa.types.is_binary(storage.type  ):
																					UpperCamelCase__				     =			pa.array([None] * len(__lowerCAmelCase  )  , type=pa.string()  )
																					UpperCamelCase__				     =			pa.StructArray.from_arrays([storage, path_array]  , ["""bytes""", """path"""]  , mask=storage.is_null()  )
														elif pa.types.is_struct(storage.type  ) and storage.type.get_all_field_indices("""array"""  ):
																					UpperCamelCase__				     =			pa.array([Audio().encode_example(__lowerCAmelCase  ) if x is not None else None for x in storage.to_pylist()]  )
														elif pa.types.is_struct(storage.type  ):
																					if storage.type.get_field_index("""bytes"""  ) >= 0:
																												UpperCamelCase__				     =			storage.field("""bytes"""  )
																					else:
																												UpperCamelCase__				     =			pa.array([None] * len(__lowerCAmelCase  )  , type=pa.binary()  )
																					if storage.type.get_field_index("""path"""  ) >= 0:
																												UpperCamelCase__				     =			storage.field("""path"""  )
																					else:
																												UpperCamelCase__				     =			pa.array([None] * len(__lowerCAmelCase  )  , type=pa.string()  )
																					UpperCamelCase__				     =			pa.StructArray.from_arrays([bytes_array, path_array]  , ["""bytes""", """path"""]  , mask=storage.is_null()  )
														return array_cast(__lowerCAmelCase  , self.pa_type  )







							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):

														@no_op_if_value_is_null
														def path_to_bytes(__lowerCAmelCase  ):
																					with xopen(__lowerCAmelCase  , """rb"""  ) as f:
																												UpperCamelCase__				     =			f.read()
																					return bytes_

														UpperCamelCase__				     =			pa.array(
														    [
														        (path_to_bytes(x["""path"""]  ) if x["""bytes"""] is None else x["""bytes"""]) if x is not None else None
														        for x in storage.to_pylist()
														    ]  , type=pa.binary()  , )
														UpperCamelCase__				     =			pa.array(
														    [os.path.basename(__lowerCAmelCase  ) if path is not None else None for path in storage.field("""path"""  ).to_pylist()]  , type=pa.string()  , )
														UpperCamelCase__				     =			pa.StructArray.from_arrays([bytes_array, path_array]  , ["""bytes""", """path"""]  , mask=bytes_array.is_null()  )
														return array_cast(__lowerCAmelCase  , self.pa_type  )











import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY

@is_pipeline_test
class        __SCREAMING_SNAKE_CASE					(     unittest.TestCase       ):
							snake_case     :      Any						     =       MODEL_FOR_MASKED_LM_MAPPING
							snake_case     :      Dict						     =       TF_MODEL_FOR_MASKED_LM_MAPPING






							def 				_lowerCamelCase		(						self  ):
														super().tearDown()
														# clean-up as much as possible GPU memory occupied by PyTorch
														gc.collect()
														if is_torch_available():
																					import torch

																					torch.cuda.empty_cache()






							@require_tf
							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			pipeline(task="""fill-mask"""  , model="""sshleifer/tiny-distilroberta-base"""  , top_k=2  , framework="""tf"""  )
														UpperCamelCase__				     =			unmasker("""My name is <mask>"""  )
														self.assertEqual(
														    nested_simplify(__lowerCAmelCase  , decimals=6  )  , [
														        {"""sequence""": """My name is grouped""", """score""": 2.1E-05, """token""": 38015, """token_str""": """ grouped"""},
														        {"""sequence""": """My name is accuser""", """score""": 2.1E-05, """token""": 25506, """token_str""": """ accuser"""},
														    ]  , )

														UpperCamelCase__				     =			unmasker("""The largest city in France is <mask>"""  )
														self.assertEqual(
														    nested_simplify(__lowerCAmelCase  , decimals=6  )  , [
														        {
														            """sequence""": """The largest city in France is grouped""",
														            """score""": 2.1E-05,
														            """token""": 38015,
														            """token_str""": """ grouped""",
														        },
														        {
														            """sequence""": """The largest city in France is accuser""",
														            """score""": 2.1E-05,
														            """token""": 25506,
														            """token_str""": """ accuser""",
														        },
														    ]  , )

														UpperCamelCase__				     =			unmasker("""My name is <mask>"""  , targets=[""" Patrick""", """ Clara""", """ Teven"""]  , top_k=3  )
														self.assertEqual(
														    nested_simplify(__lowerCAmelCase  , decimals=6  )  , [
														        {"""sequence""": """My name is Clara""", """score""": 2E-05, """token""": 13606, """token_str""": """ Clara"""},
														        {"""sequence""": """My name is Patrick""", """score""": 2E-05, """token""": 3499, """token_str""": """ Patrick"""},
														        {"""sequence""": """My name is Te""", """score""": 1.9E-05, """token""": 2941, """token_str""": """ Te"""},
														    ]  , )






							@require_torch
							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			pipeline(task="""fill-mask"""  , model="""sshleifer/tiny-distilroberta-base"""  , top_k=2  , framework="""pt"""  )

														UpperCamelCase__				     =			unmasker("""My name is <mask>"""  )
														self.assertEqual(
														    nested_simplify(__lowerCAmelCase  , decimals=6  )  , [
														        {"""sequence""": """My name is Maul""", """score""": 2.2E-05, """token""": 35676, """token_str""": """ Maul"""},
														        {"""sequence""": """My name isELS""", """score""": 2.2E-05, """token""": 16416, """token_str""": """ELS"""},
														    ]  , )

														UpperCamelCase__				     =			unmasker("""The largest city in France is <mask>"""  )
														self.assertEqual(
														    nested_simplify(__lowerCAmelCase  , decimals=6  )  , [
														        {
														            """sequence""": """The largest city in France is Maul""",
														            """score""": 2.2E-05,
														            """token""": 35676,
														            """token_str""": """ Maul""",
														        },
														        {"""sequence""": """The largest city in France isELS""", """score""": 2.2E-05, """token""": 16416, """token_str""": """ELS"""},
														    ]  , )

														UpperCamelCase__				     =			unmasker("""My name is <mask>"""  , targets=[""" Patrick""", """ Clara""", """ Teven"""]  , top_k=3  )
														self.assertEqual(
														    nested_simplify(__lowerCAmelCase  , decimals=6  )  , [
														        {"""sequence""": """My name is Patrick""", """score""": 2.1E-05, """token""": 3499, """token_str""": """ Patrick"""},
														        {"""sequence""": """My name is Te""", """score""": 2E-05, """token""": 2941, """token_str""": """ Te"""},
														        {"""sequence""": """My name is Clara""", """score""": 2E-05, """token""": 13606, """token_str""": """ Clara"""},
														    ]  , )

														UpperCamelCase__				     =			unmasker("""My name is <mask> <mask>"""  , top_k=2  )

														self.assertEqual(
														    nested_simplify(__lowerCAmelCase  , decimals=6  )  , [
														        [
														            {
														                """score""": 2.2E-05,
														                """token""": 35676,
														                """token_str""": """ Maul""",
														                """sequence""": """<s>My name is Maul<mask></s>""",
														            },
														            {"""score""": 2.2E-05, """token""": 16416, """token_str""": """ELS""", """sequence""": """<s>My name isELS<mask></s>"""},
														        ],
														        [
														            {
														                """score""": 2.2E-05,
														                """token""": 35676,
														                """token_str""": """ Maul""",
														                """sequence""": """<s>My name is<mask> Maul</s>""",
														            },
														            {"""score""": 2.2E-05, """token""": 16416, """token_str""": """ELS""", """sequence""": """<s>My name is<mask>ELS</s>"""},
														        ],
														    ]  , )






							@require_torch_gpu
							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			pipeline("""fill-mask"""  , model="""hf-internal-testing/tiny-random-distilbert"""  , device=0  , framework="""pt"""  )

														# convert model to fp16
														pipe.model.half()

														UpperCamelCase__				     =			pipe("""Paris is the [MASK] of France."""  )
														# We actually don't care about the result, we just want to make sure
														# it works, meaning the float16 tensor got casted back to float32
														# for postprocessing.
														self.assertIsInstance(__lowerCAmelCase  , __lowerCAmelCase  )






							@slow
							@require_torch
							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			pipeline(task="""fill-mask"""  , model="""distilroberta-base"""  , top_k=2  , framework="""pt"""  )
														self.run_large_test(__lowerCAmelCase  )






							@slow
							@require_tf
							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			pipeline(task="""fill-mask"""  , model="""distilroberta-base"""  , top_k=2  , framework="""tf"""  )
														self.run_large_test(__lowerCAmelCase  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			unmasker("""My name is <mask>"""  )
														self.assertEqual(
														    nested_simplify(__lowerCAmelCase  )  , [
														        {"""sequence""": """My name is John""", """score""": 0.008, """token""": 610, """token_str""": """ John"""},
														        {"""sequence""": """My name is Chris""", """score""": 0.007, """token""": 1573, """token_str""": """ Chris"""},
														    ]  , )
														UpperCamelCase__				     =			unmasker("""The largest city in France is <mask>"""  )
														self.assertEqual(
														    nested_simplify(__lowerCAmelCase  )  , [
														        {
														            """sequence""": """The largest city in France is Paris""",
														            """score""": 0.251,
														            """token""": 2201,
														            """token_str""": """ Paris""",
														        },
														        {
														            """sequence""": """The largest city in France is Lyon""",
														            """score""": 0.214,
														            """token""": 12790,
														            """token_str""": """ Lyon""",
														        },
														    ]  , )

														UpperCamelCase__				     =			unmasker("""My name is <mask>"""  , targets=[""" Patrick""", """ Clara""", """ Teven"""]  , top_k=3  )
														self.assertEqual(
														    nested_simplify(__lowerCAmelCase  )  , [
														        {"""sequence""": """My name is Patrick""", """score""": 0.005, """token""": 3499, """token_str""": """ Patrick"""},
														        {"""sequence""": """My name is Clara""", """score""": 0.000, """token""": 13606, """token_str""": """ Clara"""},
														        {"""sequence""": """My name is Te""", """score""": 0.000, """token""": 2941, """token_str""": """ Te"""},
														    ]  , )






							@require_torch
							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			pipeline(task="""fill-mask"""  , model="""sshleifer/tiny-distilroberta-base"""  , framework="""pt"""  )
														UpperCamelCase__				     =			None
														UpperCamelCase__				     =			None
														self.run_pipeline_test(__lowerCAmelCase  , []  )






							@require_tf
							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			pipeline(task="""fill-mask"""  , model="""sshleifer/tiny-distilroberta-base"""  , framework="""tf"""  )
														UpperCamelCase__				     =			None
														UpperCamelCase__				     =			None
														self.run_pipeline_test(__lowerCAmelCase  , []  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  ):
														if tokenizer is None or tokenizer.mask_token_id is None:
																					self.skipTest("""The provided tokenizer has no mask token, (probably reformer or wav2vec2)"""  )

														UpperCamelCase__				     =			FillMaskPipeline(model=__lowerCAmelCase  , tokenizer=__lowerCAmelCase  )
														UpperCamelCase__				     =			[
														    f"""This is another {tokenizer.mask_token} test""",
														]
														return fill_masker, examples






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			fill_masker.tokenizer
														UpperCamelCase__				     =			fill_masker.model

														UpperCamelCase__				     =			fill_masker(
														    f"""This is a {tokenizer.mask_token}"""  , )
														self.assertEqual(
														    __lowerCAmelCase  , [
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														    ]  , )

														UpperCamelCase__				     =			fill_masker([f"""This is a {tokenizer.mask_token}"""]  )
														self.assertEqual(
														    __lowerCAmelCase  , [
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														    ]  , )

														UpperCamelCase__				     =			fill_masker([f"""This is a {tokenizer.mask_token}""", f"""Another {tokenizer.mask_token} great test."""]  )
														self.assertEqual(
														    __lowerCAmelCase  , [
														        [
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        ],
														        [
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        ],
														    ]  , )

														with self.assertRaises(__lowerCAmelCase  ):
																					fill_masker([None]  )
														# No mask_token is not supported
														with self.assertRaises(__lowerCAmelCase  ):
																					fill_masker("""This is"""  )

														self.run_test_top_k(__lowerCAmelCase  , __lowerCAmelCase  )
														self.run_test_targets(__lowerCAmelCase  , __lowerCAmelCase  )
														self.run_test_top_k_targets(__lowerCAmelCase  , __lowerCAmelCase  )
														self.fill_mask_with_duplicate_targets_and_top_k(__lowerCAmelCase  , __lowerCAmelCase  )
														self.fill_mask_with_multiple_masks(__lowerCAmelCase  , __lowerCAmelCase  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			tokenizer.get_vocab()
														UpperCamelCase__				     =			sorted(vocab.keys()  )[:2]
														# Pipeline argument
														UpperCamelCase__				     =			FillMaskPipeline(model=__lowerCAmelCase  , tokenizer=__lowerCAmelCase  , targets=__lowerCAmelCase  )
														UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  )
														self.assertEqual(
														    __lowerCAmelCase  , [
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														    ]  , )
														UpperCamelCase__				     =			{vocab[el] for el in targets}
														self.assertEqual({el["""token"""] for el in outputs}  , __lowerCAmelCase  )
														UpperCamelCase__				     =			[tokenizer.decode([x]  ) for x in target_ids]
														self.assertEqual({el["""token_str"""] for el in outputs}  , set(__lowerCAmelCase  )  )

														# Call argument
														UpperCamelCase__				     =			FillMaskPipeline(model=__lowerCAmelCase  , tokenizer=__lowerCAmelCase  )
														UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  , targets=__lowerCAmelCase  )
														self.assertEqual(
														    __lowerCAmelCase  , [
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														    ]  , )
														UpperCamelCase__				     =			{vocab[el] for el in targets}
														self.assertEqual({el["""token"""] for el in outputs}  , __lowerCAmelCase  )
														UpperCamelCase__				     =			[tokenizer.decode([x]  ) for x in target_ids]
														self.assertEqual({el["""token_str"""] for el in outputs}  , set(__lowerCAmelCase  )  )

														# Score equivalence
														UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  , targets=__lowerCAmelCase  )
														UpperCamelCase__				     =			[top_mask["""token_str"""] for top_mask in outputs]
														UpperCamelCase__				     =			[top_mask["""score"""] for top_mask in outputs]

														# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
														if set(__lowerCAmelCase  ) == set(__lowerCAmelCase  ):
																					UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  , targets=__lowerCAmelCase  )
																					UpperCamelCase__				     =			[top_mask["""score"""] for top_mask in unmasked_targets]
																					self.assertEqual(nested_simplify(__lowerCAmelCase  )  , nested_simplify(__lowerCAmelCase  )  )

														# Raises with invalid
														with self.assertRaises(__lowerCAmelCase  ):
																					UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  , targets=[]  )
														# For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
														if "" not in tokenizer.get_vocab():
																					with self.assertRaises(__lowerCAmelCase  ):
																												UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  , targets=[""""""]  )
																					with self.assertRaises(__lowerCAmelCase  ):
																												UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  , targets=""""""  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			FillMaskPipeline(model=__lowerCAmelCase  , tokenizer=__lowerCAmelCase  , top_k=2  )
														UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  )
														self.assertEqual(
														    __lowerCAmelCase  , [
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														    ]  , )

														UpperCamelCase__				     =			FillMaskPipeline(model=__lowerCAmelCase  , tokenizer=__lowerCAmelCase  )
														UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  , top_k=2  )
														self.assertEqual(
														    __lowerCAmelCase  , [
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														    ]  , )
														self.assertEqual(nested_simplify(__lowerCAmelCase  )  , nested_simplify(__lowerCAmelCase  )  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			tokenizer.get_vocab()
														UpperCamelCase__				     =			FillMaskPipeline(model=__lowerCAmelCase  , tokenizer=__lowerCAmelCase  )

														# top_k=2, ntargets=3
														UpperCamelCase__				     =			sorted(vocab.keys()  )[:3]
														UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  , top_k=2  , targets=__lowerCAmelCase  )

														# If we use the most probably targets, and filter differently, we should still
														# have the same results
														UpperCamelCase__				     =			[el["""token_str"""] for el in sorted(__lowerCAmelCase  , key=lambda __lowerCAmelCase  : x["score"]  , reverse=__lowerCAmelCase  )]
														# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
														if set(__lowerCAmelCase  ).issubset(__lowerCAmelCase  ):
																					UpperCamelCase__				     =			fill_masker(f"""This is a {tokenizer.mask_token}"""  , top_k=3  , targets=__lowerCAmelCase  )
																					# They should yield exactly the same result
																					self.assertEqual(nested_simplify(__lowerCAmelCase  )  , nested_simplify(__lowerCAmelCase  )  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			FillMaskPipeline(model=__lowerCAmelCase  , tokenizer=__lowerCAmelCase  )
														UpperCamelCase__				     =			tokenizer.get_vocab()
														# String duplicates + id duplicates
														UpperCamelCase__				     =			sorted(vocab.keys()  )[:3]
														UpperCamelCase__				     =			[targets[0], targets[1], targets[0], targets[2], targets[1]]
														UpperCamelCase__				     =			fill_masker(f"""My name is {tokenizer.mask_token}"""  , targets=__lowerCAmelCase  , top_k=10  )

														# The target list contains duplicates, so we can't output more
														# than them
														self.assertEqual(len(__lowerCAmelCase  )  , 3  )







							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			FillMaskPipeline(model=__lowerCAmelCase  , tokenizer=__lowerCAmelCase  )

														UpperCamelCase__				     =			fill_masker(
														    f"""This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}"""  , top_k=2  )
														self.assertEqual(
														    __lowerCAmelCase  , [
														        [
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        ],
														        [
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        ],
														        [
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														            {"""sequence""": ANY(__lowerCAmelCase  ), """score""": ANY(__lowerCAmelCase  ), """token""": ANY(__lowerCAmelCase  ), """token_str""": ANY(__lowerCAmelCase  )},
														        ],
														    ]  , )




