





import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


__lowercase	: Any       =datasets.utils.logging.get_logger(__name__)

@dataclass
class   A							(						datasets.BuilderConfig    ):
      _snake_case				  =None
      _snake_case				  ="utf-8"
      _snake_case				  =None
      _snake_case				  =None
      _snake_case				  =True  # deprecated
      _snake_case				  =None  # deprecated
      _snake_case				  =10 << 20  # 10MB
      _snake_case				  =None

class   A							(						datasets.ArrowBasedBuilder    ):
      _snake_case				  =JsonConfig



      def 						lowerCAmelCase__     (						self:    Any							)    ->     Union[str, Any]:
         '''simple docstring'''

         if self.config.block_size is not None:
            logger.warning("The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"							)
            UpperCAmelCase_           =self.config.block_size
         if self.config.use_threads is not True:
            logger.warning(
                "The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."							)
         if self.config.newlines_in_values is not None:
            raise ValueError("The JSON loader parameter `newlines_in_values` is no longer supported"							)
         return datasets.DatasetInfo(features=self.config.features							)



      def 						lowerCAmelCase__     (						self:    int		,	_lowerCAmelCase:    int							)    ->     str:
         '''simple docstring'''

         if not self.config.data_files:
            raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}'							)
         UpperCAmelCase_           =dl_manager.download_and_extract(self.config.data_files							)
         if isinstance(_lowerCAmelCase		,	(str, list, tuple)							):
            UpperCAmelCase_           =data_files
            if isinstance(_lowerCAmelCase		,	_lowerCAmelCase							):
               UpperCAmelCase_           =[files]
            UpperCAmelCase_           =[dl_manager.iter_files(_lowerCAmelCase							) for file in files]
            return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,	gen_kwargs={"files": files}							)]
         UpperCAmelCase_           =[]
         for split_name, files in data_files.items():
            if isinstance(_lowerCAmelCase		,	_lowerCAmelCase							):
               UpperCAmelCase_           =[files]
            UpperCAmelCase_           =[dl_manager.iter_files(_lowerCAmelCase							) for file in files]
            splits.append(datasets.SplitGenerator(name=_lowerCAmelCase		,	gen_kwargs={"files": files}							)							)
         return splits



      def 						lowerCAmelCase__     (						self:    Tuple		,	_lowerCAmelCase:    pa.Table							)    ->     pa.Table:
         '''simple docstring'''

         if self.config.features is not None:
            # adding missing columns
            for column_name in set(self.config.features							) - set(pa_table.column_names							):
               UpperCAmelCase_           =self.config.features.arrow_schema.field(_lowerCAmelCase							).type
               UpperCAmelCase_           =pa_table.append_column(_lowerCAmelCase		,	pa.array([None] * len(_lowerCAmelCase							)		,	type=_lowerCAmelCase							)							)
            # more expensive cast to support nested structures with keys in a different order
            # allows str <-> int/float or str to Audio for example
            UpperCAmelCase_           =table_cast(_lowerCAmelCase		,	self.config.features.arrow_schema							)
         return pa_table






      def 						lowerCAmelCase__     (						self:    Optional[Any]		,	_lowerCAmelCase:    int							)    ->     Tuple:
         '''simple docstring'''

         for file_idx, file in enumerate(itertools.chain.from_iterable(_lowerCAmelCase							)							):
            # If the file is one json object and if we need to look at the list of items in one specific field
            if self.config.field is not None:
               with open(_lowerCAmelCase		,	encoding=self.config.encoding		,	errors=self.config.encoding_errors							) as f:
                  UpperCAmelCase_           =json.load(_lowerCAmelCase							)

               # We keep only the field we are interested in
               UpperCAmelCase_           =dataset[self.config.field]

               # We accept two format: a list of dicts or a dict of lists
               if isinstance(_lowerCAmelCase		,	(list, tuple)							):
                  UpperCAmelCase_           =set().union(*[row.keys() for row in dataset]							)
                  UpperCAmelCase_           ={col: [row.get(_lowerCAmelCase							) for row in dataset] for col in keys}
               else:
                  UpperCAmelCase_           =dataset
               UpperCAmelCase_           =pa.Table.from_pydict(_lowerCAmelCase							)
               yield file_idx, self._cast_table(_lowerCAmelCase							)

            # If the file has one json object per line
            else:
               with open(_lowerCAmelCase		,	"rb"							) as f:
                  UpperCAmelCase_           =0
                  # Use block_size equal to the chunk size divided by 32 to leverage multithreading
                  # Set a default minimum value of 16kB if the chunk size is really small
                  UpperCAmelCase_           =max(self.config.chunksize // 32		,	16 << 10							)
                  UpperCAmelCase_           =(
                      self.config.encoding_errors if self.config.encoding_errors is not None else "strict"
                  )
                  while True:
                     UpperCAmelCase_           =f.read(self.config.chunksize							)
                     if not batch:
                        break
                     # Finish current line
                     try:
                        batch += f.readline()
                     except (AttributeError, io.UnsupportedOperation):
                        batch += readline(_lowerCAmelCase							)
                     # PyArrow only accepts utf-8 encoded bytes
                     if self.config.encoding != "utf-8":
                        UpperCAmelCase_           =batch.decode(self.config.encoding		,	errors=_lowerCAmelCase							).encode("utf-8"							)
                     try:
                        while True:
                           try:
                              UpperCAmelCase_           =paj.read_json(
                                  io.BytesIO(_lowerCAmelCase							)		,	read_options=paj.ReadOptions(block_size=_lowerCAmelCase							)							)
                              break
                           except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                              if (
                                  isinstance(_lowerCAmelCase		,	pa.ArrowInvalid							)
                                  and "straddling" not in str(_lowerCAmelCase							)
                                  or block_size > len(_lowerCAmelCase							)
                              ):
                                 raise
                              else:
                                 # Increase the block size in case it was too small.
                                 # The block size will be reset for the next file.
                                 logger.debug(
                                     F'Batch of {len(_lowerCAmelCase							)} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'							)
                                 block_size *= 2
                     except pa.ArrowInvalid as e:
                        try:
                           with open(
                               _lowerCAmelCase		,	encoding=self.config.encoding		,	errors=self.config.encoding_errors							) as f:
                              UpperCAmelCase_           =json.load(_lowerCAmelCase							)
                        except json.JSONDecodeError:
                           logger.error(F'Failed to read file \'{file}\' with error {type(_lowerCAmelCase							)}: {e}'							)
                           raise e
                        # If possible, parse the file as a list of json objects and exit the loop
                        if isinstance(_lowerCAmelCase		,	_lowerCAmelCase							):  # list is the only sequence type supported in JSON
                           try:
                              UpperCAmelCase_           =set().union(*[row.keys() for row in dataset]							)
                              UpperCAmelCase_           ={col: [row.get(_lowerCAmelCase							) for row in dataset] for col in keys}
                              UpperCAmelCase_           =pa.Table.from_pydict(_lowerCAmelCase							)
                           except (pa.ArrowInvalid, AttributeError) as e:
                              logger.error(F'Failed to read file \'{file}\' with error {type(_lowerCAmelCase							)}: {e}'							)
                              raise ValueError(F'Not able to read records in the JSON file at {file}.'							) from None
                           yield file_idx, self._cast_table(_lowerCAmelCase							)
                           break
                        else:
                           logger.error(F'Failed to read file \'{file}\' with error {type(_lowerCAmelCase							)}: {e}'							)
                           raise ValueError(
                               F'Not able to read records in the JSON file at {file}. '
                               F'You should probably indicate the field of the JSON file containing your records. '
                               F'This JSON file contain the following fields: {str(list(dataset.keys()							)							)}. '
                               F'Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '							) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                     yield (file_idx, batch_idx), self._cast_table(_lowerCAmelCase							)
                     batch_idx += 1










import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY







@is_pipeline_test
class        __lowerCAmelCase					(			unittest.TestCase					):

			"""simple docstring"""







			snake_case_  						=			MODEL_FOR_MASKED_LM_MAPPING
			snake_case_  						=			TF_MODEL_FOR_MASKED_LM_MAPPING


			def  lowercase_       ( self				)       -> str:



							'''simple docstring'''
							super().tearDown()
							# clean-up as much as possible GPU memory occupied by PyTorch
							gc.collect()
							if is_torch_available():
											import torch

											torch.cuda.empty_cache()


			@require_tf
			def  lowercase_       ( self				)       -> Any:



							'''simple docstring'''
							__lowerCamelCase									=       pipeline(task='fill-mask'     ,		model='sshleifer/tiny-distilroberta-base'     ,		top_k=2     ,		framework='tf'				)
							__lowerCamelCase									=       unmasker('My name is <mask>'				)
							self.assertEqual(
							    nested_simplify(lowerCamelCase__     ,		decimals=6				)     ,		[
							        {'sequence': 'My name is grouped', 'score': 2.1e-05, 'token': 38_015, 'token_str': ' grouped'},
							        {'sequence': 'My name is accuser', 'score': 2.1e-05, 'token': 25_506, 'token_str': ' accuser'},
							    ]     ,		)

							__lowerCamelCase									=       unmasker('The largest city in France is <mask>'				)
							self.assertEqual(
							    nested_simplify(lowerCamelCase__     ,		decimals=6				)     ,		[
							        {
							            'sequence': 'The largest city in France is grouped',
							            'score': 2.1e-05,
							            'token': 38_015,
							            'token_str': ' grouped',
							        },
							        {
							            'sequence': 'The largest city in France is accuser',
							            'score': 2.1e-05,
							            'token': 25_506,
							            'token_str': ' accuser',
							        },
							    ]     ,		)

							__lowerCamelCase									=       unmasker('My name is <mask>'     ,		targets=[' Patrick', ' Clara', ' Teven']     ,		top_k=3				)
							self.assertEqual(
							    nested_simplify(lowerCamelCase__     ,		decimals=6				)     ,		[
							        {'sequence': 'My name is Clara', 'score': 2e-05, 'token': 13_606, 'token_str': ' Clara'},
							        {'sequence': 'My name is Patrick', 'score': 2e-05, 'token': 3_499, 'token_str': ' Patrick'},
							        {'sequence': 'My name is Te', 'score': 1.9e-05, 'token': 2_941, 'token_str': ' Te'},
							    ]     ,		)


			@require_torch
			def  lowercase_       ( self				)       -> str:



							'''simple docstring'''
							__lowerCamelCase									=       pipeline(task='fill-mask'     ,		model='sshleifer/tiny-distilroberta-base'     ,		top_k=2     ,		framework='pt'				)

							__lowerCamelCase									=       unmasker('My name is <mask>'				)
							self.assertEqual(
							    nested_simplify(lowerCamelCase__     ,		decimals=6				)     ,		[
							        {'sequence': 'My name is Maul', 'score': 2.2e-05, 'token': 35_676, 'token_str': ' Maul'},
							        {'sequence': 'My name isELS', 'score': 2.2e-05, 'token': 16_416, 'token_str': 'ELS'},
							    ]     ,		)

							__lowerCamelCase									=       unmasker('The largest city in France is <mask>'				)
							self.assertEqual(
							    nested_simplify(lowerCamelCase__     ,		decimals=6				)     ,		[
							        {
							            'sequence': 'The largest city in France is Maul',
							            'score': 2.2e-05,
							            'token': 35_676,
							            'token_str': ' Maul',
							        },
							        {'sequence': 'The largest city in France isELS', 'score': 2.2e-05, 'token': 16_416, 'token_str': 'ELS'},
							    ]     ,		)

							__lowerCamelCase									=       unmasker('My name is <mask>'     ,		targets=[' Patrick', ' Clara', ' Teven']     ,		top_k=3				)
							self.assertEqual(
							    nested_simplify(lowerCamelCase__     ,		decimals=6				)     ,		[
							        {'sequence': 'My name is Patrick', 'score': 2.1e-05, 'token': 3_499, 'token_str': ' Patrick'},
							        {'sequence': 'My name is Te', 'score': 2e-05, 'token': 2_941, 'token_str': ' Te'},
							        {'sequence': 'My name is Clara', 'score': 2e-05, 'token': 13_606, 'token_str': ' Clara'},
							    ]     ,		)

							__lowerCamelCase									=       unmasker('My name is <mask> <mask>'     ,		top_k=2				)

							self.assertEqual(
							    nested_simplify(lowerCamelCase__     ,		decimals=6				)     ,		[
							        [
							            {
							                'score': 2.2e-05,
							                'token': 35_676,
							                'token_str': ' Maul',
							                'sequence': '<s>My name is Maul<mask></s>',
							            },
							            {'score': 2.2e-05, 'token': 16_416, 'token_str': 'ELS', 'sequence': '<s>My name isELS<mask></s>'},
							        ],
							        [
							            {
							                'score': 2.2e-05,
							                'token': 35_676,
							                'token_str': ' Maul',
							                'sequence': '<s>My name is<mask> Maul</s>',
							            },
							            {'score': 2.2e-05, 'token': 16_416, 'token_str': 'ELS', 'sequence': '<s>My name is<mask>ELS</s>'},
							        ],
							    ]     ,		)


			@require_torch_gpu
			def  lowercase_       ( self				)       -> int:



							'''simple docstring'''
							__lowerCamelCase									=       pipeline('fill-mask'     ,		model='hf-internal-testing/tiny-random-distilbert'     ,		device=0     ,		framework='pt'				)

							# convert model to fp16
							pipe.model.half()

							__lowerCamelCase									=       pipe('Paris is the [MASK] of France.'				)
							# We actually don't care about the result, we just want to make sure
							# it works, meaning the float16 tensor got casted back to float32
							# for postprocessing.
							self.assertIsInstance(lowerCamelCase__     ,		lowerCamelCase__				)


			@slow
			@require_torch
			def  lowercase_       ( self				)       -> str:



							'''simple docstring'''
							__lowerCamelCase									=       pipeline(task='fill-mask'     ,		model='distilroberta-base'     ,		top_k=2     ,		framework='pt'				)
							self.run_large_test(lowerCamelCase__				)


			@slow
			@require_tf
			def  lowercase_       ( self				)       -> str:



							'''simple docstring'''
							__lowerCamelCase									=       pipeline(task='fill-mask'     ,		model='distilroberta-base'     ,		top_k=2     ,		framework='tf'				)
							self.run_large_test(lowerCamelCase__				)


			def  lowercase_       ( self     ,		lowerCamelCase__				)       -> Optional[Any]:



							'''simple docstring'''
							__lowerCamelCase									=       unmasker('My name is <mask>'				)
							self.assertEqual(
							    nested_simplify(lowerCamelCase__				)     ,		[
							        {'sequence': 'My name is John', 'score': 0.0_08, 'token': 610, 'token_str': ' John'},
							        {'sequence': 'My name is Chris', 'score': 0.0_07, 'token': 1_573, 'token_str': ' Chris'},
							    ]     ,		)
							__lowerCamelCase									=       unmasker('The largest city in France is <mask>'				)
							self.assertEqual(
							    nested_simplify(lowerCamelCase__				)     ,		[
							        {
							            'sequence': 'The largest city in France is Paris',
							            'score': 0.2_51,
							            'token': 2_201,
							            'token_str': ' Paris',
							        },
							        {
							            'sequence': 'The largest city in France is Lyon',
							            'score': 0.2_14,
							            'token': 12_790,
							            'token_str': ' Lyon',
							        },
							    ]     ,		)

							__lowerCamelCase									=       unmasker('My name is <mask>'     ,		targets=[' Patrick', ' Clara', ' Teven']     ,		top_k=3				)
							self.assertEqual(
							    nested_simplify(lowerCamelCase__				)     ,		[
							        {'sequence': 'My name is Patrick', 'score': 0.0_05, 'token': 3_499, 'token_str': ' Patrick'},
							        {'sequence': 'My name is Clara', 'score': 0.0_00, 'token': 13_606, 'token_str': ' Clara'},
							        {'sequence': 'My name is Te', 'score': 0.0_00, 'token': 2_941, 'token_str': ' Te'},
							    ]     ,		)


			@require_torch
			def  lowercase_       ( self				)       -> int:



							'''simple docstring'''
							__lowerCamelCase									=       pipeline(task='fill-mask'     ,		model='sshleifer/tiny-distilroberta-base'     ,		framework='pt'				)
							__lowerCamelCase									=       None
							__lowerCamelCase									=       None
							self.run_pipeline_test(lowerCamelCase__     ,		[]				)


			@require_tf
			def  lowercase_       ( self				)       -> str:



							'''simple docstring'''
							__lowerCamelCase									=       pipeline(task='fill-mask'     ,		model='sshleifer/tiny-distilroberta-base'     ,		framework='tf'				)
							__lowerCamelCase									=       None
							__lowerCamelCase									=       None
							self.run_pipeline_test(lowerCamelCase__     ,		[]				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__				)       -> List[str]:



							'''simple docstring'''
							if tokenizer is None or tokenizer.mask_token_id is None:
											self.skipTest('The provided tokenizer has no mask token, (probably reformer or wav2vec2)'				)

							__lowerCamelCase									=       FillMaskPipeline(model=lowerCamelCase__     ,		tokenizer=lowerCamelCase__				)
							__lowerCamelCase									=       [
							    f"""This is another {tokenizer.mask_token} test""",
							]
							return fill_masker, examples


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__				)       -> Optional[int]:



							'''simple docstring'''
							__lowerCamelCase									=       fill_masker.tokenizer
							__lowerCamelCase									=       fill_masker.model

							__lowerCamelCase									=       fill_masker(
							    f"""This is a {tokenizer.mask_token}"""     ,		)
							self.assertEqual(
							    lowerCamelCase__     ,		[
							        {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							        {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							        {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							        {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							        {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							    ]     ,		)

							__lowerCamelCase									=       fill_masker([f"""This is a {tokenizer.mask_token}"""]				)
							self.assertEqual(
							    lowerCamelCase__     ,		[
							        {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							        {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							        {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							        {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							        {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							    ]     ,		)

							__lowerCamelCase									=       fill_masker([f"""This is a {tokenizer.mask_token}""", f"""Another {tokenizer.mask_token} great test."""]				)
							self.assertEqual(
							    lowerCamelCase__     ,		[
							        [
							            {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							            {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							            {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							            {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							            {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							        ],
							        [
							            {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							            {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							            {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							            {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							            {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							        ],
							    ]     ,		)

							with self.assertRaises(lowerCamelCase__				):
											fill_masker([None]				)
							# No mask_token is not supported
							with self.assertRaises(lowerCamelCase__				):
											fill_masker('This is'				)

							self.run_test_top_k(lowerCamelCase__     ,		lowerCamelCase__				)
							self.run_test_targets(lowerCamelCase__     ,		lowerCamelCase__				)
							self.run_test_top_k_targets(lowerCamelCase__     ,		lowerCamelCase__				)
							self.fill_mask_with_duplicate_targets_and_top_k(lowerCamelCase__     ,		lowerCamelCase__				)
							self.fill_mask_with_multiple_masks(lowerCamelCase__     ,		lowerCamelCase__				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__				)       -> Dict:



							'''simple docstring'''
							__lowerCamelCase									=       tokenizer.get_vocab()
							__lowerCamelCase									=       sorted(vocab.keys()				)[:2]
							# Pipeline argument
							__lowerCamelCase									=       FillMaskPipeline(model=lowerCamelCase__     ,		tokenizer=lowerCamelCase__     ,		targets=lowerCamelCase__				)
							__lowerCamelCase									=       fill_masker(f"""This is a {tokenizer.mask_token}"""				)
							self.assertEqual(
							    lowerCamelCase__     ,		[
							        {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							        {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							    ]     ,		)
							__lowerCamelCase									=       {vocab[el] for el in targets}
							self.assertEqual({el['token'] for el in outputs}     ,		lowerCamelCase__				)
							__lowerCamelCase									=       [tokenizer.decode([x]				) for x in target_ids]
							self.assertEqual({el['token_str'] for el in outputs}     ,		set(lowerCamelCase__				)				)

							# Call argument
							__lowerCamelCase									=       FillMaskPipeline(model=lowerCamelCase__     ,		tokenizer=lowerCamelCase__				)
							__lowerCamelCase									=       fill_masker(f"""This is a {tokenizer.mask_token}"""     ,		targets=lowerCamelCase__				)
							self.assertEqual(
							    lowerCamelCase__     ,		[
							        {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							        {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							    ]     ,		)
							__lowerCamelCase									=       {vocab[el] for el in targets}
							self.assertEqual({el['token'] for el in outputs}     ,		lowerCamelCase__				)
							__lowerCamelCase									=       [tokenizer.decode([x]				) for x in target_ids]
							self.assertEqual({el['token_str'] for el in outputs}     ,		set(lowerCamelCase__				)				)

							# Score equivalence
							__lowerCamelCase									=       fill_masker(f"""This is a {tokenizer.mask_token}"""     ,		targets=lowerCamelCase__				)
							__lowerCamelCase									=       [top_mask['token_str'] for top_mask in outputs]
							__lowerCamelCase									=       [top_mask['score'] for top_mask in outputs]

							# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
							if set(lowerCamelCase__				) == set(lowerCamelCase__				):
											__lowerCamelCase									=       fill_masker(f"""This is a {tokenizer.mask_token}"""     ,		targets=lowerCamelCase__				)
											__lowerCamelCase									=       [top_mask['score'] for top_mask in unmasked_targets]
											self.assertEqual(nested_simplify(lowerCamelCase__				)     ,		nested_simplify(lowerCamelCase__				)				)

							# Raises with invalid
							with self.assertRaises(lowerCamelCase__				):
											__lowerCamelCase									=       fill_masker(f"""This is a {tokenizer.mask_token}"""     ,		targets=[]				)
							# For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
							if "" not in tokenizer.get_vocab():
											with self.assertRaises(lowerCamelCase__				):
															__lowerCamelCase									=       fill_masker(f"""This is a {tokenizer.mask_token}"""     ,		targets=['']				)
											with self.assertRaises(lowerCamelCase__				):
															__lowerCamelCase									=       fill_masker(f"""This is a {tokenizer.mask_token}"""     ,		targets=''				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__				)       -> Optional[int]:



							'''simple docstring'''
							__lowerCamelCase									=       FillMaskPipeline(model=lowerCamelCase__     ,		tokenizer=lowerCamelCase__     ,		top_k=2				)
							__lowerCamelCase									=       fill_masker(f"""This is a {tokenizer.mask_token}"""				)
							self.assertEqual(
							    lowerCamelCase__     ,		[
							        {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							        {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							    ]     ,		)

							__lowerCamelCase									=       FillMaskPipeline(model=lowerCamelCase__     ,		tokenizer=lowerCamelCase__				)
							__lowerCamelCase									=       fill_masker(f"""This is a {tokenizer.mask_token}"""     ,		top_k=2				)
							self.assertEqual(
							    lowerCamelCase__     ,		[
							        {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							        {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							    ]     ,		)
							self.assertEqual(nested_simplify(lowerCamelCase__				)     ,		nested_simplify(lowerCamelCase__				)				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__				)       -> int:



							'''simple docstring'''
							__lowerCamelCase									=       tokenizer.get_vocab()
							__lowerCamelCase									=       FillMaskPipeline(model=lowerCamelCase__     ,		tokenizer=lowerCamelCase__				)

							# top_k=2, ntargets=3
							__lowerCamelCase									=       sorted(vocab.keys()				)[:3]
							__lowerCamelCase									=       fill_masker(f"""This is a {tokenizer.mask_token}"""     ,		top_k=2     ,		targets=lowerCamelCase__				)

							# If we use the most probably targets, and filter differently, we should still
							# have the same results
							__lowerCamelCase									=       [el['token_str'] for el in sorted(lowerCamelCase__     ,		key=lambda lowerCamelCase__				: x["score"]     ,		reverse=lowerCamelCase__				)]
							# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
							if set(lowerCamelCase__				).issubset(lowerCamelCase__				):
											__lowerCamelCase									=       fill_masker(f"""This is a {tokenizer.mask_token}"""     ,		top_k=3     ,		targets=lowerCamelCase__				)
											# They should yield exactly the same result
											self.assertEqual(nested_simplify(lowerCamelCase__				)     ,		nested_simplify(lowerCamelCase__				)				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__				)       -> List[Any]:



							'''simple docstring'''
							__lowerCamelCase									=       FillMaskPipeline(model=lowerCamelCase__     ,		tokenizer=lowerCamelCase__				)
							__lowerCamelCase									=       tokenizer.get_vocab()
							# String duplicates + id duplicates
							__lowerCamelCase									=       sorted(vocab.keys()				)[:3]
							__lowerCamelCase									=       [targets[0], targets[1], targets[0], targets[2], targets[1]]
							__lowerCamelCase									=       fill_masker(f"""My name is {tokenizer.mask_token}"""     ,		targets=lowerCamelCase__     ,		top_k=10				)

							# The target list contains duplicates, so we can't output more
							# than them
							self.assertEqual(len(lowerCamelCase__				)     ,		3				)






			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__				)       -> Union[str, Any]:



							'''simple docstring'''
							__lowerCamelCase									=       FillMaskPipeline(model=lowerCamelCase__     ,		tokenizer=lowerCamelCase__				)

							__lowerCamelCase									=       fill_masker(
							    f"""This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}"""     ,		top_k=2				)
							self.assertEqual(
							    lowerCamelCase__     ,		[
							        [
							            {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							            {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							        ],
							        [
							            {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							            {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							        ],
							        [
							            {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							            {'sequence': ANY(lowerCamelCase__				), 'score': ANY(lowerCamelCase__				), 'token': ANY(lowerCamelCase__				), 'token_str': ANY(lowerCamelCase__				)},
							        ],
							    ]     ,		)


