'''simple docstring'''




import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY

@is_pipeline_test
class 	_lowercase(  unittest.TestCase					):



   """simple docstring"""





   __lowerCamelCase     =	MODEL_FOR_MASKED_LM_MAPPING
   __lowerCamelCase     =	TF_MODEL_FOR_MASKED_LM_MAPPING





   def       snake_case						(				self:     List[str]						):
    super().tearDown()
    # clean-up as much as possible GPU memory occupied by PyTorch
    gc.collect()
    if is_torch_available():
     import torch

     torch.cuda.empty_cache()





   @require_tf
   def       snake_case						(				self:     Dict						):
    __UpperCAmelCase								=       pipeline(task='fill-mask'      ,model='sshleifer/tiny-distilroberta-base'      ,top_k=2      ,framework='tf'						)
    __UpperCAmelCase								=       unmasker('My name is <mask>'						)
    self.assertEqual(
        nested_simplify(_lowercase      ,decimals=6						)      ,[
            {'sequence': 'My name is grouped', 'score': 2.1e-05, 'token': 38015, 'token_str': ' grouped'},
            {'sequence': 'My name is accuser', 'score': 2.1e-05, 'token': 25506, 'token_str': ' accuser'},
        ]      ,)

    __UpperCAmelCase								=       unmasker('The largest city in France is <mask>'						)
    self.assertEqual(
        nested_simplify(_lowercase      ,decimals=6						)      ,[
            {
                'sequence': 'The largest city in France is grouped',
                'score': 2.1e-05,
                'token': 38015,
                'token_str': ' grouped',
            },
            {
                'sequence': 'The largest city in France is accuser',
                'score': 2.1e-05,
                'token': 25506,
                'token_str': ' accuser',
            },
        ]      ,)

    __UpperCAmelCase								=       unmasker('My name is <mask>'      ,targets=[' Patrick', ' Clara', ' Teven']      ,top_k=3						)
    self.assertEqual(
        nested_simplify(_lowercase      ,decimals=6						)      ,[
            {'sequence': 'My name is Clara', 'score': 2e-05, 'token': 13606, 'token_str': ' Clara'},
            {'sequence': 'My name is Patrick', 'score': 2e-05, 'token': 3499, 'token_str': ' Patrick'},
            {'sequence': 'My name is Te', 'score': 1.9e-05, 'token': 2941, 'token_str': ' Te'},
        ]      ,)





   @require_torch
   def       snake_case						(				self:     Dict						):
    __UpperCAmelCase								=       pipeline(task='fill-mask'      ,model='sshleifer/tiny-distilroberta-base'      ,top_k=2      ,framework='pt'						)

    __UpperCAmelCase								=       unmasker('My name is <mask>'						)
    self.assertEqual(
        nested_simplify(_lowercase      ,decimals=6						)      ,[
            {'sequence': 'My name is Maul', 'score': 2.2e-05, 'token': 35676, 'token_str': ' Maul'},
            {'sequence': 'My name isELS', 'score': 2.2e-05, 'token': 16416, 'token_str': 'ELS'},
        ]      ,)

    __UpperCAmelCase								=       unmasker('The largest city in France is <mask>'						)
    self.assertEqual(
        nested_simplify(_lowercase      ,decimals=6						)      ,[
            {
                'sequence': 'The largest city in France is Maul',
                'score': 2.2e-05,
                'token': 35676,
                'token_str': ' Maul',
            },
            {'sequence': 'The largest city in France isELS', 'score': 2.2e-05, 'token': 16416, 'token_str': 'ELS'},
        ]      ,)

    __UpperCAmelCase								=       unmasker('My name is <mask>'      ,targets=[' Patrick', ' Clara', ' Teven']      ,top_k=3						)
    self.assertEqual(
        nested_simplify(_lowercase      ,decimals=6						)      ,[
            {'sequence': 'My name is Patrick', 'score': 2.1e-05, 'token': 3499, 'token_str': ' Patrick'},
            {'sequence': 'My name is Te', 'score': 2e-05, 'token': 2941, 'token_str': ' Te'},
            {'sequence': 'My name is Clara', 'score': 2e-05, 'token': 13606, 'token_str': ' Clara'},
        ]      ,)

    __UpperCAmelCase								=       unmasker('My name is <mask> <mask>'      ,top_k=2						)

    self.assertEqual(
        nested_simplify(_lowercase      ,decimals=6						)      ,[
            [
                {
                    'score': 2.2e-05,
                    'token': 35676,
                    'token_str': ' Maul',
                    'sequence': '<s>My name is Maul<mask></s>',
                },
                {'score': 2.2e-05, 'token': 16416, 'token_str': 'ELS', 'sequence': '<s>My name isELS<mask></s>'},
            ],
            [
                {
                    'score': 2.2e-05,
                    'token': 35676,
                    'token_str': ' Maul',
                    'sequence': '<s>My name is<mask> Maul</s>',
                },
                {'score': 2.2e-05, 'token': 16416, 'token_str': 'ELS', 'sequence': '<s>My name is<mask>ELS</s>'},
            ],
        ]      ,)





   @require_torch_gpu
   def       snake_case						(				self:     List[Any]						):
    __UpperCAmelCase								=       pipeline('fill-mask'      ,model='hf-internal-testing/tiny-random-distilbert'      ,device=0      ,framework='pt'						)

    # convert model to fp16
    pipe.model.half()

    __UpperCAmelCase								=       pipe('Paris is the [MASK] of France.'						)
    # We actually don't care about the result, we just want to make sure
    # it works, meaning the float16 tensor got casted back to float32
    # for postprocessing.
    self.assertIsInstance(_lowercase      ,_lowercase						)





   @slow
   @require_torch
   def       snake_case						(				self:     str						):
    __UpperCAmelCase								=       pipeline(task='fill-mask'      ,model='distilroberta-base'      ,top_k=2      ,framework='pt'						)
    self.run_large_test(_lowercase						)





   @slow
   @require_tf
   def       snake_case						(				self:     List[str]						):
    __UpperCAmelCase								=       pipeline(task='fill-mask'      ,model='distilroberta-base'      ,top_k=2      ,framework='tf'						)
    self.run_large_test(_lowercase						)





   def       snake_case						(				self:     List[str]      ,a:     str						):
    __UpperCAmelCase								=       unmasker('My name is <mask>'						)
    self.assertEqual(
        nested_simplify(_lowercase						)      ,[
            {'sequence': 'My name is John', 'score': 0.008, 'token': 610, 'token_str': ' John'},
            {'sequence': 'My name is Chris', 'score': 0.007, 'token': 1573, 'token_str': ' Chris'},
        ]      ,)
    __UpperCAmelCase								=       unmasker('The largest city in France is <mask>'						)
    self.assertEqual(
        nested_simplify(_lowercase						)      ,[
            {
                'sequence': 'The largest city in France is Paris',
                'score': 0.251,
                'token': 2201,
                'token_str': ' Paris',
            },
            {
                'sequence': 'The largest city in France is Lyon',
                'score': 0.214,
                'token': 12790,
                'token_str': ' Lyon',
            },
        ]      ,)

    __UpperCAmelCase								=       unmasker('My name is <mask>'      ,targets=[' Patrick', ' Clara', ' Teven']      ,top_k=3						)
    self.assertEqual(
        nested_simplify(_lowercase						)      ,[
            {'sequence': 'My name is Patrick', 'score': 0.005, 'token': 3499, 'token_str': ' Patrick'},
            {'sequence': 'My name is Clara', 'score': 0.000, 'token': 13606, 'token_str': ' Clara'},
            {'sequence': 'My name is Te', 'score': 0.000, 'token': 2941, 'token_str': ' Te'},
        ]      ,)





   @require_torch
   def       snake_case						(				self:     Tuple						):
    __UpperCAmelCase								=       pipeline(task='fill-mask'      ,model='sshleifer/tiny-distilroberta-base'      ,framework='pt'						)
    __UpperCAmelCase								=       None
    __UpperCAmelCase								=       None
    self.run_pipeline_test(_lowercase      ,[]						)





   @require_tf
   def       snake_case						(				self:     Tuple						):
    __UpperCAmelCase								=       pipeline(task='fill-mask'      ,model='sshleifer/tiny-distilroberta-base'      ,framework='tf'						)
    __UpperCAmelCase								=       None
    __UpperCAmelCase								=       None
    self.run_pipeline_test(_lowercase      ,[]						)





   def       snake_case						(				self:     List[Any]      ,a:     List[Any]      ,a:     int      ,a:     Optional[int]						):
    if tokenizer is None or tokenizer.mask_token_id is None:
     self.skipTest('The provided tokenizer has no mask token, (probably reformer or wav2vec2)'						)

    __UpperCAmelCase								=       FillMaskPipeline(model=_lowercase      ,tokenizer=_lowercase						)
    __UpperCAmelCase								=       [
        f"""This is another {tokenizer.mask_token} test""",
    ]
    return fill_masker, examples





   def       snake_case						(				self:     Union[str, Any]      ,a:     List[str]      ,a:     Union[str, Any]						):
    __UpperCAmelCase								=       fill_masker.tokenizer
    __UpperCAmelCase								=       fill_masker.model

    __UpperCAmelCase								=       fill_masker(
        f"""This is a {tokenizer.mask_token}"""      ,)
    self.assertEqual(
        _lowercase      ,[
            {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
            {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
            {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
            {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
            {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
        ]      ,)

    __UpperCAmelCase								=       fill_masker([f"""This is a {tokenizer.mask_token}"""]						)
    self.assertEqual(
        _lowercase      ,[
            {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
            {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
            {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
            {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
            {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
        ]      ,)

    __UpperCAmelCase								=       fill_masker([f"""This is a {tokenizer.mask_token}""", f"""Another {tokenizer.mask_token} great test."""]						)
    self.assertEqual(
        _lowercase      ,[
            [
                {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
                {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
                {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
                {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
                {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
            ],
            [
                {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
                {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
                {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
                {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
                {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
            ],
        ]      ,)

    with self.assertRaises(_lowercase						):
     fill_masker([None]						)
    # No mask_token is not supported
    with self.assertRaises(_lowercase						):
     fill_masker('This is'						)

    self.run_test_top_k(_lowercase      ,_lowercase						)
    self.run_test_targets(_lowercase      ,_lowercase						)
    self.run_test_top_k_targets(_lowercase      ,_lowercase						)
    self.fill_mask_with_duplicate_targets_and_top_k(_lowercase      ,_lowercase						)
    self.fill_mask_with_multiple_masks(_lowercase      ,_lowercase						)





   def       snake_case						(				self:     Tuple      ,a:     List[Any]      ,a:     Optional[Any]						):
    __UpperCAmelCase								=       tokenizer.get_vocab()
    __UpperCAmelCase								=       sorted(vocab.keys()						)[:2]
    # Pipeline argument
    __UpperCAmelCase								=       FillMaskPipeline(model=_lowercase      ,tokenizer=_lowercase      ,targets=_lowercase						)
    __UpperCAmelCase								=       fill_masker(f"""This is a {tokenizer.mask_token}"""						)
    self.assertEqual(
        _lowercase      ,[
            {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
            {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
        ]      ,)
    __UpperCAmelCase								=       {vocab[el] for el in targets}
    self.assertEqual({el['token'] for el in outputs}      ,_lowercase						)
    __UpperCAmelCase								=       [tokenizer.decode([x]						) for x in target_ids]
    self.assertEqual({el['token_str'] for el in outputs}      ,set(_lowercase						)						)

    # Call argument
    __UpperCAmelCase								=       FillMaskPipeline(model=_lowercase      ,tokenizer=_lowercase						)
    __UpperCAmelCase								=       fill_masker(f"""This is a {tokenizer.mask_token}"""      ,targets=_lowercase						)
    self.assertEqual(
        _lowercase      ,[
            {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
            {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
        ]      ,)
    __UpperCAmelCase								=       {vocab[el] for el in targets}
    self.assertEqual({el['token'] for el in outputs}      ,_lowercase						)
    __UpperCAmelCase								=       [tokenizer.decode([x]						) for x in target_ids]
    self.assertEqual({el['token_str'] for el in outputs}      ,set(_lowercase						)						)

    # Score equivalence
    __UpperCAmelCase								=       fill_masker(f"""This is a {tokenizer.mask_token}"""      ,targets=_lowercase						)
    __UpperCAmelCase								=       [top_mask["token_str"] for top_mask in outputs]
    __UpperCAmelCase								=       [top_mask["score"] for top_mask in outputs]

    # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
    if set(_lowercase						) == set(_lowercase						):
     __UpperCAmelCase								=       fill_masker(f"""This is a {tokenizer.mask_token}"""      ,targets=_lowercase						)
     __UpperCAmelCase								=       [top_mask["score"] for top_mask in unmasked_targets]
     self.assertEqual(nested_simplify(_lowercase						)      ,nested_simplify(_lowercase						)						)

    # Raises with invalid
    with self.assertRaises(_lowercase						):
     __UpperCAmelCase								=       fill_masker(f"""This is a {tokenizer.mask_token}"""      ,targets=[]						)
    # For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
    if "" not in tokenizer.get_vocab():
     with self.assertRaises(_lowercase						):
      __UpperCAmelCase								=       fill_masker(f"""This is a {tokenizer.mask_token}"""      ,targets=['']						)
     with self.assertRaises(_lowercase						):
      __UpperCAmelCase								=       fill_masker(f"""This is a {tokenizer.mask_token}"""      ,targets=''						)





   def       snake_case						(				self:     Any      ,a:     Optional[Any]      ,a:     List[Any]						):
    __UpperCAmelCase								=       FillMaskPipeline(model=_lowercase      ,tokenizer=_lowercase      ,top_k=2						)
    __UpperCAmelCase								=       fill_masker(f"""This is a {tokenizer.mask_token}"""						)
    self.assertEqual(
        _lowercase      ,[
            {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
            {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
        ]      ,)

    __UpperCAmelCase								=       FillMaskPipeline(model=_lowercase      ,tokenizer=_lowercase						)
    __UpperCAmelCase								=       fill_masker(f"""This is a {tokenizer.mask_token}"""      ,top_k=2						)
    self.assertEqual(
        _lowercase      ,[
            {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
            {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
        ]      ,)
    self.assertEqual(nested_simplify(_lowercase						)      ,nested_simplify(_lowercase						)						)





   def       snake_case						(				self:     Optional[int]      ,a:     int      ,a:     List[Any]						):
    __UpperCAmelCase								=       tokenizer.get_vocab()
    __UpperCAmelCase								=       FillMaskPipeline(model=_lowercase      ,tokenizer=_lowercase						)

    # top_k=2, ntargets=3
    __UpperCAmelCase								=       sorted(vocab.keys()						)[:3]
    __UpperCAmelCase								=       fill_masker(f"""This is a {tokenizer.mask_token}"""      ,top_k=2      ,targets=_lowercase						)

    # If we use the most probably targets, and filter differently, we should still
    # have the same results
    __UpperCAmelCase								=       [el["token_str"] for el in sorted(_lowercase      ,key=lambda a						: x["score"]      ,reverse=_lowercase						)]
    # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
    if set(_lowercase						).issubset(_lowercase						):
     __UpperCAmelCase								=       fill_masker(f"""This is a {tokenizer.mask_token}"""      ,top_k=3      ,targets=_lowercase						)
     # They should yield exactly the same result
     self.assertEqual(nested_simplify(_lowercase						)      ,nested_simplify(_lowercase						)						)





   def       snake_case						(				self:     Tuple      ,a:     Dict      ,a:     Union[str, Any]						):
    __UpperCAmelCase								=       FillMaskPipeline(model=_lowercase      ,tokenizer=_lowercase						)
    __UpperCAmelCase								=       tokenizer.get_vocab()
    # String duplicates + id duplicates
    __UpperCAmelCase								=       sorted(vocab.keys()						)[:3]
    __UpperCAmelCase								=       [targets[0], targets[1], targets[0], targets[2], targets[1]]
    __UpperCAmelCase								=       fill_masker(f"""My name is {tokenizer.mask_token}"""      ,targets=_lowercase      ,top_k=10						)

    # The target list contains duplicates, so we can't output more
    # than them
    self.assertEqual(len(_lowercase						)      ,3						)





   def       snake_case						(				self:     str      ,a:     str      ,a:     List[Any]						):
    __UpperCAmelCase								=       FillMaskPipeline(model=_lowercase      ,tokenizer=_lowercase						)

    __UpperCAmelCase								=       fill_masker(
        f"""This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}"""      ,top_k=2						)
    self.assertEqual(
        _lowercase      ,[
            [
                {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
                {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
            ],
            [
                {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
                {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
            ],
            [
                {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
                {'sequence': ANY(_lowercase						), 'score': ANY(_lowercase						), 'token': ANY(_lowercase						), 'token_str': ANY(_lowercase						)},
            ],
        ]      ,)








"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available


A		=						{}

try:
			if not is_sentencepiece_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			A		=						['GPTSw3Tokenizer']


if TYPE_CHECKING:
			try:
						if not is_sentencepiece_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_gpt_swa import GPTSwaTokenizer

else:
			import sys

			A		=						_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)



