






import argparse
import pathlib

import fairseq
import torch
from fairseq.models.roberta import RobertaModel as FairseqRobertaModel
from fairseq.modules import TransformerSentenceEncoderLayer
from packaging import version

from transformers import XLMRobertaConfig, XLMRobertaXLForMaskedLM, XLMRobertaXLForSequenceClassification
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.models.roberta.modeling_roberta import RobertaAttention
from transformers.utils import logging


if version.parse(fairseq.__version__) < version.parse("""1.0.0a"""):
 raise Exception("""requires fairseq >= 1.0.0a""")

logging.set_verbosity_info()
snake_case__   :						Any													=   logging.get_logger(__name__)

snake_case__   :						Dict													=   """Hello world! cécé herlolip"""






def      _snake_case     (__lowercase     ,   __lowercase     ,   __lowercase):
   UpperCamelCase_						=  FairseqRobertaModel.from_pretrained(__lowercase)
   roberta.eval()  # disable dropout
   UpperCamelCase_						=  roberta.model.encoder.sentence_encoder
   UpperCamelCase_						=  XLMRobertaConfig(
       vocab_size=roberta_sent_encoder.embed_tokens.num_embeddings     ,   hidden_size=roberta.cfg.model.encoder_embed_dim     ,   num_hidden_layers=roberta.cfg.model.encoder_layers     ,   num_attention_heads=roberta.cfg.model.encoder_attention_heads     ,   intermediate_size=roberta.cfg.model.encoder_ffn_embed_dim     ,   max_position_embeddings=514     ,   type_vocab_size=1     ,   layer_norm_eps=1e-5     ,   )
   if classification_head:
      UpperCamelCase_						=  roberta.model.classification_heads['mnli'].out_proj.weight.shape[0]

   print('Our RoBERTa config:'     ,   __lowercase)

   UpperCamelCase_						=  XLMRobertaXLForSequenceClassification(__lowercase) if classification_head else XLMRobertaXLForMaskedLM(__lowercase)
   model.eval()

   # Now let's copy all the weights.
   # Embeddings
   UpperCamelCase_						=  roberta_sent_encoder.embed_tokens.weight
   UpperCamelCase_						=  roberta_sent_encoder.embed_positions.weight
   UpperCamelCase_						=  torch.zeros_like(
       model.roberta.embeddings.token_type_embeddings.weight)  # just zero them out b/c RoBERTa doesn't use them.

   UpperCamelCase_						=  roberta_sent_encoder.layer_norm.weight
   UpperCamelCase_						=  roberta_sent_encoder.layer_norm.bias

   for i in range(config.num_hidden_layers):
      # Encoder: start of layer
      UpperCamelCase_						=  model.roberta.encoder.layer[i]
      UpperCamelCase_						=  roberta_sent_encoder.layers[i]

      UpperCamelCase_						=  layer.attention
      UpperCamelCase_						=  roberta_layer.self_attn_layer_norm.weight
      UpperCamelCase_						=  roberta_layer.self_attn_layer_norm.bias

      # self attention
      UpperCamelCase_						=  layer.attention.self
      assert (
          roberta_layer.self_attn.k_proj.weight.data.shape
          == roberta_layer.self_attn.q_proj.weight.data.shape
          == roberta_layer.self_attn.v_proj.weight.data.shape
          == torch.Size((config.hidden_size, config.hidden_size))
      )

      UpperCamelCase_						=  roberta_layer.self_attn.q_proj.weight
      UpperCamelCase_						=  roberta_layer.self_attn.q_proj.bias
      UpperCamelCase_						=  roberta_layer.self_attn.k_proj.weight
      UpperCamelCase_						=  roberta_layer.self_attn.k_proj.bias
      UpperCamelCase_						=  roberta_layer.self_attn.v_proj.weight
      UpperCamelCase_						=  roberta_layer.self_attn.v_proj.bias

      # self-attention output
      UpperCamelCase_						=  layer.attention.output
      assert self_output.dense.weight.shape == roberta_layer.self_attn.out_proj.weight.shape
      UpperCamelCase_						=  roberta_layer.self_attn.out_proj.weight
      UpperCamelCase_						=  roberta_layer.self_attn.out_proj.bias

      # this one is final layer norm
      UpperCamelCase_						=  roberta_layer.final_layer_norm.weight
      UpperCamelCase_						=  roberta_layer.final_layer_norm.bias

      # intermediate
      UpperCamelCase_						=  layer.intermediate
      assert intermediate.dense.weight.shape == roberta_layer.fca.weight.shape
      UpperCamelCase_						=  roberta_layer.fca.weight
      UpperCamelCase_						=  roberta_layer.fca.bias

      # output
      UpperCamelCase_						=  layer.output
      assert bert_output.dense.weight.shape == roberta_layer.fca.weight.shape
      UpperCamelCase_						=  roberta_layer.fca.weight
      UpperCamelCase_						=  roberta_layer.fca.bias
      # end of layer

   if classification_head:
      UpperCamelCase_						=  roberta.model.classification_heads['mnli'].dense.weight
      UpperCamelCase_						=  roberta.model.classification_heads['mnli'].dense.bias
      UpperCamelCase_						=  roberta.model.classification_heads['mnli'].out_proj.weight
      UpperCamelCase_						=  roberta.model.classification_heads['mnli'].out_proj.bias
   else:
      # LM Head
      UpperCamelCase_						=  roberta.model.encoder.lm_head.dense.weight
      UpperCamelCase_						=  roberta.model.encoder.lm_head.dense.bias
      UpperCamelCase_						=  roberta.model.encoder.lm_head.layer_norm.weight
      UpperCamelCase_						=  roberta.model.encoder.lm_head.layer_norm.bias
      UpperCamelCase_						=  roberta.model.encoder.lm_head.weight
      UpperCamelCase_						=  roberta.model.encoder.lm_head.bias

   # Let's check that we get the same results.
   UpperCamelCase_						=  roberta.encode(__lowercase).unsqueeze(0)  # batch of size 1

   UpperCamelCase_						=  model(__lowercase)[0]
   if classification_head:
      UpperCamelCase_						=  roberta.model.classification_heads['mnli'](roberta.extract_features(__lowercase))
   else:
      UpperCamelCase_						=  roberta.model(__lowercase)[0]
   print(our_output.shape     ,   their_output.shape)
   UpperCamelCase_						=  torch.max(torch.abs(our_output - their_output)).item()
   print(f"""max_absolute_diff = {max_absolute_diff}""")  # ~ 1e-7
   UpperCamelCase_						=  torch.allclose(__lowercase     ,   __lowercase     ,   atol=1e-3)
   print('Do both models output the same tensors?'     ,   '🔥' if success else '💩')
   if not success:
      raise Exception('Something went wRoNg')

   pathlib.Path(__lowercase).mkdir(parents=__lowercase     ,   exist_ok=__lowercase)
   print(f"""Saving model to {pytorch_dump_folder_path}""")
   model.save_pretrained(__lowercase)


if __name__ == "__main__":
 snake_case__   :						str													=   argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     """--roberta_checkpoint_path""", default=None, type=str, required=True, help="""Path the official PyTorch dump."""
 )
 parser.add_argument(
     """--pytorch_dump_folder_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
 )
 parser.add_argument(
     """--classification_head""", action="""store_true""", help="""Whether to convert a final classification head."""
 )
 snake_case__   :						int													=   parser.parse_args()
 convert_xlm_roberta_xl_checkpoint_to_pytorch(
     args.roberta_checkpoint_path, args.pytorch_dump_folder_path, args.classification_head
 )













import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY


@is_pipeline_test
class     _a							(	unittest.TestCase     ):






  """simple docstring"""





  A_						       = MODEL_FOR_MASKED_LM_MAPPING
  A_						       = TF_MODEL_FOR_MASKED_LM_MAPPING




  def  _UpperCAmelCase  (  self			)      ->			List[str]:
     super().tearDown()
     # clean-up as much as possible GPU memory occupied by PyTorch
     gc.collect()
     if is_torch_available():
        import torch

        torch.cuda.empty_cache()




  @require_tf
  def  _UpperCAmelCase  (  self			)      ->			str:
     UpperCamelCase_						=  pipeline(task='fill-mask'		,		model='sshleifer/tiny-distilroberta-base'		,		top_k=2		,		framework='tf'			)
     UpperCamelCase_						=  unmasker('My name is <mask>'			)
     self.assertEqual(
         nested_simplify(_UpperCAmelCase		,		decimals=6			)		,		[
             {'sequence': 'My name is grouped', 'score': 2.1e-05, 'token': 38015, 'token_str': ' grouped'},
             {'sequence': 'My name is accuser', 'score': 2.1e-05, 'token': 25506, 'token_str': ' accuser'},
         ]		,		)

     UpperCamelCase_						=  unmasker('The largest city in France is <mask>'			)
     self.assertEqual(
         nested_simplify(_UpperCAmelCase		,		decimals=6			)		,		[
             {
                 'sequence': 'The largest city in France is grouped',
                 'score': 2.1e-05,
                 'token': 38015,
                 'token_str': ' grouped',
             },
             {
                 'sequence': 'The largest city in France is accuser',
                 'score': 2.1e-05,
                 'token': 25506,
                 'token_str': ' accuser',
             },
         ]		,		)

     UpperCamelCase_						=  unmasker('My name is <mask>'		,		targets=[' Patrick', ' Clara', ' Teven']		,		top_k=3			)
     self.assertEqual(
         nested_simplify(_UpperCAmelCase		,		decimals=6			)		,		[
             {'sequence': 'My name is Clara', 'score': 2e-05, 'token': 13606, 'token_str': ' Clara'},
             {'sequence': 'My name is Patrick', 'score': 2e-05, 'token': 3499, 'token_str': ' Patrick'},
             {'sequence': 'My name is Te', 'score': 1.9e-05, 'token': 2941, 'token_str': ' Te'},
         ]		,		)




  @require_torch
  def  _UpperCAmelCase  (  self			)      ->			Union[str, Any]:
     UpperCamelCase_						=  pipeline(task='fill-mask'		,		model='sshleifer/tiny-distilroberta-base'		,		top_k=2		,		framework='pt'			)

     UpperCamelCase_						=  unmasker('My name is <mask>'			)
     self.assertEqual(
         nested_simplify(_UpperCAmelCase		,		decimals=6			)		,		[
             {'sequence': 'My name is Maul', 'score': 2.2e-05, 'token': 35676, 'token_str': ' Maul'},
             {'sequence': 'My name isELS', 'score': 2.2e-05, 'token': 16416, 'token_str': 'ELS'},
         ]		,		)

     UpperCamelCase_						=  unmasker('The largest city in France is <mask>'			)
     self.assertEqual(
         nested_simplify(_UpperCAmelCase		,		decimals=6			)		,		[
             {
                 'sequence': 'The largest city in France is Maul',
                 'score': 2.2e-05,
                 'token': 35676,
                 'token_str': ' Maul',
             },
             {'sequence': 'The largest city in France isELS', 'score': 2.2e-05, 'token': 16416, 'token_str': 'ELS'},
         ]		,		)

     UpperCamelCase_						=  unmasker('My name is <mask>'		,		targets=[' Patrick', ' Clara', ' Teven']		,		top_k=3			)
     self.assertEqual(
         nested_simplify(_UpperCAmelCase		,		decimals=6			)		,		[
             {'sequence': 'My name is Patrick', 'score': 2.1e-05, 'token': 3499, 'token_str': ' Patrick'},
             {'sequence': 'My name is Te', 'score': 2e-05, 'token': 2941, 'token_str': ' Te'},
             {'sequence': 'My name is Clara', 'score': 2e-05, 'token': 13606, 'token_str': ' Clara'},
         ]		,		)

     UpperCamelCase_						=  unmasker('My name is <mask> <mask>'		,		top_k=2			)

     self.assertEqual(
         nested_simplify(_UpperCAmelCase		,		decimals=6			)		,		[
             [
                 {
                     'score': 2.2e-05,
                     'token': 35676,
                     'token_str': ' Maul',
                     'sequence': '<s>My name is Maul<mask></s>',
                 },
                 {'score': 2.2e-05, 'token': 16416, 'token_str': 'ELS', 'sequence': '<s>My name isELS<mask></s>'},
             ],
             [
                 {
                     'score': 2.2e-05,
                     'token': 35676,
                     'token_str': ' Maul',
                     'sequence': '<s>My name is<mask> Maul</s>',
                 },
                 {'score': 2.2e-05, 'token': 16416, 'token_str': 'ELS', 'sequence': '<s>My name is<mask>ELS</s>'},
             ],
         ]		,		)




  @require_torch_gpu
  def  _UpperCAmelCase  (  self			)      ->			Optional[Any]:
     UpperCamelCase_						=  pipeline('fill-mask'		,		model='hf-internal-testing/tiny-random-distilbert'		,		device=0		,		framework='pt'			)

     # convert model to fp16
     pipe.model.half()

     UpperCamelCase_						=  pipe('Paris is the [MASK] of France.'			)
     # We actually don't care about the result, we just want to make sure
     # it works, meaning the float16 tensor got casted back to float32
     # for postprocessing.
     self.assertIsInstance(_UpperCAmelCase		,		_UpperCAmelCase			)




  @slow
  @require_torch
  def  _UpperCAmelCase  (  self			)      ->			List[Any]:
     UpperCamelCase_						=  pipeline(task='fill-mask'		,		model='distilroberta-base'		,		top_k=2		,		framework='pt'			)
     self.run_large_test(_UpperCAmelCase			)




  @slow
  @require_tf
  def  _UpperCAmelCase  (  self			)      ->			Union[str, Any]:
     UpperCamelCase_						=  pipeline(task='fill-mask'		,		model='distilroberta-base'		,		top_k=2		,		framework='tf'			)
     self.run_large_test(_UpperCAmelCase			)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase			)      ->			Tuple:
     UpperCamelCase_						=  unmasker('My name is <mask>'			)
     self.assertEqual(
         nested_simplify(_UpperCAmelCase			)		,		[
             {'sequence': 'My name is John', 'score': 0.0_0_8, 'token': 610, 'token_str': ' John'},
             {'sequence': 'My name is Chris', 'score': 0.0_0_7, 'token': 1573, 'token_str': ' Chris'},
         ]		,		)
     UpperCamelCase_						=  unmasker('The largest city in France is <mask>'			)
     self.assertEqual(
         nested_simplify(_UpperCAmelCase			)		,		[
             {
                 'sequence': 'The largest city in France is Paris',
                 'score': 0.2_5_1,
                 'token': 2201,
                 'token_str': ' Paris',
             },
             {
                 'sequence': 'The largest city in France is Lyon',
                 'score': 0.2_1_4,
                 'token': 12790,
                 'token_str': ' Lyon',
             },
         ]		,		)

     UpperCamelCase_						=  unmasker('My name is <mask>'		,		targets=[' Patrick', ' Clara', ' Teven']		,		top_k=3			)
     self.assertEqual(
         nested_simplify(_UpperCAmelCase			)		,		[
             {'sequence': 'My name is Patrick', 'score': 0.0_0_5, 'token': 3499, 'token_str': ' Patrick'},
             {'sequence': 'My name is Clara', 'score': 0.0_0_0, 'token': 13606, 'token_str': ' Clara'},
             {'sequence': 'My name is Te', 'score': 0.0_0_0, 'token': 2941, 'token_str': ' Te'},
         ]		,		)




  @require_torch
  def  _UpperCAmelCase  (  self			)      ->			List[Any]:
     UpperCamelCase_						=  pipeline(task='fill-mask'		,		model='sshleifer/tiny-distilroberta-base'		,		framework='pt'			)
     UpperCamelCase_						=  None
     UpperCamelCase_						=  None
     self.run_pipeline_test(_UpperCAmelCase		,		[]			)




  @require_tf
  def  _UpperCAmelCase  (  self			)      ->			Optional[int]:
     UpperCamelCase_						=  pipeline(task='fill-mask'		,		model='sshleifer/tiny-distilroberta-base'		,		framework='tf'			)
     UpperCamelCase_						=  None
     UpperCamelCase_						=  None
     self.run_pipeline_test(_UpperCAmelCase		,		[]			)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Optional[Any]:
     if tokenizer is None or tokenizer.mask_token_id is None:
        self.skipTest('The provided tokenizer has no mask token, (probably reformer or wav2vec2)'			)

     UpperCamelCase_						=  FillMaskPipeline(model=_UpperCAmelCase		,		tokenizer=_UpperCAmelCase			)
     UpperCamelCase_						=  [
         f"""This is another {tokenizer.mask_token} test""",
     ]
     return fill_masker, examples




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Union[str, Any]:
     UpperCamelCase_						=  fill_masker.tokenizer
     UpperCamelCase_						=  fill_masker.model

     UpperCamelCase_						=  fill_masker(
         f"""This is a {tokenizer.mask_token}"""		,		)
     self.assertEqual(
         _UpperCAmelCase		,		[
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
         ]		,		)

     UpperCamelCase_						=  fill_masker([f"""This is a {tokenizer.mask_token}"""]			)
     self.assertEqual(
         _UpperCAmelCase		,		[
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
         ]		,		)

     UpperCamelCase_						=  fill_masker([f"""This is a {tokenizer.mask_token}""", f"""Another {tokenizer.mask_token} great test."""]			)
     self.assertEqual(
         _UpperCAmelCase		,		[
             [
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             ],
             [
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             ],
         ]		,		)

     with self.assertRaises(_UpperCAmelCase			):
        fill_masker([None]			)
     # No mask_token is not supported
     with self.assertRaises(_UpperCAmelCase			):
        fill_masker('This is'			)

     self.run_test_top_k(_UpperCAmelCase		,		_UpperCAmelCase			)
     self.run_test_targets(_UpperCAmelCase		,		_UpperCAmelCase			)
     self.run_test_top_k_targets(_UpperCAmelCase		,		_UpperCAmelCase			)
     self.fill_mask_with_duplicate_targets_and_top_k(_UpperCAmelCase		,		_UpperCAmelCase			)
     self.fill_mask_with_multiple_masks(_UpperCAmelCase		,		_UpperCAmelCase			)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Optional[Any]:
     UpperCamelCase_						=  tokenizer.get_vocab()
     UpperCamelCase_						=  sorted(vocab.keys()			)[:2]
     # Pipeline argument
     UpperCamelCase_						=  FillMaskPipeline(model=_UpperCAmelCase		,		tokenizer=_UpperCAmelCase		,		targets=_UpperCAmelCase			)
     UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""			)
     self.assertEqual(
         _UpperCAmelCase		,		[
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
         ]		,		)
     UpperCamelCase_						=  {vocab[el] for el in targets}
     self.assertEqual({el['token'] for el in outputs}		,		_UpperCAmelCase			)
     UpperCamelCase_						=  [tokenizer.decode([x]			) for x in target_ids]
     self.assertEqual({el['token_str'] for el in outputs}		,		set(_UpperCAmelCase			)			)

     # Call argument
     UpperCamelCase_						=  FillMaskPipeline(model=_UpperCAmelCase		,		tokenizer=_UpperCAmelCase			)
     UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""		,		targets=_UpperCAmelCase			)
     self.assertEqual(
         _UpperCAmelCase		,		[
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
         ]		,		)
     UpperCamelCase_						=  {vocab[el] for el in targets}
     self.assertEqual({el['token'] for el in outputs}		,		_UpperCAmelCase			)
     UpperCamelCase_						=  [tokenizer.decode([x]			) for x in target_ids]
     self.assertEqual({el['token_str'] for el in outputs}		,		set(_UpperCAmelCase			)			)

     # Score equivalence
     UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""		,		targets=_UpperCAmelCase			)
     UpperCamelCase_						=  [top_mask['token_str'] for top_mask in outputs]
     UpperCamelCase_						=  [top_mask['score'] for top_mask in outputs]

     # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
     if set(_UpperCAmelCase			) == set(_UpperCAmelCase			):
        UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""		,		targets=_UpperCAmelCase			)
        UpperCamelCase_						=  [top_mask['score'] for top_mask in unmasked_targets]
        self.assertEqual(nested_simplify(_UpperCAmelCase			)		,		nested_simplify(_UpperCAmelCase			)			)

     # Raises with invalid
     with self.assertRaises(_UpperCAmelCase			):
        UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""		,		targets=[]			)
     # For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
     if "" not in tokenizer.get_vocab():
        with self.assertRaises(_UpperCAmelCase			):
           UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""		,		targets=['']			)
        with self.assertRaises(_UpperCAmelCase			):
           UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""		,		targets=''			)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Tuple:
     UpperCamelCase_						=  FillMaskPipeline(model=_UpperCAmelCase		,		tokenizer=_UpperCAmelCase		,		top_k=2			)
     UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""			)
     self.assertEqual(
         _UpperCAmelCase		,		[
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
         ]		,		)

     UpperCamelCase_						=  FillMaskPipeline(model=_UpperCAmelCase		,		tokenizer=_UpperCAmelCase			)
     UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""		,		top_k=2			)
     self.assertEqual(
         _UpperCAmelCase		,		[
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
         ]		,		)
     self.assertEqual(nested_simplify(_UpperCAmelCase			)		,		nested_simplify(_UpperCAmelCase			)			)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			List[Any]:
     UpperCamelCase_						=  tokenizer.get_vocab()
     UpperCamelCase_						=  FillMaskPipeline(model=_UpperCAmelCase		,		tokenizer=_UpperCAmelCase			)

     # top_k=2, ntargets=3
     UpperCamelCase_						=  sorted(vocab.keys()			)[:3]
     UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""		,		top_k=2		,		targets=_UpperCAmelCase			)

     # If we use the most probably targets, and filter differently, we should still
     # have the same results
     UpperCamelCase_						=  [el['token_str'] for el in sorted(_UpperCAmelCase		,		key=lambda _UpperCAmelCase			: x["score"]		,		reverse=_UpperCAmelCase			)]
     # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
     if set(_UpperCAmelCase			).issubset(_UpperCAmelCase			):
        UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""		,		top_k=3		,		targets=_UpperCAmelCase			)
        # They should yield exactly the same result
        self.assertEqual(nested_simplify(_UpperCAmelCase			)		,		nested_simplify(_UpperCAmelCase			)			)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Optional[int]:
     UpperCamelCase_						=  FillMaskPipeline(model=_UpperCAmelCase		,		tokenizer=_UpperCAmelCase			)
     UpperCamelCase_						=  tokenizer.get_vocab()
     # String duplicates + id duplicates
     UpperCamelCase_						=  sorted(vocab.keys()			)[:3]
     UpperCamelCase_						=  [targets[0], targets[1], targets[0], targets[2], targets[1]]
     UpperCamelCase_						=  fill_masker(f"""My name is {tokenizer.mask_token}"""		,		targets=_UpperCAmelCase		,		top_k=10			)

     # The target list contains duplicates, so we can't output more
     # than them
     self.assertEqual(len(_UpperCAmelCase			)		,		3			)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			List[str]:
     UpperCamelCase_						=  FillMaskPipeline(model=_UpperCAmelCase		,		tokenizer=_UpperCAmelCase			)

     UpperCamelCase_						=  fill_masker(
         f"""This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}"""		,		top_k=2			)
     self.assertEqual(
         _UpperCAmelCase		,		[
             [
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             ],
             [
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             ],
             [
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             ],
         ]		,		)





