






import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY


@is_pipeline_test
class     _a							(	unittest.TestCase     ):






  """simple docstring"""





  A_						       = MODEL_FOR_MASKED_LM_MAPPING
  A_						       = TF_MODEL_FOR_MASKED_LM_MAPPING




  def  _UpperCAmelCase  (  self			)      ->			List[str]:
     super().tearDown()
     # clean-up as much as possible GPU memory occupied by PyTorch
     gc.collect()
     if is_torch_available():
        import torch

        torch.cuda.empty_cache()




  @require_tf
  def  _UpperCAmelCase  (  self			)      ->			str:
     UpperCamelCase_						=  pipeline(task='fill-mask'		,		model='sshleifer/tiny-distilroberta-base'		,		top_k=2		,		framework='tf'			)
     UpperCamelCase_						=  unmasker('My name is <mask>'			)
     self.assertEqual(
         nested_simplify(_UpperCAmelCase		,		decimals=6			)		,		[
             {'sequence': 'My name is grouped', 'score': 2.1e-05, 'token': 38015, 'token_str': ' grouped'},
             {'sequence': 'My name is accuser', 'score': 2.1e-05, 'token': 25506, 'token_str': ' accuser'},
         ]		,		)

     UpperCamelCase_						=  unmasker('The largest city in France is <mask>'			)
     self.assertEqual(
         nested_simplify(_UpperCAmelCase		,		decimals=6			)		,		[
             {
                 'sequence': 'The largest city in France is grouped',
                 'score': 2.1e-05,
                 'token': 38015,
                 'token_str': ' grouped',
             },
             {
                 'sequence': 'The largest city in France is accuser',
                 'score': 2.1e-05,
                 'token': 25506,
                 'token_str': ' accuser',
             },
         ]		,		)

     UpperCamelCase_						=  unmasker('My name is <mask>'		,		targets=[' Patrick', ' Clara', ' Teven']		,		top_k=3			)
     self.assertEqual(
         nested_simplify(_UpperCAmelCase		,		decimals=6			)		,		[
             {'sequence': 'My name is Clara', 'score': 2e-05, 'token': 13606, 'token_str': ' Clara'},
             {'sequence': 'My name is Patrick', 'score': 2e-05, 'token': 3499, 'token_str': ' Patrick'},
             {'sequence': 'My name is Te', 'score': 1.9e-05, 'token': 2941, 'token_str': ' Te'},
         ]		,		)




  @require_torch
  def  _UpperCAmelCase  (  self			)      ->			Union[str, Any]:
     UpperCamelCase_						=  pipeline(task='fill-mask'		,		model='sshleifer/tiny-distilroberta-base'		,		top_k=2		,		framework='pt'			)

     UpperCamelCase_						=  unmasker('My name is <mask>'			)
     self.assertEqual(
         nested_simplify(_UpperCAmelCase		,		decimals=6			)		,		[
             {'sequence': 'My name is Maul', 'score': 2.2e-05, 'token': 35676, 'token_str': ' Maul'},
             {'sequence': 'My name isELS', 'score': 2.2e-05, 'token': 16416, 'token_str': 'ELS'},
         ]		,		)

     UpperCamelCase_						=  unmasker('The largest city in France is <mask>'			)
     self.assertEqual(
         nested_simplify(_UpperCAmelCase		,		decimals=6			)		,		[
             {
                 'sequence': 'The largest city in France is Maul',
                 'score': 2.2e-05,
                 'token': 35676,
                 'token_str': ' Maul',
             },
             {'sequence': 'The largest city in France isELS', 'score': 2.2e-05, 'token': 16416, 'token_str': 'ELS'},
         ]		,		)

     UpperCamelCase_						=  unmasker('My name is <mask>'		,		targets=[' Patrick', ' Clara', ' Teven']		,		top_k=3			)
     self.assertEqual(
         nested_simplify(_UpperCAmelCase		,		decimals=6			)		,		[
             {'sequence': 'My name is Patrick', 'score': 2.1e-05, 'token': 3499, 'token_str': ' Patrick'},
             {'sequence': 'My name is Te', 'score': 2e-05, 'token': 2941, 'token_str': ' Te'},
             {'sequence': 'My name is Clara', 'score': 2e-05, 'token': 13606, 'token_str': ' Clara'},
         ]		,		)

     UpperCamelCase_						=  unmasker('My name is <mask> <mask>'		,		top_k=2			)

     self.assertEqual(
         nested_simplify(_UpperCAmelCase		,		decimals=6			)		,		[
             [
                 {
                     'score': 2.2e-05,
                     'token': 35676,
                     'token_str': ' Maul',
                     'sequence': '<s>My name is Maul<mask></s>',
                 },
                 {'score': 2.2e-05, 'token': 16416, 'token_str': 'ELS', 'sequence': '<s>My name isELS<mask></s>'},
             ],
             [
                 {
                     'score': 2.2e-05,
                     'token': 35676,
                     'token_str': ' Maul',
                     'sequence': '<s>My name is<mask> Maul</s>',
                 },
                 {'score': 2.2e-05, 'token': 16416, 'token_str': 'ELS', 'sequence': '<s>My name is<mask>ELS</s>'},
             ],
         ]		,		)




  @require_torch_gpu
  def  _UpperCAmelCase  (  self			)      ->			Optional[Any]:
     UpperCamelCase_						=  pipeline('fill-mask'		,		model='hf-internal-testing/tiny-random-distilbert'		,		device=0		,		framework='pt'			)

     # convert model to fp16
     pipe.model.half()

     UpperCamelCase_						=  pipe('Paris is the [MASK] of France.'			)
     # We actually don't care about the result, we just want to make sure
     # it works, meaning the float16 tensor got casted back to float32
     # for postprocessing.
     self.assertIsInstance(_UpperCAmelCase		,		_UpperCAmelCase			)




  @slow
  @require_torch
  def  _UpperCAmelCase  (  self			)      ->			List[Any]:
     UpperCamelCase_						=  pipeline(task='fill-mask'		,		model='distilroberta-base'		,		top_k=2		,		framework='pt'			)
     self.run_large_test(_UpperCAmelCase			)




  @slow
  @require_tf
  def  _UpperCAmelCase  (  self			)      ->			Union[str, Any]:
     UpperCamelCase_						=  pipeline(task='fill-mask'		,		model='distilroberta-base'		,		top_k=2		,		framework='tf'			)
     self.run_large_test(_UpperCAmelCase			)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase			)      ->			Tuple:
     UpperCamelCase_						=  unmasker('My name is <mask>'			)
     self.assertEqual(
         nested_simplify(_UpperCAmelCase			)		,		[
             {'sequence': 'My name is John', 'score': 0.0_0_8, 'token': 610, 'token_str': ' John'},
             {'sequence': 'My name is Chris', 'score': 0.0_0_7, 'token': 1573, 'token_str': ' Chris'},
         ]		,		)
     UpperCamelCase_						=  unmasker('The largest city in France is <mask>'			)
     self.assertEqual(
         nested_simplify(_UpperCAmelCase			)		,		[
             {
                 'sequence': 'The largest city in France is Paris',
                 'score': 0.2_5_1,
                 'token': 2201,
                 'token_str': ' Paris',
             },
             {
                 'sequence': 'The largest city in France is Lyon',
                 'score': 0.2_1_4,
                 'token': 12790,
                 'token_str': ' Lyon',
             },
         ]		,		)

     UpperCamelCase_						=  unmasker('My name is <mask>'		,		targets=[' Patrick', ' Clara', ' Teven']		,		top_k=3			)
     self.assertEqual(
         nested_simplify(_UpperCAmelCase			)		,		[
             {'sequence': 'My name is Patrick', 'score': 0.0_0_5, 'token': 3499, 'token_str': ' Patrick'},
             {'sequence': 'My name is Clara', 'score': 0.0_0_0, 'token': 13606, 'token_str': ' Clara'},
             {'sequence': 'My name is Te', 'score': 0.0_0_0, 'token': 2941, 'token_str': ' Te'},
         ]		,		)




  @require_torch
  def  _UpperCAmelCase  (  self			)      ->			List[Any]:
     UpperCamelCase_						=  pipeline(task='fill-mask'		,		model='sshleifer/tiny-distilroberta-base'		,		framework='pt'			)
     UpperCamelCase_						=  None
     UpperCamelCase_						=  None
     self.run_pipeline_test(_UpperCAmelCase		,		[]			)




  @require_tf
  def  _UpperCAmelCase  (  self			)      ->			Optional[int]:
     UpperCamelCase_						=  pipeline(task='fill-mask'		,		model='sshleifer/tiny-distilroberta-base'		,		framework='tf'			)
     UpperCamelCase_						=  None
     UpperCamelCase_						=  None
     self.run_pipeline_test(_UpperCAmelCase		,		[]			)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Optional[Any]:
     if tokenizer is None or tokenizer.mask_token_id is None:
        self.skipTest('The provided tokenizer has no mask token, (probably reformer or wav2vec2)'			)

     UpperCamelCase_						=  FillMaskPipeline(model=_UpperCAmelCase		,		tokenizer=_UpperCAmelCase			)
     UpperCamelCase_						=  [
         f"""This is another {tokenizer.mask_token} test""",
     ]
     return fill_masker, examples




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Union[str, Any]:
     UpperCamelCase_						=  fill_masker.tokenizer
     UpperCamelCase_						=  fill_masker.model

     UpperCamelCase_						=  fill_masker(
         f"""This is a {tokenizer.mask_token}"""		,		)
     self.assertEqual(
         _UpperCAmelCase		,		[
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
         ]		,		)

     UpperCamelCase_						=  fill_masker([f"""This is a {tokenizer.mask_token}"""]			)
     self.assertEqual(
         _UpperCAmelCase		,		[
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
         ]		,		)

     UpperCamelCase_						=  fill_masker([f"""This is a {tokenizer.mask_token}""", f"""Another {tokenizer.mask_token} great test."""]			)
     self.assertEqual(
         _UpperCAmelCase		,		[
             [
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             ],
             [
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             ],
         ]		,		)

     with self.assertRaises(_UpperCAmelCase			):
        fill_masker([None]			)
     # No mask_token is not supported
     with self.assertRaises(_UpperCAmelCase			):
        fill_masker('This is'			)

     self.run_test_top_k(_UpperCAmelCase		,		_UpperCAmelCase			)
     self.run_test_targets(_UpperCAmelCase		,		_UpperCAmelCase			)
     self.run_test_top_k_targets(_UpperCAmelCase		,		_UpperCAmelCase			)
     self.fill_mask_with_duplicate_targets_and_top_k(_UpperCAmelCase		,		_UpperCAmelCase			)
     self.fill_mask_with_multiple_masks(_UpperCAmelCase		,		_UpperCAmelCase			)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Optional[Any]:
     UpperCamelCase_						=  tokenizer.get_vocab()
     UpperCamelCase_						=  sorted(vocab.keys()			)[:2]
     # Pipeline argument
     UpperCamelCase_						=  FillMaskPipeline(model=_UpperCAmelCase		,		tokenizer=_UpperCAmelCase		,		targets=_UpperCAmelCase			)
     UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""			)
     self.assertEqual(
         _UpperCAmelCase		,		[
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
         ]		,		)
     UpperCamelCase_						=  {vocab[el] for el in targets}
     self.assertEqual({el['token'] for el in outputs}		,		_UpperCAmelCase			)
     UpperCamelCase_						=  [tokenizer.decode([x]			) for x in target_ids]
     self.assertEqual({el['token_str'] for el in outputs}		,		set(_UpperCAmelCase			)			)

     # Call argument
     UpperCamelCase_						=  FillMaskPipeline(model=_UpperCAmelCase		,		tokenizer=_UpperCAmelCase			)
     UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""		,		targets=_UpperCAmelCase			)
     self.assertEqual(
         _UpperCAmelCase		,		[
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
         ]		,		)
     UpperCamelCase_						=  {vocab[el] for el in targets}
     self.assertEqual({el['token'] for el in outputs}		,		_UpperCAmelCase			)
     UpperCamelCase_						=  [tokenizer.decode([x]			) for x in target_ids]
     self.assertEqual({el['token_str'] for el in outputs}		,		set(_UpperCAmelCase			)			)

     # Score equivalence
     UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""		,		targets=_UpperCAmelCase			)
     UpperCamelCase_						=  [top_mask['token_str'] for top_mask in outputs]
     UpperCamelCase_						=  [top_mask['score'] for top_mask in outputs]

     # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
     if set(_UpperCAmelCase			) == set(_UpperCAmelCase			):
        UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""		,		targets=_UpperCAmelCase			)
        UpperCamelCase_						=  [top_mask['score'] for top_mask in unmasked_targets]
        self.assertEqual(nested_simplify(_UpperCAmelCase			)		,		nested_simplify(_UpperCAmelCase			)			)

     # Raises with invalid
     with self.assertRaises(_UpperCAmelCase			):
        UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""		,		targets=[]			)
     # For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
     if "" not in tokenizer.get_vocab():
        with self.assertRaises(_UpperCAmelCase			):
           UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""		,		targets=['']			)
        with self.assertRaises(_UpperCAmelCase			):
           UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""		,		targets=''			)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Tuple:
     UpperCamelCase_						=  FillMaskPipeline(model=_UpperCAmelCase		,		tokenizer=_UpperCAmelCase		,		top_k=2			)
     UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""			)
     self.assertEqual(
         _UpperCAmelCase		,		[
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
         ]		,		)

     UpperCamelCase_						=  FillMaskPipeline(model=_UpperCAmelCase		,		tokenizer=_UpperCAmelCase			)
     UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""		,		top_k=2			)
     self.assertEqual(
         _UpperCAmelCase		,		[
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
         ]		,		)
     self.assertEqual(nested_simplify(_UpperCAmelCase			)		,		nested_simplify(_UpperCAmelCase			)			)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			List[Any]:
     UpperCamelCase_						=  tokenizer.get_vocab()
     UpperCamelCase_						=  FillMaskPipeline(model=_UpperCAmelCase		,		tokenizer=_UpperCAmelCase			)

     # top_k=2, ntargets=3
     UpperCamelCase_						=  sorted(vocab.keys()			)[:3]
     UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""		,		top_k=2		,		targets=_UpperCAmelCase			)

     # If we use the most probably targets, and filter differently, we should still
     # have the same results
     UpperCamelCase_						=  [el['token_str'] for el in sorted(_UpperCAmelCase		,		key=lambda _UpperCAmelCase			: x["score"]		,		reverse=_UpperCAmelCase			)]
     # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
     if set(_UpperCAmelCase			).issubset(_UpperCAmelCase			):
        UpperCamelCase_						=  fill_masker(f"""This is a {tokenizer.mask_token}"""		,		top_k=3		,		targets=_UpperCAmelCase			)
        # They should yield exactly the same result
        self.assertEqual(nested_simplify(_UpperCAmelCase			)		,		nested_simplify(_UpperCAmelCase			)			)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			Optional[int]:
     UpperCamelCase_						=  FillMaskPipeline(model=_UpperCAmelCase		,		tokenizer=_UpperCAmelCase			)
     UpperCamelCase_						=  tokenizer.get_vocab()
     # String duplicates + id duplicates
     UpperCamelCase_						=  sorted(vocab.keys()			)[:3]
     UpperCamelCase_						=  [targets[0], targets[1], targets[0], targets[2], targets[1]]
     UpperCamelCase_						=  fill_masker(f"""My name is {tokenizer.mask_token}"""		,		targets=_UpperCAmelCase		,		top_k=10			)

     # The target list contains duplicates, so we can't output more
     # than them
     self.assertEqual(len(_UpperCAmelCase			)		,		3			)




  def  _UpperCAmelCase  (  self		,		_UpperCAmelCase		,		_UpperCAmelCase			)      ->			List[str]:
     UpperCamelCase_						=  FillMaskPipeline(model=_UpperCAmelCase		,		tokenizer=_UpperCAmelCase			)

     UpperCamelCase_						=  fill_masker(
         f"""This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}"""		,		top_k=2			)
     self.assertEqual(
         _UpperCAmelCase		,		[
             [
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             ],
             [
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             ],
             [
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
                 {'sequence': ANY(_UpperCAmelCase			), 'score': ANY(_UpperCAmelCase			), 'token': ANY(_UpperCAmelCase			), 'token_str': ANY(_UpperCAmelCase			)},
             ],
         ]		,		)













"""simple docstring"""





from typing import Dict, List, Optional, Union

import numpy as np

from .feature_extraction_utils import BatchFeature, FeatureExtractionMixin
from .utils import PaddingStrategy, TensorType, is_tf_tensor, is_torch_tensor, logging, to_numpy


__lowerCamelCase							=  logging.get_logger(__name__)



class       __A		(   SCREAMING_SNAKE_CASE_			):

					def __init__(			self  :      Union[str, Any]     ,		__snake_case  :      int     ,		__snake_case  :      int     ,		__snake_case  :      float     ,		**__snake_case  :      Optional[Any]					)			->							str:
											__magic_name__:	Optional[Any]							=       feature_size
											__magic_name__:	List[Any]							=       sampling_rate
											__magic_name__:	Tuple							=       padding_value

											__magic_name__:	int							=       kwargs.pop("""padding_side"""     ,		"""right"""					)
											__magic_name__:	Optional[Any]							=       kwargs.pop("""return_attention_mask"""     ,		__snake_case					)

											super().__init__(**__snake_case					)

					def   lowerCamelCase__   (			self  :      Union[str, Any]     ,		__snake_case  :      Union[
					        BatchFeature,
					        List[BatchFeature],
					        Dict[str, BatchFeature],
					        Dict[str, List[BatchFeature]],
					        List[Dict[str, BatchFeature]],
					    ]     ,		__snake_case  :      Union[bool, str, PaddingStrategy] = True     ,		__snake_case  :      Optional[int] = None     ,		__snake_case  :      bool = False     ,		__snake_case  :      Optional[int] = None     ,		__snake_case  :      Optional[bool] = None     ,		__snake_case  :      Optional[Union[str, TensorType]] = None     ,		)			->							BatchFeature:
											# If we have a list of dicts, let's convert it in a dict of lists
											# We do this to allow using this method as a collate_fn function in PyTorch Dataloader
											if isinstance(__snake_case     ,		(list, tuple)					) and isinstance(processed_features[0]     ,		(dict, BatchFeature)					):
																	__magic_name__:	Union[str, Any]							=       {
																	    key: [example[key] for example in processed_features] for key in processed_features[0].keys()
																	}

											# The model's main input name, usually `input_values`, has be passed for padding
											if self.model_input_names[0] not in processed_features:
																	raise ValueError(
																	    """You should supply an instance of `transformers.BatchFeature` or list of `transformers.BatchFeature`"""
																	    F' to this method that includes {self.model_input_names[0]}, but you provided'
																	    F' {list(processed_features.keys()					)}'					)

											__magic_name__:	Any							=       processed_features[self.model_input_names[0]]
											__magic_name__:	Tuple							=       (
											    return_attention_mask if return_attention_mask is not None else self.return_attention_mask
											)

											if len(__snake_case					) == 0:
																	if return_attention_mask:
																							__magic_name__:	Optional[Any]							=       []
																	return processed_features

											# If we have PyTorch/TF tensors or lists as inputs, we cast them as Numpy arrays
											# and rebuild them afterwards if no return_tensors is specified
											# Note that we lose the specific device the tensor may be on for PyTorch

											__magic_name__:	Tuple							=       required_input[0]
											if isinstance(__snake_case     ,		(list, tuple)					):
																	# first_element might be an empty list/tuple in some edge cases so we grab the first non empty element.
																	__magic_name__:	str							=       0
																	while len(required_input[index]					) == 0:
																							index += 1
																	if index < len(__snake_case					):
																							__magic_name__:	Optional[int]							=       required_input[index][0]

											if return_tensors is None:
																	if is_tf_tensor(__snake_case					):
																							__magic_name__:	List[str]							=       """tf"""
																	elif is_torch_tensor(__snake_case					):
																							__magic_name__:	Any							=       """pt"""
																	elif isinstance(__snake_case     ,		(int, float, list, tuple, np.ndarray)					):
																							__magic_name__:	int							=       """np"""
																	else:
																							raise ValueError(
																							    F'type of {first_element} unknown: {type(__snake_case					)}. '
																							    """Should be one of a python, numpy, pytorch or tensorflow object."""					)

											for key, value in processed_features.items():
																	if isinstance(value[0]     ,		(int, float)					):
																							__magic_name__:	List[str]							=       to_numpy(__snake_case					)
																	else:
																							__magic_name__:	Any							=       [to_numpy(__snake_case					) for v in value]

        # Convert padding_strategy in PaddingStrategy
											__magic_name__:	str							=       self._get_padding_strategies(padding=__snake_case     ,		max_length=__snake_case					)

											__magic_name__:	str							=       processed_features[self.model_input_names[0]]

											__magic_name__:	str							=       len(__snake_case					)
											if not all(len(__snake_case					) == batch_size for v in processed_features.values()					):
																	raise ValueError("""Some items in the output dictionary have a different batch size than others."""					)

											__magic_name__:	List[Any]							=       []
											for i in range(__snake_case					):
																	__magic_name__:	List[Any]							=       {k: v[i] for k, v in processed_features.items()}
																	# truncation
																	__magic_name__:	List[Any]							=       self._truncate(
																	    __snake_case     ,		max_length=__snake_case     ,		pad_to_multiple_of=__snake_case     ,		truncation=__snake_case     ,		)
																	truncated_inputs.append(__snake_case					)

											if padding_strategy == PaddingStrategy.LONGEST:
																	# make sure that `max_length` cannot be longer than the longest truncated length
																	__magic_name__:	Union[str, Any]							=       max(len(input_slice[self.model_input_names[0]]					) for input_slice in truncated_inputs					)
																	__magic_name__:	Union[str, Any]							=       PaddingStrategy.MAX_LENGTH

											__magic_name__:	List[str]							=       {}
											for i in range(__snake_case					):
																	# padding
																	__magic_name__:	str							=       self._pad(
																	    truncated_inputs[i]     ,		max_length=__snake_case     ,		padding_strategy=__snake_case     ,		pad_to_multiple_of=__snake_case     ,		return_attention_mask=__snake_case     ,		)

																	for key, value in outputs.items():
																							if key not in batch_outputs:
																													__magic_name__:	Optional[Any]							=       []
																							if value.dtype is np.dtype(np.floataa					):
																													__magic_name__:	Any							=       value.astype(np.floataa					)
																							batch_outputs[key].append(__snake_case					)

											return BatchFeature(__snake_case     ,		tensor_type=__snake_case					)

					def   lowerCamelCase__   (			self  :      Tuple     ,		__snake_case  :      Union[Dict[str, np.ndarray], BatchFeature]     ,		__snake_case  :      Optional[int] = None     ,		__snake_case  :      PaddingStrategy = PaddingStrategy.DO_NOT_PAD     ,		__snake_case  :      Optional[int] = None     ,		__snake_case  :      Optional[bool] = None     ,		)			->							dict:
											__magic_name__:	List[str]							=       processed_features[self.model_input_names[0]]

											if padding_strategy == PaddingStrategy.LONGEST:
																	__magic_name__:	List[Any]							=       len(__snake_case					)

											if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
																	__magic_name__:	List[str]							=       ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

											__magic_name__:	str							=       padding_strategy != PaddingStrategy.DO_NOT_PAD and len(__snake_case					) < max_length

											if return_attention_mask and "attention_mask" not in processed_features:
																	__magic_name__:	int							=       np.ones(len(__snake_case					)     ,		dtype=np.intaa					)

											if needs_to_be_padded:
																	__magic_name__:	str							=       max_length - len(__snake_case					)
																	if self.padding_side == "right":
																							if return_attention_mask:
																													__magic_name__:	List[Any]							=       np.pad(
																													    processed_features["""attention_mask"""]     ,		(0, difference)					)
																							__magic_name__:	Union[str, Any]							=       ((0, difference), (0, 0)) if self.feature_size > 1 else (0, difference)
																							__magic_name__:	int							=       np.pad(
																							    __snake_case     ,		__snake_case     ,		"""constant"""     ,		constant_values=self.padding_value					)
																	elif self.padding_side == "left":
																							if return_attention_mask:
																													__magic_name__:	Dict							=       np.pad(
																													    processed_features["""attention_mask"""]     ,		(difference, 0)					)
																							__magic_name__:	Optional[Any]							=       ((difference, 0), (0, 0)) if self.feature_size > 1 else (difference, 0)
																							__magic_name__:	int							=       np.pad(
																							    __snake_case     ,		__snake_case     ,		"""constant"""     ,		constant_values=self.padding_value					)
																	else:
																							raise ValueError("""Invalid padding strategy:""" + str(self.padding_side					)					)

											return processed_features

					def   lowerCamelCase__   (			self  :      Optional[Any]     ,		__snake_case  :      Union[Dict[str, np.ndarray], BatchFeature]     ,		__snake_case  :      Optional[int] = None     ,		__snake_case  :      Optional[int] = None     ,		__snake_case  :      Optional[bool] = None     ,		)			->							int:
											if not truncation:
																	return processed_features
											elif truncation and max_length is None:
																	raise ValueError("""When setting ``truncation=True``, make sure that ``max_length`` is defined."""					)

											__magic_name__:	Dict							=       processed_features[self.model_input_names[0]]

											# find `max_length` that fits `pad_to_multiple_of`
											if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
																	__magic_name__:	Optional[Any]							=       ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

											__magic_name__:	Tuple							=       len(__snake_case					) > max_length

											if needs_to_be_truncated:
																	__magic_name__:	Any							=       processed_features[self.model_input_names[0]][:max_length]
																	if "attention_mask" in processed_features:
																							__magic_name__:	List[Any]							=       processed_features["""attention_mask"""][:max_length]

											return processed_features






					def   lowerCamelCase__   (			self  :      List[Any]     ,		__snake_case  :      int=False     ,		__snake_case  :      Tuple=None					)			->							Optional[Any]:

											# Get padding strategy
											if padding is not False:
																	if padding is True:
																							__magic_name__:	Optional[Any]							=       PaddingStrategy.LONGEST  # Default to pad to the longest sequence in the batch
																	elif not isinstance(__snake_case     ,		__snake_case					):
																							__magic_name__:	Tuple							=       PaddingStrategy(__snake_case					)
																	elif isinstance(__snake_case     ,		__snake_case					):
																							__magic_name__:	Dict							=       padding
											else:
																	__magic_name__:	int							=       PaddingStrategy.DO_NOT_PAD

											# Set max length if needed
											if max_length is None:
																	if padding_strategy == PaddingStrategy.MAX_LENGTH:
																							raise ValueError(
																							    F'When setting ``padding={PaddingStrategy.MAX_LENGTH}``, make sure that max_length is defined'					)

        # Test if we have a padding value
											if padding_strategy != PaddingStrategy.DO_NOT_PAD and (self.padding_value is None):
																	raise ValueError(
																	    """Asking to pad but the feature_extractor does not have a padding value. Please select a value to use"""
																	    """ as `padding_value`. For example: `feature_extractor.padding_value = 0.0`."""					)

											return padding_strategy



