


"""simple docstring"""






import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY


@is_pipeline_test
class     __lowerCamelCase  (					unittest.TestCase				):

     '''simple docstring'''
     A_       :	List[str]    =		MODEL_FOR_MASKED_LM_MAPPING
     A_       :	Optional[Any]    =		TF_MODEL_FOR_MASKED_LM_MAPPING


     def 							_UpperCAmelCase      (      self							)       ->							int:
          super().tearDown()
          # clean-up as much as possible GPU memory occupied by PyTorch
          gc.collect()
          if is_torch_available():
               import torch

               torch.cuda.empty_cache()


     @require_tf
     def 							_UpperCAmelCase      (      self							)       ->							Dict:
          _a    	=				pipeline(task='''fill-mask'''    ,				model='''sshleifer/tiny-distilroberta-base'''    ,				top_k=2    ,				framework='''tf'''							)
          _a    	=				unmasker('''My name is <mask>'''							)
          self.assertEqual(
              nested_simplify(__UpperCAmelCase    ,				decimals=6							)    ,				[
                  {'''sequence''': '''My name is grouped''', '''score''': 2.1e-0_5, '''token''': 38015, '''token_str''': ''' grouped'''},
                  {'''sequence''': '''My name is accuser''', '''score''': 2.1e-0_5, '''token''': 25506, '''token_str''': ''' accuser'''},
              ]    ,				)

          _a    	=				unmasker('''The largest city in France is <mask>'''							)
          self.assertEqual(
              nested_simplify(__UpperCAmelCase    ,				decimals=6							)    ,				[
                  {
                      '''sequence''': '''The largest city in France is grouped''',
                      '''score''': 2.1e-0_5,
                      '''token''': 38015,
                      '''token_str''': ''' grouped''',
                  },
                  {
                      '''sequence''': '''The largest city in France is accuser''',
                      '''score''': 2.1e-0_5,
                      '''token''': 25506,
                      '''token_str''': ''' accuser''',
                  },
              ]    ,				)

          _a    	=				unmasker('''My name is <mask>'''    ,				targets=[''' Patrick''', ''' Clara''', ''' Teven''']    ,				top_k=3							)
          self.assertEqual(
              nested_simplify(__UpperCAmelCase    ,				decimals=6							)    ,				[
                  {'''sequence''': '''My name is Clara''', '''score''': 2e-0_5, '''token''': 13606, '''token_str''': ''' Clara'''},
                  {'''sequence''': '''My name is Patrick''', '''score''': 2e-0_5, '''token''': 3499, '''token_str''': ''' Patrick'''},
                  {'''sequence''': '''My name is Te''', '''score''': 1.9e-0_5, '''token''': 2941, '''token_str''': ''' Te'''},
              ]    ,				)


     @require_torch
     def 							_UpperCAmelCase      (      self							)       ->							int:
          _a    	=				pipeline(task='''fill-mask'''    ,				model='''sshleifer/tiny-distilroberta-base'''    ,				top_k=2    ,				framework='''pt'''							)

          _a    	=				unmasker('''My name is <mask>'''							)
          self.assertEqual(
              nested_simplify(__UpperCAmelCase    ,				decimals=6							)    ,				[
                  {'''sequence''': '''My name is Maul''', '''score''': 2.2e-0_5, '''token''': 35676, '''token_str''': ''' Maul'''},
                  {'''sequence''': '''My name isELS''', '''score''': 2.2e-0_5, '''token''': 16416, '''token_str''': '''ELS'''},
              ]    ,				)

          _a    	=				unmasker('''The largest city in France is <mask>'''							)
          self.assertEqual(
              nested_simplify(__UpperCAmelCase    ,				decimals=6							)    ,				[
                  {
                      '''sequence''': '''The largest city in France is Maul''',
                      '''score''': 2.2e-0_5,
                      '''token''': 35676,
                      '''token_str''': ''' Maul''',
                  },
                  {'''sequence''': '''The largest city in France isELS''', '''score''': 2.2e-0_5, '''token''': 16416, '''token_str''': '''ELS'''},
              ]    ,				)

          _a    	=				unmasker('''My name is <mask>'''    ,				targets=[''' Patrick''', ''' Clara''', ''' Teven''']    ,				top_k=3							)
          self.assertEqual(
              nested_simplify(__UpperCAmelCase    ,				decimals=6							)    ,				[
                  {'''sequence''': '''My name is Patrick''', '''score''': 2.1e-0_5, '''token''': 3499, '''token_str''': ''' Patrick'''},
                  {'''sequence''': '''My name is Te''', '''score''': 2e-0_5, '''token''': 2941, '''token_str''': ''' Te'''},
                  {'''sequence''': '''My name is Clara''', '''score''': 2e-0_5, '''token''': 13606, '''token_str''': ''' Clara'''},
              ]    ,				)

          _a    	=				unmasker('''My name is <mask> <mask>'''    ,				top_k=2							)

          self.assertEqual(
              nested_simplify(__UpperCAmelCase    ,				decimals=6							)    ,				[
                  [
                      {
                          '''score''': 2.2e-0_5,
                          '''token''': 35676,
                          '''token_str''': ''' Maul''',
                          '''sequence''': '''<s>My name is Maul<mask></s>''',
                      },
                      {'''score''': 2.2e-0_5, '''token''': 16416, '''token_str''': '''ELS''', '''sequence''': '''<s>My name isELS<mask></s>'''},
                  ],
                  [
                      {
                          '''score''': 2.2e-0_5,
                          '''token''': 35676,
                          '''token_str''': ''' Maul''',
                          '''sequence''': '''<s>My name is<mask> Maul</s>''',
                      },
                      {'''score''': 2.2e-0_5, '''token''': 16416, '''token_str''': '''ELS''', '''sequence''': '''<s>My name is<mask>ELS</s>'''},
                  ],
              ]    ,				)


     @require_torch_gpu
     def 							_UpperCAmelCase      (      self							)       ->							str:
          _a    	=				pipeline('''fill-mask'''    ,				model='''hf-internal-testing/tiny-random-distilbert'''    ,				device=0    ,				framework='''pt'''							)

          # convert model to fp16
          pipe.model.half()

          _a    	=				pipe('''Paris is the [MASK] of France.'''							)
          # We actually don't care about the result, we just want to make sure
          # it works, meaning the float16 tensor got casted back to float32
          # for postprocessing.
          self.assertIsInstance(__UpperCAmelCase    ,				__UpperCAmelCase							)


     @slow
     @require_torch
     def 							_UpperCAmelCase      (      self							)       ->							Any:
          _a    	=				pipeline(task='''fill-mask'''    ,				model='''distilroberta-base'''    ,				top_k=2    ,				framework='''pt'''							)
          self.run_large_test(__UpperCAmelCase							)


     @slow
     @require_tf
     def 							_UpperCAmelCase      (      self							)       ->							int:
          _a    	=				pipeline(task='''fill-mask'''    ,				model='''distilroberta-base'''    ,				top_k=2    ,				framework='''tf'''							)
          self.run_large_test(__UpperCAmelCase							)


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase							)       ->							Optional[Any]:
          _a    	=				unmasker('''My name is <mask>'''							)
          self.assertEqual(
              nested_simplify(__UpperCAmelCase							)    ,				[
                  {'''sequence''': '''My name is John''', '''score''': 0.008, '''token''': 610, '''token_str''': ''' John'''},
                  {'''sequence''': '''My name is Chris''', '''score''': 0.007, '''token''': 1573, '''token_str''': ''' Chris'''},
              ]    ,				)
          _a    	=				unmasker('''The largest city in France is <mask>'''							)
          self.assertEqual(
              nested_simplify(__UpperCAmelCase							)    ,				[
                  {
                      '''sequence''': '''The largest city in France is Paris''',
                      '''score''': 0.251,
                      '''token''': 2201,
                      '''token_str''': ''' Paris''',
                  },
                  {
                      '''sequence''': '''The largest city in France is Lyon''',
                      '''score''': 0.214,
                      '''token''': 12790,
                      '''token_str''': ''' Lyon''',
                  },
              ]    ,				)

          _a    	=				unmasker('''My name is <mask>'''    ,				targets=[''' Patrick''', ''' Clara''', ''' Teven''']    ,				top_k=3							)
          self.assertEqual(
              nested_simplify(__UpperCAmelCase							)    ,				[
                  {'''sequence''': '''My name is Patrick''', '''score''': 0.005, '''token''': 3499, '''token_str''': ''' Patrick'''},
                  {'''sequence''': '''My name is Clara''', '''score''': 0.000, '''token''': 13606, '''token_str''': ''' Clara'''},
                  {'''sequence''': '''My name is Te''', '''score''': 0.000, '''token''': 2941, '''token_str''': ''' Te'''},
              ]    ,				)


     @require_torch
     def 							_UpperCAmelCase      (      self							)       ->							Union[str, Any]:
          _a    	=				pipeline(task='''fill-mask'''    ,				model='''sshleifer/tiny-distilroberta-base'''    ,				framework='''pt'''							)
          _a    	=				None
          _a    	=				None
          self.run_pipeline_test(__UpperCAmelCase    ,				[]							)


     @require_tf
     def 							_UpperCAmelCase      (      self							)       ->							str:
          _a    	=				pipeline(task='''fill-mask'''    ,				model='''sshleifer/tiny-distilroberta-base'''    ,				framework='''tf'''							)
          _a    	=				None
          _a    	=				None
          self.run_pipeline_test(__UpperCAmelCase    ,				[]							)


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase    ,				__UpperCAmelCase							)       ->							List[str]:
          if tokenizer is None or tokenizer.mask_token_id is None:
               self.skipTest('''The provided tokenizer has no mask token, (probably reformer or wav2vec2)'''							)

          _a    	=				FillMaskPipeline(model=__UpperCAmelCase    ,				tokenizer=__UpperCAmelCase							)
          _a    	=				[
              F'This is another {tokenizer.mask_token} test',
          ]
          return fill_masker, examples


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase							)       ->							Optional[Any]:
          _a    	=				fill_masker.tokenizer
          _a    	=				fill_masker.model

          _a    	=				fill_masker(
              F'This is a {tokenizer.mask_token}'    ,				)
          self.assertEqual(
              __UpperCAmelCase    ,				[
                  {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                  {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                  {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                  {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                  {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
              ]    ,				)

          _a    	=				fill_masker([F'This is a {tokenizer.mask_token}']							)
          self.assertEqual(
              __UpperCAmelCase    ,				[
                  {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                  {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                  {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                  {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                  {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
              ]    ,				)

          _a    	=				fill_masker([F'This is a {tokenizer.mask_token}', F'Another {tokenizer.mask_token} great test.']							)
          self.assertEqual(
              __UpperCAmelCase    ,				[
                  [
                      {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                      {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                      {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                      {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                      {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                  ],
                  [
                      {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                      {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                      {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                      {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                      {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                  ],
              ]    ,				)

          with self.assertRaises(__UpperCAmelCase							):
               fill_masker([None]							)
          # No mask_token is not supported
          with self.assertRaises(__UpperCAmelCase							):
               fill_masker('''This is'''							)

          self.run_test_top_k(__UpperCAmelCase    ,				__UpperCAmelCase							)
          self.run_test_targets(__UpperCAmelCase    ,				__UpperCAmelCase							)
          self.run_test_top_k_targets(__UpperCAmelCase    ,				__UpperCAmelCase							)
          self.fill_mask_with_duplicate_targets_and_top_k(__UpperCAmelCase    ,				__UpperCAmelCase							)
          self.fill_mask_with_multiple_masks(__UpperCAmelCase    ,				__UpperCAmelCase							)


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase							)       ->							Any:
          _a    	=				tokenizer.get_vocab()
          _a    	=				sorted(vocab.keys()							)[:2]
          # Pipeline argument
          _a    	=				FillMaskPipeline(model=__UpperCAmelCase    ,				tokenizer=__UpperCAmelCase    ,				targets=__UpperCAmelCase							)
          _a    	=				fill_masker(F'This is a {tokenizer.mask_token}'							)
          self.assertEqual(
              __UpperCAmelCase    ,				[
                  {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                  {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
              ]    ,				)
          _a    	=				{vocab[el] for el in targets}
          self.assertEqual({el['''token'''] for el in outputs}    ,				__UpperCAmelCase							)
          _a    	=				[tokenizer.decode([x]							) for x in target_ids]
          self.assertEqual({el['''token_str'''] for el in outputs}    ,				set(__UpperCAmelCase							)							)

          # Call argument
          _a    	=				FillMaskPipeline(model=__UpperCAmelCase    ,				tokenizer=__UpperCAmelCase							)
          _a    	=				fill_masker(F'This is a {tokenizer.mask_token}'    ,				targets=__UpperCAmelCase							)
          self.assertEqual(
              __UpperCAmelCase    ,				[
                  {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                  {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
              ]    ,				)
          _a    	=				{vocab[el] for el in targets}
          self.assertEqual({el['''token'''] for el in outputs}    ,				__UpperCAmelCase							)
          _a    	=				[tokenizer.decode([x]							) for x in target_ids]
          self.assertEqual({el['''token_str'''] for el in outputs}    ,				set(__UpperCAmelCase							)							)

          # Score equivalence
          _a    	=				fill_masker(F'This is a {tokenizer.mask_token}'    ,				targets=__UpperCAmelCase							)
          _a    	=				[top_mask['''token_str'''] for top_mask in outputs]
          _a    	=				[top_mask['''score'''] for top_mask in outputs]

          # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
          if set(__UpperCAmelCase							) == set(__UpperCAmelCase							):
               _a    	=				fill_masker(F'This is a {tokenizer.mask_token}'    ,				targets=__UpperCAmelCase							)
               _a    	=				[top_mask['''score'''] for top_mask in unmasked_targets]
               self.assertEqual(nested_simplify(__UpperCAmelCase							)    ,				nested_simplify(__UpperCAmelCase							)							)

          # Raises with invalid
          with self.assertRaises(__UpperCAmelCase							):
               _a    	=				fill_masker(F'This is a {tokenizer.mask_token}'    ,				targets=[]							)
          # For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
          if "" not in tokenizer.get_vocab():
               with self.assertRaises(__UpperCAmelCase							):
                    _a    	=				fill_masker(F'This is a {tokenizer.mask_token}'    ,				targets=['''''']							)
               with self.assertRaises(__UpperCAmelCase							):
                    _a    	=				fill_masker(F'This is a {tokenizer.mask_token}'    ,				targets=''''''							)


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase							)       ->							Any:
          _a    	=				FillMaskPipeline(model=__UpperCAmelCase    ,				tokenizer=__UpperCAmelCase    ,				top_k=2							)
          _a    	=				fill_masker(F'This is a {tokenizer.mask_token}'							)
          self.assertEqual(
              __UpperCAmelCase    ,				[
                  {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                  {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
              ]    ,				)

          _a    	=				FillMaskPipeline(model=__UpperCAmelCase    ,				tokenizer=__UpperCAmelCase							)
          _a    	=				fill_masker(F'This is a {tokenizer.mask_token}'    ,				top_k=2							)
          self.assertEqual(
              __UpperCAmelCase    ,				[
                  {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                  {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
              ]    ,				)
          self.assertEqual(nested_simplify(__UpperCAmelCase							)    ,				nested_simplify(__UpperCAmelCase							)							)


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase							)       ->							Tuple:
          _a    	=				tokenizer.get_vocab()
          _a    	=				FillMaskPipeline(model=__UpperCAmelCase    ,				tokenizer=__UpperCAmelCase							)

          # top_k=2, ntargets=3
          _a    	=				sorted(vocab.keys()							)[:3]
          _a    	=				fill_masker(F'This is a {tokenizer.mask_token}'    ,				top_k=2    ,				targets=__UpperCAmelCase							)

          # If we use the most probably targets, and filter differently, we should still
          # have the same results
          _a    	=				[el['''token_str'''] for el in sorted(__UpperCAmelCase    ,				key=lambda __UpperCAmelCase							: x["score"]    ,				reverse=__UpperCAmelCase							)]
          # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
          if set(__UpperCAmelCase							).issubset(__UpperCAmelCase							):
               _a    	=				fill_masker(F'This is a {tokenizer.mask_token}'    ,				top_k=3    ,				targets=__UpperCAmelCase							)
               # They should yield exactly the same result
               self.assertEqual(nested_simplify(__UpperCAmelCase							)    ,				nested_simplify(__UpperCAmelCase							)							)


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase							)       ->							Union[str, Any]:
          _a    	=				FillMaskPipeline(model=__UpperCAmelCase    ,				tokenizer=__UpperCAmelCase							)
          _a    	=				tokenizer.get_vocab()
          # String duplicates + id duplicates
          _a    	=				sorted(vocab.keys()							)[:3]
          _a    	=				[targets[0], targets[1], targets[0], targets[2], targets[1]]
          _a    	=				fill_masker(F'My name is {tokenizer.mask_token}'    ,				targets=__UpperCAmelCase    ,				top_k=10							)

          # The target list contains duplicates, so we can't output more
          # than them
          self.assertEqual(len(__UpperCAmelCase							)    ,				3							)





     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase							)       ->							int:
          _a    	=				FillMaskPipeline(model=__UpperCAmelCase    ,				tokenizer=__UpperCAmelCase							)

          _a    	=				fill_masker(
              F'This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}'    ,				top_k=2							)
          self.assertEqual(
              __UpperCAmelCase    ,				[
                  [
                      {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                      {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                  ],
                  [
                      {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                      {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                  ],
                  [
                      {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                      {'''sequence''': ANY(__UpperCAmelCase							), '''score''': ANY(__UpperCAmelCase							), '''token''': ANY(__UpperCAmelCase							), '''token_str''': ANY(__UpperCAmelCase							)},
                  ],
              ]    ,				)



"""simple docstring"""






from typing import List, Optional, Union

import numpy as np

from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


__snake_case		=       logging.get_logger(__name__)


class     __lowerCamelCase  (					a__				):

     '''simple docstring'''
     A_       :	Union[str, Any]    =		['input_values', 'padding_mask']


     def __init__(      self    ,				__UpperCAmelCase = 1    ,				__UpperCAmelCase = 24000    ,				__UpperCAmelCase = 0.0    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				**__UpperCAmelCase    ,				)       ->							List[str]:
          super().__init__(feature_size=__UpperCAmelCase    ,				sampling_rate=__UpperCAmelCase    ,				padding_value=__UpperCAmelCase    ,				**__UpperCAmelCase							)
          _a    	=				chunk_length_s
          _a    	=				overlap


     @property
     def 							_UpperCAmelCase      (      self							)       ->							Optional[int]:
          if self.chunk_length_s is None:
               return None
          else:
               return int(self.chunk_length_s * self.sampling_rate							)


     @property
     def 							_UpperCAmelCase      (      self							)       ->							Optional[int]:
          if self.chunk_length_s is None or self.overlap is None:
               return None
          else:
               return max(1    ,				int((1.0 - self.overlap) * self.chunk_length							)							)





     def __call__(      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = False    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				)       ->							BatchFeature:
          if sampling_rate is not None:
               if sampling_rate != self.sampling_rate:
                    raise ValueError(
                        F'The model corresponding to this feature extractor: {self} was trained using a sampling rate of'
                        F' {self.sampling_rate}. Please make sure that the provided audio input was sampled with'
                        F' {self.sampling_rate} and not {sampling_rate}.'							)
          else:
               logger.warning(
                   '''It is strongly recommended to pass the `sampling_rate` argument to this function. '''
                   '''Failing to do so can result in silent errors that might be hard to debug.'''							)

          if padding and truncation:
               raise ValueError('''Both padding and truncation were set. Make sure you only set one.'''							)
          elif padding is None:
               # by default let's pad the inputs
               _a    	=				True

          _a    	=				bool(
              isinstance(__UpperCAmelCase    ,				(list, tuple)							) and (isinstance(raw_audio[0]    ,				(np.ndarray, tuple, list)							))							)

          if is_batched:
               _a    	=				[np.asarray(__UpperCAmelCase    ,				dtype=np.floataa							).T for audio in raw_audio]
          elif not is_batched and not isinstance(__UpperCAmelCase    ,				np.ndarray							):
               _a    	=				np.asarray(__UpperCAmelCase    ,				dtype=np.floataa							)
          elif isinstance(__UpperCAmelCase    ,				np.ndarray							) and raw_audio.dtype is np.dtype(np.floataa							):
               _a    	=				raw_audio.astype(np.floataa							)

          # always return batch
          if not is_batched:
               _a    	=				[np.asarray(__UpperCAmelCase							).T]

          # verify inputs are valid
          for idx, example in enumerate(__UpperCAmelCase							):
               if example.ndim > 2:
                    raise ValueError(F'Expected input shape (channels, length) but got shape {example.shape}'							)
               if self.feature_size == 1 and example.ndim != 1:
                    raise ValueError(F'Expected mono audio but example has {example.shape[-1]} channels'							)
               if self.feature_size == 2 and example.shape[-1] != 2:
                    raise ValueError(F'Expected stereo audio but example has {example.shape[-1]} channels'							)

          _a    	=				None
          _a    	=				BatchFeature({'''input_values''': raw_audio}							)
          if self.chunk_stride is not None and self.chunk_length is not None and max_length is None:
               if truncation:
                    _a    	=				min(array.shape[0] for array in raw_audio							)
                    _a    	=				int(np.floor(max_length / self.chunk_stride							)							)
                    _a    	=				(nb_step - 1) * self.chunk_stride + self.chunk_length
               elif padding:
                    _a    	=				max(array.shape[0] for array in raw_audio							)
                    _a    	=				int(np.ceil(max_length / self.chunk_stride							)							)
                    _a    	=				(nb_step - 1) * self.chunk_stride + self.chunk_length
                    _a    	=				'''max_length'''
               else:
                    _a    	=				input_values

        # normal padding on batch
          if padded_inputs is None:
               _a    	=				self.pad(
                   __UpperCAmelCase    ,				max_length=__UpperCAmelCase    ,				truncation=__UpperCAmelCase    ,				padding=__UpperCAmelCase    ,				return_attention_mask=__UpperCAmelCase    ,				)
               if padding:
                    _a    	=				padded_inputs.pop('''attention_mask'''							)

          _a    	=				[]
          for example in padded_inputs.pop('''input_values'''							):
               if self.feature_size == 1:
                    _a    	=				example[..., None]
               input_values.append(example.T							)

          _a    	=				input_values
          if return_tensors is not None:
               _a    	=				padded_inputs.convert_to_tensors(__UpperCAmelCase							)

          return padded_inputs