


import itertools
import random
import unittest

import numpy as np

from transformers import BatchFeature, SpeechTaFeatureExtractor
from transformers.testing_utils import require_torch
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_torch_available():
 import torch


lowerCamelCase		   =			random.Random()




def       a_ (       SCREAMING_SNAKE_CASE__   : Any						,     SCREAMING_SNAKE_CASE__   : Any=1.0						,     SCREAMING_SNAKE_CASE__   : Union[str, Any]=None						,     SCREAMING_SNAKE_CASE__   : int=None	):
       '''simple docstring'''





       if rng is None:
              _lowerCamelCase			:						List[Any]            =global_rng

       _lowerCamelCase			:						Optional[int]            =[]
       for batch_idx in range(shape[0]	):
              values.append([]	)
              for _ in range(shape[1]	):
                     values[-1].append(rng.random() * scale	)

       return values




@require_torch
class A    (  unittest.TestCase	):






      def __init__(   self		:	Optional[Any]     ,  lowercase_		:	int     ,  lowercase_		:	Any=7     ,  lowercase_		:	List[str]=400     ,  lowercase_		:	Tuple=2000     ,  lowercase_		:	List[str]=1     ,  lowercase_		:	Optional[Any]=0.0     ,  lowercase_		:	Union[str, Any]=1_6000     ,  lowercase_		:	Tuple=True     ,  lowercase_		:	Dict=80     ,  lowercase_		:	List[str]=16     ,  lowercase_		:	Union[str, Any]=64     ,  lowercase_		:	List[Any]="hann_window"     ,  lowercase_		:	Dict=80     ,  lowercase_		:	Any=7600     ,  lowercase_		:	Any=1E-10     ,  lowercase_		:	List[Any]=True     ,  )   ->		Any:
             """simple docstring"""




             _lowerCamelCase			:						List[str]            =parent
             _lowerCamelCase			:						int            =batch_size
             _lowerCamelCase			:						List[str]            =min_seq_length
             _lowerCamelCase			:						List[str]            =max_seq_length
             _lowerCamelCase			:						Dict            =(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
             _lowerCamelCase			:						Dict            =feature_size
             _lowerCamelCase			:						Tuple            =padding_value
             _lowerCamelCase			:						Tuple            =sampling_rate
             _lowerCamelCase			:						Optional[Any]            =do_normalize
             _lowerCamelCase			:						List[str]            =num_mel_bins
             _lowerCamelCase			:						Any            =hop_length
             _lowerCamelCase			:						Optional[Any]            =win_length
             _lowerCamelCase			:						Tuple            =win_function
             _lowerCamelCase			:						int            =fmin
             _lowerCamelCase			:						Union[str, Any]            =fmax
             _lowerCamelCase			:						List[str]            =mel_floor
             _lowerCamelCase			:						List[str]            =return_attention_mask






      def 			lowerCamelCase			(   self		:	Union[str, Any]					)   ->		List[str]:
             """simple docstring"""




             return {
                 "feature_size": self.feature_size,
                 "padding_value": self.padding_value,
                 "sampling_rate": self.sampling_rate,
                 "do_normalize": self.do_normalize,
                 "num_mel_bins": self.num_mel_bins,
                 "hop_length": self.hop_length,
                 "win_length": self.win_length,
                 "win_function": self.win_function,
                 "fmin": self.fmin,
                 "fmax": self.fmax,
                 "mel_floor": self.mel_floor,
                 "return_attention_mask": self.return_attention_mask,
             }






      def 			lowerCamelCase			(   self		:	Tuple     ,  lowercase_		:	Optional[int]=False     ,  lowercase_		:	Union[str, Any]=False					)   ->		Union[str, Any]:
             """simple docstring"""




             def _flatten(lowercase_		:	Optional[Any]					):
                    return list(itertools.chain(*UpperCamelCase__					)					)

             if equal_length:
                    _lowerCamelCase			:						List[str]            =floats_list((self.batch_size, self.max_seq_length)					)
             else:
                    # make sure that inputs increase in size
                    _lowerCamelCase			:						Union[str, Any]            =[
                        _flatten(floats_list((x, self.feature_size)					)					)
                        for x in range(self.min_seq_length     ,  self.max_seq_length     ,  self.seq_length_diff					)
                    ]

             if numpify:
                    _lowerCamelCase			:						Optional[int]            =[np.asarray(UpperCamelCase__					) for x in speech_inputs]

             return speech_inputs






      def 			lowerCamelCase			(   self		:	Optional[Any]     ,  lowercase_		:	Union[str, Any]=False     ,  lowercase_		:	Union[str, Any]=False					)   ->		Optional[int]:
             """simple docstring"""




             if equal_length:
                    _lowerCamelCase			:						Optional[Any]            =[floats_list((self.max_seq_length, self.num_mel_bins)					) for _ in range(self.batch_size					)]
             else:
                    # make sure that inputs increase in size
                    _lowerCamelCase			:						Tuple            =[
                        floats_list((x, self.num_mel_bins)					)
                        for x in range(self.min_seq_length     ,  self.max_seq_length     ,  self.seq_length_diff					)
                    ]

             if numpify:
                    _lowerCamelCase			:						Dict            =[np.asarray(UpperCamelCase__					) for x in speech_inputs]

             return speech_inputs




@require_torch
class A    (  _A       ,				unittest.TestCase	):
      UpperCamelCase__				:    Tuple =SpeechTaFeatureExtractor






      def 			lowerCamelCase			(   self		:	Dict					)   ->		int:
             """simple docstring"""




             _lowerCamelCase			:						Any            =SpeechTaFeatureExtractionTester(self					)






      def 			lowerCamelCase			(   self		:	str     ,  lowercase_		:	List[Any]					)   ->		Tuple:
             """simple docstring"""




             self.assertTrue(np.all(np.mean(UpperCamelCase__     ,  axis=0					) < 1E-3					)					)
             self.assertTrue(np.all(np.abs(np.var(UpperCamelCase__     ,  axis=0					) - 1					) < 1E-3					)					)






      def 			lowerCamelCase			(   self		:	str					)   ->		str:
             """simple docstring"""




             _lowerCamelCase			:						str            =self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()					)
             # create three inputs of length 800, 1000, and 1200
             _lowerCamelCase			:						int            =[floats_list((1, x)					)[0] for x in range(800     ,  1400     ,  200					)]
             _lowerCamelCase			:						Optional[Any]            =[np.asarray(UpperCamelCase__					) for speech_input in speech_inputs]

             # Test not batched input
             _lowerCamelCase			:						str            =feat_extract(speech_inputs[0]     ,  return_tensors='np'					).input_values
             _lowerCamelCase			:						int            =feat_extract(np_speech_inputs[0]     ,  return_tensors='np'					).input_values
             self.assertTrue(np.allclose(UpperCamelCase__     ,  UpperCamelCase__     ,  atol=1E-3					)					)

             # Test batched
             _lowerCamelCase			:						List[str]            =feat_extract(UpperCamelCase__     ,  return_tensors='np'					).input_values
             _lowerCamelCase			:						Optional[Any]            =feat_extract(UpperCamelCase__     ,  return_tensors='np'					).input_values
             for enc_seq_a, enc_seq_a in zip(UpperCamelCase__     ,  UpperCamelCase__					):
                    self.assertTrue(np.allclose(UpperCamelCase__     ,  UpperCamelCase__     ,  atol=1E-3					)					)






      def 			lowerCamelCase			(   self		:	Union[str, Any]					)   ->		Optional[int]:
             """simple docstring"""




             _lowerCamelCase			:						Any            =self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()					)
             _lowerCamelCase			:						Dict            =[floats_list((1, x)					)[0] for x in range(800     ,  1400     ,  200					)]

             _lowerCamelCase			:						Optional[Any]            =['longest', 'max_length', 'do_not_pad']
             _lowerCamelCase			:						Optional[Any]            =[None, 1600, None]
             for max_length, padding in zip(UpperCamelCase__     ,  UpperCamelCase__					):
                    _lowerCamelCase			:						Tuple            =feat_extract(UpperCamelCase__     ,  padding=UpperCamelCase__     ,  max_length=UpperCamelCase__     ,  return_tensors='np'					)
                    _lowerCamelCase			:						int            =processed.input_values

                    self._check_zero_mean_unit_variance(input_values[0][:800]					)
                    self.assertTrue(input_values[0][800:].sum() < 1E-6					)
                    self._check_zero_mean_unit_variance(input_values[1][:1000]					)
                    self.assertTrue(input_values[0][1000:].sum() < 1E-6					)
                    self._check_zero_mean_unit_variance(input_values[2][:1200]					)






      def 			lowerCamelCase			(   self		:	Dict					)   ->		Dict:
             """simple docstring"""




             _lowerCamelCase			:						List[Any]            =self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()					)
             _lowerCamelCase			:						Tuple            =range(800     ,  1400     ,  200					)
             _lowerCamelCase			:						List[str]            =[floats_list((1, x)					)[0] for x in lengths]

             _lowerCamelCase			:						int            =['longest', 'max_length', 'do_not_pad']
             _lowerCamelCase			:						int            =[None, 1600, None]

             for max_length, padding in zip(UpperCamelCase__     ,  UpperCamelCase__					):
                    _lowerCamelCase			:						Optional[Any]            =feat_extract(UpperCamelCase__     ,  max_length=UpperCamelCase__     ,  padding=UpperCamelCase__					)
                    _lowerCamelCase			:						Optional[int]            =processed.input_values

                    self._check_zero_mean_unit_variance(input_values[0][:800]					)
                    self._check_zero_mean_unit_variance(input_values[1][:1000]					)
                    self._check_zero_mean_unit_variance(input_values[2][:1200]					)






      def 			lowerCamelCase			(   self		:	Union[str, Any]					)   ->		Dict:
             """simple docstring"""




             _lowerCamelCase			:						int            =self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()					)
             _lowerCamelCase			:						Tuple            =[floats_list((1, x)					)[0] for x in range(800     ,  1400     ,  200					)]
             _lowerCamelCase			:						Optional[Any]            =feat_extract(
                 UpperCamelCase__     ,  truncation=UpperCamelCase__     ,  max_length=1000     ,  padding='max_length'     ,  return_tensors='np'					)
             _lowerCamelCase			:						Union[str, Any]            =processed.input_values

             self._check_zero_mean_unit_variance(input_values[0, :800]					)
             self._check_zero_mean_unit_variance(input_values[1]					)
             self._check_zero_mean_unit_variance(input_values[2]					)






      def 			lowerCamelCase			(   self		:	Optional[int]					)   ->		Tuple:
             """simple docstring"""




             _lowerCamelCase			:						Optional[int]            =self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()					)
             _lowerCamelCase			:						Any            =[floats_list((1, x)					)[0] for x in range(800     ,  1400     ,  200					)]
             _lowerCamelCase			:						Tuple            =feat_extract(
                 UpperCamelCase__     ,  truncation=UpperCamelCase__     ,  max_length=1000     ,  padding='longest'     ,  return_tensors='np'					)
             _lowerCamelCase			:						List[str]            =processed.input_values

             self._check_zero_mean_unit_variance(input_values[0, :800]					)
             self._check_zero_mean_unit_variance(input_values[1, :1000]					)
             self._check_zero_mean_unit_variance(input_values[2]					)

             # make sure that if max_length < longest -> then pad to max_length
             self.assertTrue(input_values.shape == (3, 1000)					)

             _lowerCamelCase			:						Optional[int]            =[floats_list((1, x)					)[0] for x in range(800     ,  1400     ,  200					)]
             _lowerCamelCase			:						Union[str, Any]            =feat_extract(
                 UpperCamelCase__     ,  truncation=UpperCamelCase__     ,  max_length=2000     ,  padding='longest'     ,  return_tensors='np'					)
             _lowerCamelCase			:						Optional[int]            =processed.input_values

             self._check_zero_mean_unit_variance(input_values[0, :800]					)
             self._check_zero_mean_unit_variance(input_values[1, :1000]					)
             self._check_zero_mean_unit_variance(input_values[2]					)

             # make sure that if max_length > longest -> then pad to longest
             self.assertTrue(input_values.shape == (3, 1200)					)






      def 			lowerCamelCase			(   self		:	int					)   ->		Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase			:						List[str]            =self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()					)
             _lowerCamelCase			:						str            =np.random.rand(100					).astype(np.floataa					)
             _lowerCamelCase			:						List[str]            =np_speech_inputs.tolist()

             for inputs in [py_speech_inputs, np_speech_inputs]:
                    _lowerCamelCase			:						Tuple            =feature_extractor.pad([{'input_values': inputs}]     ,  return_tensors='np'					)
                    self.assertTrue(np_processed.input_values.dtype == np.floataa					)
                    _lowerCamelCase			:						int            =feature_extractor.pad([{'input_values': inputs}]     ,  return_tensors='pt'					)
                    self.assertTrue(pt_processed.input_values.dtype == torch.floataa					)






      def 			lowerCamelCase			(   self		:	Union[str, Any]					)   ->		Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase			:						Tuple            =self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()					)
             # create three inputs of length 800, 1000, and 1200
             _lowerCamelCase			:						Dict            =[floats_list((1, x)					)[0] for x in range(800     ,  1400     ,  200					)]
             _lowerCamelCase			:						List[Any]            =[np.asarray(UpperCamelCase__					) for speech_input in speech_inputs]

             # Test feature size
             _lowerCamelCase			:						Any            =feature_extractor(audio_target=UpperCamelCase__     ,  padding=UpperCamelCase__     ,  return_tensors='np'					).input_values
             self.assertTrue(input_values.ndim == 3					)
             self.assertTrue(input_values.shape[-1] == feature_extractor.num_mel_bins					)

             # Test not batched input
             _lowerCamelCase			:						str            =feature_extractor(speech_inputs[0]     ,  return_tensors='np'					).input_values
             _lowerCamelCase			:						Dict            =feature_extractor(np_speech_inputs[0]     ,  return_tensors='np'					).input_values
             self.assertTrue(np.allclose(UpperCamelCase__     ,  UpperCamelCase__     ,  atol=1E-3					)					)

             # Test batched
             _lowerCamelCase			:						Optional[Any]            =feature_extractor(UpperCamelCase__     ,  return_tensors='np'					).input_values
             _lowerCamelCase			:						Any            =feature_extractor(UpperCamelCase__     ,  return_tensors='np'					).input_values
             for enc_seq_a, enc_seq_a in zip(UpperCamelCase__     ,  UpperCamelCase__					):
                    self.assertTrue(np.allclose(UpperCamelCase__     ,  UpperCamelCase__     ,  atol=1E-3					)					)

             # Test 2-D numpy arrays are batched.
             _lowerCamelCase			:						str            =[floats_list((1, x)					)[0] for x in (800, 800, 800)]
             _lowerCamelCase			:						Tuple            =np.asarray(UpperCamelCase__					)
             _lowerCamelCase			:						Optional[Any]            =feature_extractor(UpperCamelCase__     ,  return_tensors='np'					).input_values
             _lowerCamelCase			:						Any            =feature_extractor(UpperCamelCase__     ,  return_tensors='np'					).input_values
             for enc_seq_a, enc_seq_a in zip(UpperCamelCase__     ,  UpperCamelCase__					):
                    self.assertTrue(np.allclose(UpperCamelCase__     ,  UpperCamelCase__     ,  atol=1E-3					)					)






      def 			lowerCamelCase			(   self		:	List[str]					)   ->		Optional[Any]:
             """simple docstring"""




             _lowerCamelCase			:						List[Any]            =self.feat_extract_tester.prepare_inputs_for_target()
             _lowerCamelCase			:						Dict            =self.feature_extraction_class(**self.feat_extract_dict					)
             _lowerCamelCase			:						List[str]            =feat_extract.model_input_names[0]

             _lowerCamelCase			:						Dict            =BatchFeature({input_name: speech_inputs}					)

             self.assertTrue(all(len(UpperCamelCase__					) == len(UpperCamelCase__					) for x, y in zip(UpperCamelCase__     ,  processed_features[input_name]					)					)					)

             _lowerCamelCase			:						List[str]            =self.feat_extract_tester.prepare_inputs_for_target(equal_length=UpperCamelCase__					)
             _lowerCamelCase			:						Dict            =BatchFeature({input_name: speech_inputs}     ,  tensor_type='np'					)

             _lowerCamelCase			:						Any            =processed_features[input_name]

             if len(batch_features_input.shape					) < 3:
                    _lowerCamelCase			:						str            =batch_features_input[:, :, None]

             self.assertTrue(
                 batch_features_input.shape
                 == (self.feat_extract_tester.batch_size, len(speech_inputs[0]					), self.feat_extract_tester.num_mel_bins)					)






      @require_torch
      def 			lowerCamelCase			(   self		:	List[str]					)   ->		Tuple:
             """simple docstring"""




             _lowerCamelCase			:						Dict            =self.feat_extract_tester.prepare_inputs_for_target(equal_length=UpperCamelCase__					)
             _lowerCamelCase			:						Optional[Any]            =self.feature_extraction_class(**self.feat_extract_dict					)
             _lowerCamelCase			:						List[str]            =feat_extract.model_input_names[0]

             _lowerCamelCase			:						List[str]            =BatchFeature({input_name: speech_inputs}     ,  tensor_type='pt'					)

             _lowerCamelCase			:						Optional[int]            =processed_features[input_name]

             if len(batch_features_input.shape					) < 3:
                    _lowerCamelCase			:						Tuple            =batch_features_input[:, :, None]

             self.assertTrue(
                 batch_features_input.shape
                 == (self.feat_extract_tester.batch_size, len(speech_inputs[0]					), self.feat_extract_tester.num_mel_bins)					)






      @require_torch
      def 			lowerCamelCase			(   self		:	Tuple					)   ->		str:
             """simple docstring"""




             _lowerCamelCase			:						Any            =self.feature_extraction_class(**self.feat_extract_dict					)
             _lowerCamelCase			:						Tuple            =self.feat_extract_tester.prepare_inputs_for_target()
             _lowerCamelCase			:						Any            =feat_extract.model_input_names[0]

             _lowerCamelCase			:						Optional[Any]            =BatchFeature({input_name: speech_inputs}					)

             _lowerCamelCase			:						int            =feat_extract.num_mel_bins  # hack!

             _lowerCamelCase			:						Tuple            =feat_extract.pad(UpperCamelCase__     ,  padding='longest'     ,  return_tensors='np'					)[input_name]
             _lowerCamelCase			:						Dict            =feat_extract.pad(UpperCamelCase__     ,  padding='longest'     ,  return_tensors='pt'					)[input_name]

             self.assertTrue(abs(input_np.astype(np.floataa					).sum() - input_pt.numpy().astype(np.floataa					).sum()					) < 1E-2					)






      def 			lowerCamelCase			(   self		:	str					)   ->		str:
             """simple docstring"""




             _lowerCamelCase			:						Union[str, Any]            =self.feat_extract_dict
             _lowerCamelCase			:						Optional[Any]            =True
             _lowerCamelCase			:						List[str]            =self.feature_extraction_class(**UpperCamelCase__					)
             _lowerCamelCase			:						Union[str, Any]            =self.feat_extract_tester.prepare_inputs_for_target()
             _lowerCamelCase			:						Dict            =[len(UpperCamelCase__					) for x in speech_inputs]
             _lowerCamelCase			:						List[Any]            =feat_extract.model_input_names[0]

             _lowerCamelCase			:						Dict            =BatchFeature({input_name: speech_inputs}					)

             _lowerCamelCase			:						Tuple            =feat_extract.num_mel_bins  # hack!

             _lowerCamelCase			:						int            =feat_extract.pad(UpperCamelCase__     ,  padding='longest'     ,  return_tensors='np'					)
             self.assertIn('attention_mask'     ,  UpperCamelCase__					)
             self.assertListEqual(list(processed.attention_mask.shape					)     ,  list(processed[input_name].shape[:2]					)					)
             self.assertListEqual(processed.attention_mask.sum(-1					).tolist()     ,  UpperCamelCase__					)






      def 			lowerCamelCase			(   self		:	int					)   ->		List[str]:
             """simple docstring"""




             _lowerCamelCase			:						Dict            =self.feat_extract_dict
             _lowerCamelCase			:						List[str]            =True
             _lowerCamelCase			:						int            =self.feature_extraction_class(**UpperCamelCase__					)
             _lowerCamelCase			:						int            =self.feat_extract_tester.prepare_inputs_for_target()
             _lowerCamelCase			:						List[str]            =[len(UpperCamelCase__					) for x in speech_inputs]
             _lowerCamelCase			:						int            =feat_extract.model_input_names[0]

             _lowerCamelCase			:						Optional[Any]            =BatchFeature({input_name: speech_inputs}					)
             _lowerCamelCase			:						Optional[int]            =min(UpperCamelCase__					)

             _lowerCamelCase			:						Any            =feat_extract.num_mel_bins  # hack!

             _lowerCamelCase			:						int            =feat_extract.pad(
                 UpperCamelCase__     ,  padding='max_length'     ,  max_length=UpperCamelCase__     ,  truncation=UpperCamelCase__     ,  return_tensors='np'					)
             self.assertIn('attention_mask'     ,  UpperCamelCase__					)
             self.assertListEqual(
                 list(processed_pad.attention_mask.shape					)     ,  [processed_pad[input_name].shape[0], max_length]					)
             self.assertListEqual(
                 processed_pad.attention_mask[:, :max_length].sum(-1					).tolist()     ,  [max_length for x in speech_inputs]					)






      def 			lowerCamelCase			(   self		:	Any     ,  lowercase_		:	Tuple					)   ->		Tuple:
             """simple docstring"""




             from datasets import load_dataset

             _lowerCamelCase			:						List[Any]            =load_dataset('hf-internal-testing/librispeech_asr_dummy'     ,  'clean'     ,  split='validation'					)
             # automatic decoding with librispeech
             _lowerCamelCase			:						int            =ds.sort('id'					).select(range(UpperCamelCase__					)					)[:num_samples]['audio']

             return [x["array"] for x in speech_samples]






      def 			lowerCamelCase			(   self		:	Optional[Any]					)   ->		List[str]:
             """simple docstring"""




             _lowerCamelCase			:						Any            =torch.tensor(
                 [2.3804E-03, 2.0752E-03, 1.9836E-03, 2.1057E-03, 1.6174E-03,
                  3.0518E-04, 9.1553E-05, 3.3569E-04, 9.7656E-04, 1.8311E-03,
                  2.0142E-03, 2.1057E-03, 1.7395E-03, 4.5776E-04, -3.9673E-04,
                  4.5776E-04, 1.0071E-03, 9.1553E-05, 4.8828E-04, 1.1597E-03,
                  7.3242E-04, 9.4604E-04, 1.8005E-03, 1.8311E-03, 8.8501E-04,
                  4.2725E-04, 4.8828E-04, 7.3242E-04, 1.0986E-03, 2.1057E-03]					)
             # fmt: on

             _lowerCamelCase			:						int            =self._load_datasamples(1					)
             _lowerCamelCase			:						List[str]            =SpeechTaFeatureExtractor()
             _lowerCamelCase			:						Optional[int]            =feature_extractor(UpperCamelCase__     ,  return_tensors='pt'					).input_values
             self.assertEquals(input_values.shape     ,  (1, 9_3680)					)
             self.assertTrue(torch.allclose(input_values[0, :30]     ,  UpperCamelCase__     ,  atol=1E-6					)					)






      def 			lowerCamelCase			(   self		:	Tuple					)   ->		Tuple:
             """simple docstring"""




             _lowerCamelCase			:						str            =torch.tensor(
                 [-2.6870, -3.0104, -3.1356, -3.5352, -3.0044, -3.0353, -3.4719, -3.6777,
                  -3.1520, -2.9435, -2.6553, -2.8795, -2.9944, -2.5921, -3.0279, -3.0386,
                  -3.0864, -3.1291, -3.2353, -2.7444, -2.6831, -2.7287, -3.1761, -3.1571,
                  -3.2726, -3.0582, -3.1007, -3.4533, -3.4695, -3.0998]					)
             # fmt: on

             _lowerCamelCase			:						List[str]            =self._load_datasamples(1					)
             _lowerCamelCase			:						Tuple            =SpeechTaFeatureExtractor()
             _lowerCamelCase			:						List[Any]            =feature_extractor(audio_target=UpperCamelCase__     ,  return_tensors='pt'					).input_values
             self.assertEquals(input_values.shape     ,  (1, 366, 80)					)
             self.assertTrue(torch.allclose(input_values[0, 0, :30]     ,  UpperCamelCase__     ,  atol=1E-4					)					)




import collections
from typing import List, Optional, Union

from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, add_end_docstrings, add_start_docstrings, logging
from ..bert.tokenization_bert_fast import BertTokenizerFast
from .tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer, DPRReaderTokenizer


__lowerCAmelCase     :	Optional[Any]       			=  logging.get_logger(__name__)

__lowerCAmelCase     :	List[str]       			=  {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__lowerCAmelCase     :	int       			=  {
    'vocab_file': {
        'facebook/dpr-ctx_encoder-single-nq-base': (
            'https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/vocab.txt'
        ),
        'facebook/dpr-ctx_encoder-multiset-base': (
            'https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'facebook/dpr-ctx_encoder-single-nq-base': (
            'https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/tokenizer.json'
        ),
        'facebook/dpr-ctx_encoder-multiset-base': (
            'https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/tokenizer.json'
        ),
    },
}
__lowerCAmelCase     :	Any       			=  {
    'vocab_file': {
        'facebook/dpr-question_encoder-single-nq-base': (
            'https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/vocab.txt'
        ),
        'facebook/dpr-question_encoder-multiset-base': (
            'https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'facebook/dpr-question_encoder-single-nq-base': (
            'https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/tokenizer.json'
        ),
        'facebook/dpr-question_encoder-multiset-base': (
            'https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/tokenizer.json'
        ),
    },
}
__lowerCAmelCase     :	List[Any]       			=  {
    'vocab_file': {
        'facebook/dpr-reader-single-nq-base': (
            'https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/vocab.txt'
        ),
        'facebook/dpr-reader-multiset-base': (
            'https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'facebook/dpr-reader-single-nq-base': (
            'https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/tokenizer.json'
        ),
        'facebook/dpr-reader-multiset-base': (
            'https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/tokenizer.json'
        ),
    },
}

__lowerCAmelCase     :	Any       			=  {
    'facebook/dpr-ctx_encoder-single-nq-base': 512,
    'facebook/dpr-ctx_encoder-multiset-base': 512,
}
__lowerCAmelCase     :	Optional[int]       			=  {
    'facebook/dpr-question_encoder-single-nq-base': 512,
    'facebook/dpr-question_encoder-multiset-base': 512,
}
__lowerCAmelCase     :	Optional[int]       			=  {
    'facebook/dpr-reader-single-nq-base': 512,
    'facebook/dpr-reader-multiset-base': 512,
}


__lowerCAmelCase     :	List[str]       			=  {
    'facebook/dpr-ctx_encoder-single-nq-base': {'do_lower_case': True},
    'facebook/dpr-ctx_encoder-multiset-base': {'do_lower_case': True},
}
__lowerCAmelCase     :	str       			=  {
    'facebook/dpr-question_encoder-single-nq-base': {'do_lower_case': True},
    'facebook/dpr-question_encoder-multiset-base': {'do_lower_case': True},
}
__lowerCAmelCase     :	Dict       			=  {
    'facebook/dpr-reader-single-nq-base': {'do_lower_case': True},
    'facebook/dpr-reader-multiset-base': {'do_lower_case': True},
}




class 			UpperCAmelCase_     (					_A	):


  '''simple docstring'''







  a__       			=						VOCAB_FILES_NAMES
  a__       			=						CONTEXT_ENCODER_PRETRAINED_VOCAB_FILES_MAP
  a__       			=						CONTEXT_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a__       			=						CONTEXT_ENCODER_PRETRAINED_INIT_CONFIGURATION
  a__       			=						DPRContextEncoderTokenizer







class 			UpperCAmelCase_     (					_A	):


  '''simple docstring'''







  a__       			=						VOCAB_FILES_NAMES
  a__       			=						QUESTION_ENCODER_PRETRAINED_VOCAB_FILES_MAP
  a__       			=						QUESTION_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a__       			=						QUESTION_ENCODER_PRETRAINED_INIT_CONFIGURATION
  a__       			=						DPRQuestionEncoderTokenizer







__lowerCAmelCase     :	Optional[int]       			=  collections.namedtuple(
    'DPRSpanPrediction', ['span_score', 'relevance_score', 'doc_id', 'start_index', 'end_index', 'text']
)

__lowerCAmelCase     :	Union[str, Any]       			=  collections.namedtuple('DPRReaderOutput', ['start_logits', 'end_logits', 'relevance_logits'])


__lowerCAmelCase     :	str       			=  R'\n    Return a dictionary with the token ids of the input strings and other information to give to `.decode_best_spans`.\n    It converts the strings of a question and different passages (title and text) in a sequence of IDs (integers),\n    using the tokenizer and vocabulary. The resulting `input_ids` is a matrix of size `(n_passages, sequence_length)`\n    with the format:\n\n    [CLS] <question token ids> [SEP] <titles ids> [SEP] <texts ids>\n\n    Args:\n        questions (`str` or `List[str]`):\n            The questions to be encoded. You can specify one question for many passages. In this case, the question\n            will be duplicated like `[questions] * n_passages`. Otherwise you have to specify as many questions as in\n            `titles` or `texts`.\n        titles (`str` or `List[str]`):\n            The passages titles to be encoded. This can be a string or a list of strings if there are several passages.\n        texts (`str` or `List[str]`):\n            The passages texts to be encoded. This can be a string or a list of strings if there are several passages.\n        padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):\n            Activates and controls padding. Accepts the following values:\n\n            - `True` or `\'longest\'`: Pad to the longest sequence in the batch (or no padding if only a single sequence\n              if provided).\n            - `\'max_length\'`: Pad to a maximum length specified with the argument `max_length` or to the maximum\n              acceptable input length for the model if that argument is not provided.\n            - `False` or `\'do_not_pad\'` (default): No padding (i.e., can output a batch with sequences of different\n              lengths).\n        truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):\n            Activates and controls truncation. Accepts the following values:\n\n            - `True` or `\'longest_first\'`: Truncate to a maximum length specified with the argument `max_length` or to\n              the maximum acceptable input length for the model if that argument is not provided. This will truncate\n              token by token, removing a token from the longest sequence in the pair if a pair of sequences (or a batch\n              of pairs) is provided.\n            - `\'only_first\'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum\n              acceptable input length for the model if that argument is not provided. This will only truncate the first\n              sequence of a pair if a pair of sequences (or a batch of pairs) is provided.\n            - `\'only_second\'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum\n              acceptable input length for the model if that argument is not provided. This will only truncate the\n              second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.\n            - `False` or `\'do_not_truncate\'` (default): No truncation (i.e., can output batch with sequence lengths\n              greater than the model maximum admissible input size).\n        max_length (`int`, *optional*):\n                Controls the maximum length to use by one of the truncation/padding parameters.\n\n                If left unset or set to `None`, this will use the predefined model maximum length if a maximum length\n                is required by one of the truncation/padding parameters. If the model has no specific maximum input\n                length (like XLNet) truncation/padding to a maximum length will be deactivated.\n        return_tensors (`str` or [`~utils.TensorType`], *optional*):\n                If set, will return tensors instead of list of python integers. Acceptable values are:\n\n                - `\'tf\'`: Return TensorFlow `tf.constant` objects.\n                - `\'pt\'`: Return PyTorch `torch.Tensor` objects.\n                - `\'np\'`: Return Numpy `np.ndarray` objects.\n        return_attention_mask (`bool`, *optional*):\n            Whether or not to return the attention mask. If not set, will return the attention mask according to the\n            specific tokenizer\'s default, defined by the `return_outputs` attribute.\n\n            [What are attention masks?](../glossary#attention-mask)\n\n    Return:\n        `Dict[str, List[List[int]]]`: A dictionary with the following keys:\n\n        - `input_ids`: List of token ids to be fed to a model.\n        - `attention_mask`: List of indices specifying which tokens should be attended to by the model.\n    '




@add_start_docstrings(_A	)
class 			UpperCAmelCase_     :


  '''simple docstring'''







  def __call__(					self     :					Tuple     ,				UpperCamelCase__     :					List[str]     ,				UpperCamelCase__     :					Optional[str] = None     ,				UpperCamelCase__     :					Optional[str] = None     ,				UpperCamelCase__     :					Union[bool, str] = False     ,				UpperCamelCase__     :					Union[bool, str] = False     ,				UpperCamelCase__     :					Optional[int] = None     ,				UpperCamelCase__     :					Optional[Union[str, TensorType]] = None     ,				UpperCamelCase__     :					Optional[bool] = None     ,				**UpperCamelCase__     :					List[str]     ,				)     ->      BatchEncoding:







    """simple docstring"""






    if titles is None and texts is None:
      return super().__call__(
          UpperCamelCase__     ,				padding=UpperCamelCase__     ,				truncation=UpperCamelCase__     ,				max_length=UpperCamelCase__     ,				return_tensors=UpperCamelCase__     ,				return_attention_mask=UpperCamelCase__     ,				**UpperCamelCase__     ,				)
    elif titles is None or texts is None:
      __magic_name__								=			titles if texts is None else texts
      return super().__call__(
          UpperCamelCase__     ,				UpperCamelCase__     ,				padding=UpperCamelCase__     ,				truncation=UpperCamelCase__     ,				max_length=UpperCamelCase__     ,				return_tensors=UpperCamelCase__     ,				return_attention_mask=UpperCamelCase__     ,				**UpperCamelCase__     ,				)
    __magic_name__								=			titles if not isinstance(UpperCamelCase__     ,				UpperCamelCase__       ) else [titles]
    __magic_name__								=			texts if not isinstance(UpperCamelCase__     ,				UpperCamelCase__       ) else [texts]
    __magic_name__								=			len(UpperCamelCase__       )
    __magic_name__								=			questions if not isinstance(UpperCamelCase__     ,				UpperCamelCase__       ) else [questions] * n_passages
    assert len(UpperCamelCase__       ) == len(
        UpperCamelCase__       ), F'''There should be as many titles than texts but got {len(UpperCamelCase__       )} titles and {len(UpperCamelCase__       )} texts.'''
    __magic_name__								=			super().__call__(UpperCamelCase__     ,				UpperCamelCase__     ,				padding=UpperCamelCase__     ,				truncation=UpperCamelCase__       )["""input_ids"""]
    __magic_name__								=			super().__call__(UpperCamelCase__     ,				add_special_tokens=UpperCamelCase__     ,				padding=UpperCamelCase__     ,				truncation=UpperCamelCase__       )["""input_ids"""]
    __magic_name__								=			{
        """input_ids""": [
            (encoded_question_and_title + encoded_text)[:max_length]
            if max_length is not None and truncation
            else encoded_question_and_title + encoded_text
            for encoded_question_and_title, encoded_text in zip(UpperCamelCase__     ,				UpperCamelCase__       )
        ]
    }
    if return_attention_mask is not False:
      __magic_name__								=			[]
      for input_ids in encoded_inputs["input_ids"]:
        attention_mask.append([int(input_id != self.pad_token_id       ) for input_id in input_ids]       )
      __magic_name__								=			attention_mask
    return self.pad(UpperCamelCase__     ,				padding=UpperCamelCase__     ,				max_length=UpperCamelCase__     ,				return_tensors=UpperCamelCase__       )
  def 				_lowercase				(					self     :					List[Any]     ,				UpperCamelCase__     :					BatchEncoding     ,				UpperCamelCase__     :					DPRReaderOutput     ,				UpperCamelCase__     :					int = 16     ,				UpperCamelCase__     :					int = 64     ,				UpperCamelCase__     :					int = 4     ,				)     ->      List[DPRSpanPrediction]:







    """simple docstring"""






    __magic_name__								=			reader_input["""input_ids"""]
    __magic_name__	,       __magic_name__	,       __magic_name__								=			reader_output[:3]
    __magic_name__								=			len(UpperCamelCase__       )
    __magic_name__								=			sorted(range(UpperCamelCase__       )     ,				reverse=UpperCamelCase__     ,				key=relevance_logits.__getitem__       )
    __magic_name__								=			[]
    for doc_id in sorted_docs:
      __magic_name__								=			list(input_ids[doc_id]       )
      # assuming question & title information is at the beginning of the sequence
      __magic_name__								=			sequence_ids.index(self.sep_token_id     ,				2       ) + 1  # second sep id
      if sequence_ids[-1] == self.pad_token_id:
        __magic_name__								=			sequence_ids.index(self.pad_token_id       )
      else:
        __magic_name__								=			len(UpperCamelCase__       )

      __magic_name__								=			self._get_best_spans(
          start_logits=start_logits[doc_id][passage_offset:sequence_len]     ,				end_logits=end_logits[doc_id][passage_offset:sequence_len]     ,				max_answer_length=UpperCamelCase__     ,				top_spans=UpperCamelCase__     ,				)
      for start_index, end_index in best_spans:
        start_index += passage_offset
        end_index += passage_offset
        nbest_spans_predictions.append(
            DPRSpanPrediction(
                span_score=start_logits[doc_id][start_index] + end_logits[doc_id][end_index]     ,				relevance_score=relevance_logits[doc_id]     ,				doc_id=UpperCamelCase__     ,				start_index=UpperCamelCase__     ,				end_index=UpperCamelCase__     ,				text=self.decode(sequence_ids[start_index : end_index + 1]       )     ,				)       )
      if len(UpperCamelCase__       ) >= num_spans:
        break
    return nbest_spans_predictions[:num_spans]





  def 				_lowercase				(					self     :					Optional[int]     ,				UpperCamelCase__     :					List[int]     ,				UpperCamelCase__     :					List[int]     ,				UpperCamelCase__     :					int     ,				UpperCamelCase__     :					int     ,				)     ->      List[DPRSpanPrediction]:







    """simple docstring"""






    __magic_name__								=			[]
    for start_index, start_score in enumerate(UpperCamelCase__       ):
      for answer_length, end_score in enumerate(end_logits[start_index : start_index + max_answer_length]       ):
        scores.append(((start_index, start_index + answer_length), start_score + end_score)       )
    __magic_name__								=			sorted(UpperCamelCase__     ,				key=lambda UpperCamelCase__       : x[1]     ,				reverse=UpperCamelCase__       )
    __magic_name__								=			[]
    for (start_index, end_index), score in scores:
      assert start_index <= end_index, F'''Wrong span indices: [{start_index}:{end_index}]'''
      __magic_name__								=			end_index - start_index + 1
      assert length <= max_answer_length, F'''Span is too long: {length} > {max_answer_length}'''
      if any(
          start_index <= prev_start_index <= prev_end_index <= end_index
          or prev_start_index <= start_index <= end_index <= prev_end_index
          for (prev_start_index, prev_end_index) in chosen_span_intervals       ):
        continue
      chosen_span_intervals.append((start_index, end_index)       )

      if len(UpperCamelCase__       ) == top_spans:
        break
    return chosen_span_intervals







@add_end_docstrings(_A	)
class 			UpperCAmelCase_     (					_A    ,			_A	):


  '''simple docstring'''







  a__       			=						VOCAB_FILES_NAMES
  a__       			=						READER_PRETRAINED_VOCAB_FILES_MAP
  a__       			=						READER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a__       			=						READER_PRETRAINED_INIT_CONFIGURATION
  a__       			=						["""input_ids""", """attention_mask"""]
  a__       			=						DPRReaderTokenizer

