



import itertools
import random
import unittest

import numpy as np

from transformers import ASTFeatureExtractor
from transformers.testing_utils import require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


_lowercase		:				str      		=random.Random()

if is_torch_available():
		import torch

def      lowerCAmelCase_					(  _lowercase    :       Optional[int]						,       _lowercase    :       Any=1.0						,       _lowercase    :       List[Any]=None						,       _lowercase    :       Tuple=None)							->	Tuple:







			"""simple docstring"""
			if rng is None:
						a__       :						List[Any]							=  global_rng

			a__       :						Optional[Any]							=  []
			for batch_idx in range(shape[0]):
						values.append([])
						for _ in range(shape[1]):
									values[-1].append(rng.random() * scale)

			return values

class        snake_case__    (unittest.TestCase		):







						"""simple docstring"""




						def __init__(       self ,       __lowercase ,       __lowercase=7 ,       __lowercase=4_0_0 ,       __lowercase=2_0_0_0 ,       __lowercase=1 ,       __lowercase=0.0 ,       __lowercase=1_6_0_0_0 ,       __lowercase=True ,       __lowercase=True ,       ) ->				int:
									"""simple docstring"""




									a__       :						List[Any]							=  parent
									a__       :						List[str]							=  batch_size
									a__       :						List[Any]							=  min_seq_length
									a__       :						Optional[int]							=  max_seq_length
									a__       :						str							=  (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
									a__       :						Any							=  feature_size
									a__       :						Optional[Any]							=  padding_value
									a__       :						int							=  sampling_rate
									a__       :						int							=  return_attention_mask
									a__       :						int							=  do_normalize

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				int:
									"""simple docstring"""




									return {
									    "feature_size": self.feature_size,
									    "padding_value": self.padding_value,
									    "sampling_rate": self.sampling_rate,
									    "return_attention_mask": self.return_attention_mask,
									    "do_normalize": self.do_normalize,
									}

						def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase=False ,       __lowercase=False  ) ->				Union[str, Any]:
									"""simple docstring"""




									def _flatten(__lowercase  ):
												return list(itertools.chain(*__lowercase  )  )

									if equal_length:
												a__       :						Tuple							=  floats_list((self.batch_size, self.max_seq_length)  )
									else:
												# make sure that inputs increase in size
												a__       :						Union[str, Any]							=  [
												    _flatten(floats_list((x, self.feature_size)  )  )
												    for x in range(self.min_seq_length ,       self.max_seq_length ,       self.seq_length_diff  )
												]

									if numpify:
												a__       :						Union[str, Any]							=  [np.asarray(__lowercase  ) for x in speech_inputs]

									return speech_inputs




@require_torch
@require_torchaudio
class        snake_case__    (A__ ,     unittest.TestCase		):







						"""simple docstring"""




						__lowerCAmelCase					:List[Any]      =					ASTFeatureExtractor

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Any:
									"""simple docstring"""




									a__       :						Union[str, Any]							=  ASTFeatureExtractionTester(self  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Dict:
									"""simple docstring"""




									a__       :						Tuple							=  self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()  )
									# create three inputs of length 800, 1000, and 1200
									a__       :						Optional[Any]							=  [floats_list((1, x)  )[0] for x in range(8_0_0 ,       1_4_0_0 ,       2_0_0  )]
									a__       :						Union[str, Any]							=  [np.asarray(__lowercase  ) for speech_input in speech_inputs]

									# Test not batched input
									a__       :						Union[str, Any]							=  feat_extract(speech_inputs[0] ,       return_tensors="""np"""  ).input_values
									a__       :						List[str]							=  feat_extract(np_speech_inputs[0] ,       return_tensors="""np"""  ).input_values
									self.assertTrue(np.allclose(__lowercase ,       __lowercase ,       atol=1E-3  )  )

									# Test batched
									a__       :						int							=  feat_extract(__lowercase ,       padding=__lowercase ,       return_tensors="""np"""  ).input_values
									a__       :						Optional[int]							=  feat_extract(__lowercase ,       padding=__lowercase ,       return_tensors="""np"""  ).input_values
									for enc_seq_a, enc_seq_a in zip(__lowercase ,       __lowercase  ):
												self.assertTrue(np.allclose(__lowercase ,       __lowercase ,       atol=1E-3  )  )

									# Test 2-D numpy arrays are batched.
									a__       :						Union[str, Any]							=  [floats_list((1, x)  )[0] for x in (8_0_0, 8_0_0, 8_0_0)]
									a__       :						int							=  np.asarray(__lowercase  )
									a__       :						Optional[Any]							=  feat_extract(__lowercase ,       return_tensors="""np"""  ).input_values
									a__       :						Optional[int]							=  feat_extract(__lowercase ,       return_tensors="""np"""  ).input_values
									for enc_seq_a, enc_seq_a in zip(__lowercase ,       __lowercase  ):
												self.assertTrue(np.allclose(__lowercase ,       __lowercase ,       atol=1E-3  )  )

						@require_torch
						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Optional[int]:
									"""simple docstring"""




									import torch

									a__       :						int							=  self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()  )
									a__       :						Any							=  np.random.rand(1_0_0  ).astype(np.floataa  )
									a__       :						Any							=  np_speech_inputs.tolist()

									for inputs in [py_speech_inputs, np_speech_inputs]:
												a__       :						int							=  feature_extractor.pad([{"""input_values""": inputs}] ,       return_tensors="""np"""  )
												self.assertTrue(np_processed.input_values.dtype == np.floataa  )
												a__       :						List[Any]							=  feature_extractor.pad([{"""input_values""": inputs}] ,       return_tensors="""pt"""  )
												self.assertTrue(pt_processed.input_values.dtype == torch.floataa  )

						def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase  ) ->				Optional[int]:
									"""simple docstring"""




									from datasets import load_dataset

									a__       :						List[Any]							=  load_dataset("""hf-internal-testing/librispeech_asr_dummy""" ,       """clean""" ,       split="""validation"""  )
									# automatic decoding with librispeech
									a__       :						Union[str, Any]							=  ds.sort("""id"""  ).select(range(__lowercase  )  )[:num_samples]["""audio"""]

									return [x["array"] for x in speech_samples]

						@require_torch
						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Union[str, Any]:
									"""simple docstring"""




									a__       :						Optional[int]							=  torch.tensor(
									    [-0.9_8_9_4, -1.2_7_7_6, -0.9_0_6_6, -1.2_7_7_6, -0.9_3_4_9, -1.2_6_0_9, -1.0_3_8_6, -1.2_7_7_6,
									     -1.1_5_6_1, -1.2_7_7_6, -1.2_0_5_2, -1.2_7_2_3, -1.2_1_9_0, -1.2_1_3_2, -1.2_7_7_6, -1.1_1_3_3,
									     -1.1_9_5_3, -1.1_3_4_3, -1.1_5_8_4, -1.2_2_0_3, -1.1_7_7_0, -1.2_4_7_4, -1.2_3_8_1, -1.1_9_3_6,
									     -0.9_2_7_0, -0.8_3_1_7, -0.8_0_4_9, -0.7_7_0_6, -0.7_5_6_5, -0.7_8_6_9]  )
									# fmt: on

									a__       :						Any							=  self._load_datasamples(1  )
									a__       :						Tuple							=  ASTFeatureExtractor()
									a__       :						int							=  feature_extractor(__lowercase ,       return_tensors="""pt"""  ).input_values
									self.assertEquals(input_values.shape ,       (1, 1_0_2_4, 1_2_8)  )
									self.assertTrue(torch.allclose(input_values[0, 0, :3_0] ,       __lowercase ,       atol=1E-4  )  )









import numpy as np
import torch
from torch.utils.data import Dataset

from utils import logger

class        snake_case__    (A__		):







						"""simple docstring"""




						def __init__(       self ,       __lowercase ,       __lowercase  ) ->				int:
									"""simple docstring"""




									a__       :						Tuple							=  params

									a__       :						str							=  np.array(__lowercase  )
									a__       :						List[Any]							=  np.array([len(__lowercase  ) for t in data]  )

									self.check()
									self.remove_long_sequences()
									self.remove_empty_sequences()
									self.remove_unknown_sequences()
									self.check()
									self.print_statistics()

						def __getitem__(       self ,       __lowercase  ) ->				Any:
									"""simple docstring"""




									return (self.token_ids[index], self.lengths[index])

						def __len__(       self  ) ->				Dict:
									"""simple docstring"""




									return len(self.lengths  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Optional[int]:
									"""simple docstring"""




									assert len(self.token_ids  ) == len(self.lengths  )
									assert all(self.lengths[i] == len(self.token_ids[i]  ) for i in range(len(self.lengths  )  )  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Tuple:
									"""simple docstring"""




									a__       :						int							=  self.params.max_model_input_size
									a__       :						int							=  self.lengths > max_len
									logger.info(F'''Splitting {sum(__lowercase  )} too long sequences.'''  )

									def divide_chunks(__lowercase ,       __lowercase  ):
												return [l[i : i + n] for i in range(0 ,       len(__lowercase  ) ,       __lowercase  )]

									a__       :						Any							=  []
									a__       :						Optional[int]							=  []
									if self.params.mlm:
												a__	,						a__       :						Any							=  self.params.special_tok_ids["""cls_token"""], self.params.special_tok_ids["""sep_token"""]
									else:
												a__	,						a__       :						Dict							=  self.params.special_tok_ids["""bos_token"""], self.params.special_tok_ids["""eos_token"""]

									for seq_, len_ in zip(self.token_ids ,       self.lengths  ):
												assert (seq_[0] == cls_id) and (seq_[-1] == sep_id), seq_
												if len_ <= max_len:
															new_tok_ids.append(seq_  )
															new_lengths.append(len_  )
												else:
															a__       :						int							=  []
															for sub_s in divide_chunks(seq_ ,       max_len - 2  ):
																		if sub_s[0] != cls_id:
																					a__       :						str							=  np.insert(__lowercase ,       0 ,       __lowercase  )
																		if sub_s[-1] != sep_id:
																					a__       :						List[str]							=  np.insert(__lowercase ,       len(__lowercase  ) ,       __lowercase  )
																		assert len(__lowercase  ) <= max_len
																		assert (sub_s[0] == cls_id) and (sub_s[-1] == sep_id), sub_s
																		sub_seqs.append(__lowercase  )

															new_tok_ids.extend(__lowercase  )
															new_lengths.extend([len(__lowercase  ) for l in sub_seqs]  )

									a__       :						Optional[int]							=  np.array(__lowercase  )
									a__       :						Any							=  np.array(__lowercase  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Optional[Any]:
									"""simple docstring"""




									a__       :						Union[str, Any]							=  len(self  )
									a__       :						List[str]							=  self.lengths > 1_1
									a__       :						Dict							=  self.token_ids[indices]
									a__       :						List[str]							=  self.lengths[indices]
									a__       :						int							=  len(self  )
									logger.info(F'''Remove {init_size - new_size} too short (<=11 tokens) sequences.'''  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				List[Any]:
									"""simple docstring"""




									if "unk_token" not in self.params.special_tok_ids:
												return
									else:
												a__       :						Union[str, Any]							=  self.params.special_tok_ids["""unk_token"""]
									a__       :						List[Any]							=  len(self  )
									a__       :						Optional[int]							=  np.array([np.count_nonzero(a == unk_token_id  ) for a in self.token_ids]  )
									a__       :						Optional[Any]							=  (unk_occs / self.lengths) < 0.5
									a__       :						Tuple							=  self.token_ids[indices]
									a__       :						Union[str, Any]							=  self.lengths[indices]
									a__       :						Tuple							=  len(self  )
									logger.info(F'''Remove {init_size - new_size} sequences with a high level of unknown tokens (50%).'''  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				str:
									"""simple docstring"""




									if not self.params.is_master:
												return
									logger.info(F'''{len(self  )} sequences'''  )
									# data_len = sum(self.lengths)
									# nb_unique_tokens = len(Counter(list(chain(*self.token_ids))))
									# logger.info(f'{data_len} tokens ({nb_unique_tokens} unique)')

									# unk_idx = self.params.special_tok_ids['unk_token']
									# nb_unknown = sum([(t==unk_idx).sum() for t in self.token_ids])
									# logger.info(f'{nb_unknown} unknown tokens (covering {100*nb_unknown/data_len:.2f}% of the data)')

						def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase  ) ->				Optional[int]:
									"""simple docstring"""




									a__       :						Optional[int]							=  [t[0] for t in batch]
									a__       :						Any							=  [t[1] for t in batch]
									assert len(__lowercase  ) == len(__lowercase  )

									# Max for paddings
									a__       :						List[Any]							=  max(__lowercase  )

									# Pad token ids
									if self.params.mlm:
												a__       :						int							=  self.params.special_tok_ids["""pad_token"""]
									else:
												a__       :						List[str]							=  self.params.special_tok_ids["""unk_token"""]
									a__       :						int							=  [list(t.astype(__lowercase  )  ) + [pad_idx] * (max_seq_len_ - len(__lowercase  )) for t in token_ids]
									assert len(tk_  ) == len(__lowercase  )
									assert all(len(__lowercase  ) == max_seq_len_ for t in tk_  )

									a__       :						List[Any]							=  torch.tensor(tk_  )  # (bs, max_seq_len_)
									a__       :						Optional[int]							=  torch.tensor(__lowercase  )  # (bs)
									return tk_t, lg_t




