





'''simple docstring'''






import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


lowerCAmelCase									=   pytest.mark.integration
@require_faiss
class 		lowerCamelCase	(   _A							):



	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     Dict       						= Dataset.from_dict({"filename": ["my_name-train" + "_" + str(a_						) for x in np.arange(30						).tolist()]}						)
			return dset



	def 	_lowerCamelCase     ( self						):
			import faiss

			lowerCAmelCase  :     Dataset       						= self._create_dummy_dataset()
			lowerCAmelCase  :     Optional[int]       						= dset.map(
			    lambda a_		,   a_						: {"vecs": i * np.ones(5		,   dtype=np.floataa						)}		,   with_indices=a_		,   keep_in_memory=a_						)
			lowerCAmelCase  :     List[str]       						= dset.add_faiss_index("vecs"		,   batch_size=100		,   metric_type=faiss.METRIC_INNER_PRODUCT						)
			lowerCAmelCase						,		lowerCAmelCase  :     str       						= dset.get_nearest_examples("vecs"		,   np.ones(5		,   dtype=np.floataa						)						)
			self.assertEqual(examples["filename"][0]		,   "my_name-train_29"						)
			dset.drop_index("vecs"						)



	def 	_lowerCamelCase     ( self						):
			import faiss

			lowerCAmelCase  :     Dataset       						= self._create_dummy_dataset()
			dset.add_faiss_index_from_external_arrays(
			    external_arrays=np.ones((30, 5)						) * np.arange(30						).reshape(-1		,   1						)		,   index_name="vecs"		,   batch_size=100		,   metric_type=faiss.METRIC_INNER_PRODUCT		,   )
			lowerCAmelCase						,		lowerCAmelCase  :     Tuple       						= dset.get_nearest_examples("vecs"		,   np.ones(5		,   dtype=np.floataa						)						)
			self.assertEqual(examples["filename"][0]		,   "my_name-train_29"						)



	def 	_lowerCamelCase     ( self						):
			import faiss

			lowerCAmelCase  :     Dataset       						= self._create_dummy_dataset()
			dset.add_faiss_index_from_external_arrays(
			    external_arrays=np.ones((30, 5)						) * np.arange(30						).reshape(-1		,   1						)		,   index_name="vecs"		,   metric_type=faiss.METRIC_INNER_PRODUCT		,   )

			# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
			# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
			# see https://bugs.python.org/issue14243 and
			# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
			with tempfile.NamedTemporaryFile(delete=a_						) as tmp_file:
					dset.save_faiss_index("vecs"		,   tmp_file.name						)
					dset.load_faiss_index("vecs2"		,   tmp_file.name						)
			os.unlink(tmp_file.name						)

			lowerCAmelCase						,		lowerCAmelCase  :     Dict       						= dset.get_nearest_examples("vecs2"		,   np.ones(5		,   dtype=np.floataa						)						)
			self.assertEqual(examples["filename"][0]		,   "my_name-train_29"						)



	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     Dataset       						= self._create_dummy_dataset()
			dset.add_faiss_index_from_external_arrays(
			    external_arrays=np.ones((30, 5)						) * np.arange(30						).reshape(-1		,   1						)		,   index_name="vecs"						)
			dset.drop_index("vecs"						)
			self.assertRaises(a_		,   partial(dset.get_nearest_examples		,   "vecs2"		,   np.ones(5		,   dtype=np.floataa						)						)						)





	def 	_lowerCamelCase     ( self						):
			from elasticsearch import Elasticsearch

			lowerCAmelCase  :     Dataset       						= self._create_dummy_dataset()
			with patch("elasticsearch.Elasticsearch.search"						) as mocked_search, patch(
			    "elasticsearch.client.IndicesClient.create"						) as mocked_index_create, patch("elasticsearch.helpers.streaming_bulk"						) as mocked_bulk:
					lowerCAmelCase  :     Tuple       						= {"acknowledged": True}
					mocked_bulk.return_value([(True, None)] * 30						)
					lowerCAmelCase  :     Optional[int]       						= {"hits": {"hits": [{"_score": 1, "_id": 29}]}}
					lowerCAmelCase  :     Optional[int]       						= Elasticsearch()

					dset.add_elasticsearch_index("filename"		,   es_client=a_						)
					lowerCAmelCase						,		lowerCAmelCase  :     List[str]       						= dset.get_nearest_examples("filename"		,   "my_name-train_29"						)
					self.assertEqual(examples["filename"][0]		,   "my_name-train_29"						)






@require_faiss
class 		lowerCamelCase	(   _A							):



	def 	_lowerCamelCase     ( self						):
			import faiss

			lowerCAmelCase  :     Dict       						= FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT						)

			# add vectors
			index.add_vectors(np.eye(5		,   dtype=np.floataa						)						)
			self.assertIsNotNone(index.faiss_index						)
			self.assertEqual(index.faiss_index.ntotal		,   5						)
			index.add_vectors(np.zeros((5, 5)		,   dtype=np.floataa						)						)
			self.assertEqual(index.faiss_index.ntotal		,   10						)

			# single query
			lowerCAmelCase  :     Tuple       						= np.zeros(5		,   dtype=np.floataa						)
			lowerCAmelCase  :     Dict       						= 1
			lowerCAmelCase						,		lowerCAmelCase  :     List[str]       						= index.search(a_						)
			self.assertRaises(a_		,   index.search		,   query.reshape(-1		,   1						)						)
			self.assertGreater(scores[0]		,   0						)
			self.assertEqual(indices[0]		,   1						)

			# batched queries
			lowerCAmelCase  :     Union[str, Any]       						= np.eye(5		,   dtype=np.floataa						)[::-1]
			lowerCAmelCase						,		lowerCAmelCase  :     Optional[int]       						= index.search_batch(a_						)
			self.assertRaises(a_		,   index.search_batch		,   queries[0]						)
			lowerCAmelCase  :     List[Any]       						= [scores[0] for scores in total_scores]
			lowerCAmelCase  :     str       						= [indices[0] for indices in total_indices]
			self.assertGreater(np.min(a_						)		,   0						)
			self.assertListEqual([4, 3, 2, 1, 0]		,   a_						)



	def 	_lowerCamelCase     ( self						):
			import faiss

			lowerCAmelCase  :     Optional[int]       						= FaissIndex(string_factory="Flat"						)
			index.add_vectors(np.eye(5		,   dtype=np.floataa						)						)
			self.assertIsInstance(index.faiss_index		,   faiss.IndexFlat						)
			lowerCAmelCase  :     Tuple       						= FaissIndex(string_factory="LSH"						)
			index.add_vectors(np.eye(5		,   dtype=np.floataa						)						)
			self.assertIsInstance(index.faiss_index		,   faiss.IndexLSH						)
			with self.assertRaises(a_						):
					lowerCAmelCase  :     Tuple       						= FaissIndex(string_factory="Flat"		,   custom_index=faiss.IndexFlat(5						)						)



	def 	_lowerCamelCase     ( self						):
			import faiss

			lowerCAmelCase  :     Optional[Any]       						= faiss.IndexFlat(5						)
			lowerCAmelCase  :     Optional[int]       						= FaissIndex(custom_index=a_						)
			index.add_vectors(np.eye(5		,   dtype=np.floataa						)						)
			self.assertIsInstance(index.faiss_index		,   faiss.IndexFlat						)





	def 	_lowerCamelCase     ( self						):
			import faiss

			lowerCAmelCase  :     str       						= FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT						)
			index.add_vectors(np.eye(5		,   dtype=np.floataa						)						)

			# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
			# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
			# see https://bugs.python.org/issue14243 and
			# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
			with tempfile.NamedTemporaryFile(delete=a_						) as tmp_file:
					index.save(tmp_file.name						)
					lowerCAmelCase  :     int       						= FaissIndex.load(tmp_file.name						)
			os.unlink(tmp_file.name						)

			lowerCAmelCase  :     Any       						= np.zeros(5		,   dtype=np.floataa						)
			lowerCAmelCase  :     Tuple       						= 1
			lowerCAmelCase						,		lowerCAmelCase  :     Dict       						= index.search(a_						)
			self.assertGreater(scores[0]		,   0						)
			self.assertEqual(indices[0]		,   1						)






@require_faiss
def   __A    (				a_    :	Union[str, Any]						):
		import faiss

		lowerCAmelCase  :     Any       						= FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT						)
		index.add_vectors(np.eye(5   ,dtype=np.floataa						)						)

		lowerCAmelCase  :     Tuple       						= "index.faiss"
		lowerCAmelCase  :     Optional[int]       						= f'''mock://{index_name}'''
		index.save(a_   ,storage_options=mockfs.storage_options						)
		lowerCAmelCase  :     List[Any]       						= FaissIndex.load(a_   ,storage_options=mockfs.storage_options						)

		lowerCAmelCase  :     Optional[Any]       						= np.zeros(5   ,dtype=np.floataa						)
		lowerCAmelCase  :     Dict       						= 1
		lowerCAmelCase						,		lowerCAmelCase  :     Optional[int]       						= index.search(a_						)
		assert scores[0] > 0
		assert indices[0] == 1

@require_elasticsearch
class 		lowerCamelCase	(   _A							):



	def 	_lowerCamelCase     ( self						):
			from elasticsearch import Elasticsearch

			with patch("elasticsearch.Elasticsearch.search"						) as mocked_search, patch(
			    "elasticsearch.client.IndicesClient.create"						) as mocked_index_create, patch("elasticsearch.helpers.streaming_bulk"						) as mocked_bulk:
					lowerCAmelCase  :     Union[str, Any]       						= Elasticsearch()
					lowerCAmelCase  :     Optional[int]       						= {"acknowledged": True}
					lowerCAmelCase  :     Tuple       						= ElasticSearchIndex(es_client=a_						)
					mocked_bulk.return_value([(True, None)] * 3						)
					index.add_documents(["foo", "bar", "foobar"]						)

					# single query
					lowerCAmelCase  :     Dict       						= "foo"
					lowerCAmelCase  :     List[str]       						= {"hits": {"hits": [{"_score": 1, "_id": 0}]}}
					lowerCAmelCase						,		lowerCAmelCase  :     Tuple       						= index.search(a_						)
					self.assertEqual(scores[0]		,   1						)
					self.assertEqual(indices[0]		,   0						)

					# single query with timeout
					lowerCAmelCase  :     Union[str, Any]       						= "foo"
					lowerCAmelCase  :     int       						= {"hits": {"hits": [{"_score": 1, "_id": 0}]}}
					lowerCAmelCase						,		lowerCAmelCase  :     Dict       						= index.search(a_		,   request_timeout=30						)
					self.assertEqual(scores[0]		,   1						)
					self.assertEqual(indices[0]		,   0						)

					# batched queries
					lowerCAmelCase  :     Optional[Any]       						= ["foo", "bar", "foobar"]
					lowerCAmelCase  :     Union[str, Any]       						= {"hits": {"hits": [{"_score": 1, "_id": 1}]}}
					lowerCAmelCase						,		lowerCAmelCase  :     List[str]       						= index.search_batch(a_						)
					lowerCAmelCase  :     Any       						= [scores[0] for scores in total_scores]
					lowerCAmelCase  :     Tuple       						= [indices[0] for indices in total_indices]
					self.assertGreater(np.min(a_						)		,   0						)
					self.assertListEqual([1, 1, 1]		,   a_						)

					# batched queries with timeout
					lowerCAmelCase  :     Dict       						= ["foo", "bar", "foobar"]
					lowerCAmelCase  :     Optional[Any]       						= {"hits": {"hits": [{"_score": 1, "_id": 1}]}}
					lowerCAmelCase						,		lowerCAmelCase  :     int       						= index.search_batch(a_		,   request_timeout=30						)
					lowerCAmelCase  :     int       						= [scores[0] for scores in total_scores]
					lowerCAmelCase  :     List[Any]       						= [indices[0] for indices in total_indices]
					self.assertGreater(np.min(a_						)		,   0						)
					self.assertListEqual([1, 1, 1]		,   a_						)










'''simple docstring'''






import itertools
import random
import unittest

import numpy as np

from transformers import WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST, WavaVecaConfig, WavaVecaFeatureExtractor
from transformers.testing_utils import require_torch, slow

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


lowerCAmelCase									=   random.Random()






def   __A    (				a_    :	Union[str, Any]   ,a_    :	Tuple=1.0   ,a_    :	Optional[int]=None   ,a_    :	Union[str, Any]=None						):
		if rng is None:
				lowerCAmelCase  :     str       						= global_rng

		lowerCAmelCase  :     Any       						= []
		for batch_idx in range(shape[0]						):
				values.append([]						)
				for _ in range(shape[1]						):
						values[-1].append(rng.random() * scale						)

		return values
class 		lowerCamelCase	(   unittest.TestCase							):



	def __init__( self		,   a_		,   a_=7		,   a_=400		,   a_=2_000		,   a_=1		,   a_=0.0		,   a_=16_000		,   a_=True		,   a_=True		,   ):
			lowerCAmelCase  :     Dict       						= parent
			lowerCAmelCase  :     Optional[int]       						= batch_size
			lowerCAmelCase  :     Dict       						= min_seq_length
			lowerCAmelCase  :     Dict       						= max_seq_length
			lowerCAmelCase  :     Optional[Any]       						= (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
			lowerCAmelCase  :     Union[str, Any]       						= feature_size
			lowerCAmelCase  :     Tuple       						= padding_value
			lowerCAmelCase  :     Dict       						= sampling_rate
			lowerCAmelCase  :     int       						= return_attention_mask
			lowerCAmelCase  :     Optional[Any]       						= do_normalize



	def 	_lowerCamelCase     ( self						):
			return {
			    "feature_size": self.feature_size,
			    "padding_value": self.padding_value,
			    "sampling_rate": self.sampling_rate,
			    "return_attention_mask": self.return_attention_mask,
			    "do_normalize": self.do_normalize,
			}





	def 	_lowerCamelCase     ( self		,   a_=False		,   a_=False						):
			def _flatten(a_						):
					return list(itertools.chain(*a_						)						)

			if equal_length:
					lowerCAmelCase  :     List[Any]       						= floats_list((self.batch_size, self.max_seq_length)						)
			else:
					# make sure that inputs increase in size
					lowerCAmelCase  :     Any       						= [
					    _flatten(floats_list((x, self.feature_size)						)						)
					    for x in range(self.min_seq_length		,   self.max_seq_length		,   self.seq_length_diff						)
					]

			if numpify:
					lowerCAmelCase  :     List[str]       						= [np.asarray(a_						) for x in speech_inputs]

			return speech_inputs






class 		lowerCamelCase	(   _A   , unittest.TestCase							):
	snake_case_   = WavaVecaFeatureExtractor



	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     Tuple       						= WavaVecaFeatureExtractionTester(self						)



	def 	_lowerCamelCase     ( self		,   a_						):
			self.assertTrue(np.all(np.mean(a_		,   axis=0						) < 1e-3						)						)
			self.assertTrue(np.all(np.abs(np.var(a_		,   axis=0						) - 1						) < 1e-3						)						)



	def 	_lowerCamelCase     ( self						):
			# Tests that all call wrap to encode_plus and batch_encode_plus
			lowerCAmelCase  :     int       						= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
			# create three inputs of length 800, 1000, and 1200
			lowerCAmelCase  :     str       						= [floats_list((1, x)						)[0] for x in range(800		,   1_400		,   200						)]
			lowerCAmelCase  :     Dict       						= [np.asarray(a_						) for speech_input in speech_inputs]

			# Test not batched input
			lowerCAmelCase  :     Optional[int]       						= feat_extract(speech_inputs[0]		,   return_tensors="np"						).input_values
			lowerCAmelCase  :     Any       						= feat_extract(np_speech_inputs[0]		,   return_tensors="np"						).input_values
			self.assertTrue(np.allclose(a_		,   a_		,   atol=1e-3						)						)

			# Test batched
			lowerCAmelCase  :     Any       						= feat_extract(a_		,   return_tensors="np"						).input_values
			lowerCAmelCase  :     int       						= feat_extract(a_		,   return_tensors="np"						).input_values
			for enc_seq_a, enc_seq_a in zip(a_		,   a_						):
					self.assertTrue(np.allclose(a_		,   a_		,   atol=1e-3						)						)

			# Test 2-D numpy arrays are batched.
			lowerCAmelCase  :     Dict       						= [floats_list((1, x)						)[0] for x in (800, 800, 800)]
			lowerCAmelCase  :     Tuple       						= np.asarray(a_						)
			lowerCAmelCase  :     int       						= feat_extract(a_		,   return_tensors="np"						).input_values
			lowerCAmelCase  :     Union[str, Any]       						= feat_extract(a_		,   return_tensors="np"						).input_values
			for enc_seq_a, enc_seq_a in zip(a_		,   a_						):
					self.assertTrue(np.allclose(a_		,   a_		,   atol=1e-3						)						)



	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     int       						= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
			lowerCAmelCase  :     Any       						= [floats_list((1, x)						)[0] for x in range(800		,   1_400		,   200						)]

			lowerCAmelCase  :     Union[str, Any]       						= ["longest", "max_length", "do_not_pad"]
			lowerCAmelCase  :     Optional[int]       						= [None, 1_600, None]
			for max_length, padding in zip(a_		,   a_						):
					lowerCAmelCase  :     List[Any]       						= feat_extract(a_		,   padding=a_		,   max_length=a_		,   return_tensors="np"						)
					lowerCAmelCase  :     Any       						= processed.input_values

					self._check_zero_mean_unit_variance(input_values[0][:800]						)
					self.assertTrue(input_values[0][800:].sum() < 1e-6						)
					self._check_zero_mean_unit_variance(input_values[1][:1_000]						)
					self.assertTrue(input_values[0][1_000:].sum() < 1e-6						)
					self._check_zero_mean_unit_variance(input_values[2][:1_200]						)



	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     List[Any]       						= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
			lowerCAmelCase  :     Optional[int]       						= range(800		,   1_400		,   200						)
			lowerCAmelCase  :     Tuple       						= [floats_list((1, x)						)[0] for x in lengths]

			lowerCAmelCase  :     str       						= ["longest", "max_length", "do_not_pad"]
			lowerCAmelCase  :     Any       						= [None, 1_600, None]

			for max_length, padding in zip(a_		,   a_						):
					lowerCAmelCase  :     Dict       						= feat_extract(a_		,   max_length=a_		,   padding=a_						)
					lowerCAmelCase  :     int       						= processed.input_values

					self._check_zero_mean_unit_variance(input_values[0][:800]						)
					self._check_zero_mean_unit_variance(input_values[1][:1_000]						)
					self._check_zero_mean_unit_variance(input_values[2][:1_200]						)



	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     List[Any]       						= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
			lowerCAmelCase  :     Tuple       						= [floats_list((1, x)						)[0] for x in range(800		,   1_400		,   200						)]
			lowerCAmelCase  :     Dict       						= feat_extract(
			    a_		,   truncation=a_		,   max_length=1_000		,   padding="max_length"		,   return_tensors="np"						)
			lowerCAmelCase  :     Union[str, Any]       						= processed.input_values

			self._check_zero_mean_unit_variance(input_values[0, :800]						)
			self._check_zero_mean_unit_variance(input_values[1]						)
			self._check_zero_mean_unit_variance(input_values[2]						)



	def 	_lowerCamelCase     ( self						):
			lowerCAmelCase  :     Optional[int]       						= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
			lowerCAmelCase  :     Tuple       						= [floats_list((1, x)						)[0] for x in range(800		,   1_400		,   200						)]
			lowerCAmelCase  :     Union[str, Any]       						= feat_extract(
			    a_		,   truncation=a_		,   max_length=1_000		,   padding="longest"		,   return_tensors="np"						)
			lowerCAmelCase  :     Dict       						= processed.input_values

			self._check_zero_mean_unit_variance(input_values[0, :800]						)
			self._check_zero_mean_unit_variance(input_values[1, :1_000]						)
			self._check_zero_mean_unit_variance(input_values[2]						)

			# make sure that if max_length < longest -> then pad to max_length
			self.assertTrue(input_values.shape == (3, 1_000)						)

			lowerCAmelCase  :     List[str]       						= [floats_list((1, x)						)[0] for x in range(800		,   1_400		,   200						)]
			lowerCAmelCase  :     Union[str, Any]       						= feat_extract(
			    a_		,   truncation=a_		,   max_length=2_000		,   padding="longest"		,   return_tensors="np"						)
			lowerCAmelCase  :     Optional[Any]       						= processed.input_values

			self._check_zero_mean_unit_variance(input_values[0, :800]						)
			self._check_zero_mean_unit_variance(input_values[1, :1_000]						)
			self._check_zero_mean_unit_variance(input_values[2]						)

			# make sure that if max_length > longest -> then pad to longest
			self.assertTrue(input_values.shape == (3, 1_200)						)



	@require_torch
	def 	_lowerCamelCase     ( self						):
			import torch

			lowerCAmelCase  :     Optional[int]       						= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
			lowerCAmelCase  :     List[Any]       						= np.random.rand(100						).astype(np.floataa						)
			lowerCAmelCase  :     List[Any]       						= np_speech_inputs.tolist()

			for inputs in [py_speech_inputs, np_speech_inputs]:
					lowerCAmelCase  :     Union[str, Any]       						= feature_extractor.pad([{"input_values": inputs}]		,   return_tensors="np"						)
					self.assertTrue(np_processed.input_values.dtype == np.floataa						)
					lowerCAmelCase  :     Tuple       						= feature_extractor.pad([{"input_values": inputs}]		,   return_tensors="pt"						)
					self.assertTrue(pt_processed.input_values.dtype == torch.floataa						)





	@slow
	@require_torch
	def 	_lowerCamelCase     ( self						):
			# this test makes sure that models that are using
			# group norm don't have their feature extractor return the
			# attention_mask
			for model_id in WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST:
					lowerCAmelCase  :     Any       						= WavaVecaConfig.from_pretrained(a_						)
					lowerCAmelCase  :     List[str]       						= WavaVecaFeatureExtractor.from_pretrained(a_						)

					# only "layer" feature extraction norm should make use of
					# attention_mask
					self.assertEqual(feat_extract.return_attention_mask		,   config.feat_extract_norm == "layer"						)



