





import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


UpperCamelCase__											= pytest.mark.integration

@require_faiss
class        __SCREAMING_SNAKE_CASE					(     _a       ):






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			Dataset.from_dict({"""filename""": ["""my_name-train""" + """_""" + str(__lowerCAmelCase  ) for x in np.arange(30  ).tolist()]}  )
														return dset






							def 				_lowerCamelCase		(						self  ):
														import faiss

														UpperCamelCase__				     =			self._create_dummy_dataset()
														UpperCamelCase__				     =			dset.map(
														    lambda __lowerCAmelCase  , __lowerCAmelCase  : {"vecs": i * np.ones(5  , dtype=np.floataa  )}  , with_indices=__lowerCAmelCase  , keep_in_memory=__lowerCAmelCase  )
														UpperCamelCase__				     =			dset.add_faiss_index("""vecs"""  , batch_size=100  , metric_type=faiss.METRIC_INNER_PRODUCT  )
														UpperCamelCase__     ,     UpperCamelCase__				     =			dset.get_nearest_examples("""vecs"""  , np.ones(5  , dtype=np.floataa  )  )
														self.assertEqual(examples["""filename"""][0]  , """my_name-train_29"""  )
														dset.drop_index("""vecs"""  )






							def 				_lowerCamelCase		(						self  ):
														import faiss

														UpperCamelCase__				     =			self._create_dummy_dataset()
														dset.add_faiss_index_from_external_arrays(
														    external_arrays=np.ones((30, 5)  ) * np.arange(30  ).reshape(-1  , 1  )  , index_name="""vecs"""  , batch_size=100  , metric_type=faiss.METRIC_INNER_PRODUCT  , )
														UpperCamelCase__     ,     UpperCamelCase__				     =			dset.get_nearest_examples("""vecs"""  , np.ones(5  , dtype=np.floataa  )  )
														self.assertEqual(examples["""filename"""][0]  , """my_name-train_29"""  )






							def 				_lowerCamelCase		(						self  ):
														import faiss

														UpperCamelCase__				     =			self._create_dummy_dataset()
														dset.add_faiss_index_from_external_arrays(
														    external_arrays=np.ones((30, 5)  ) * np.arange(30  ).reshape(-1  , 1  )  , index_name="""vecs"""  , metric_type=faiss.METRIC_INNER_PRODUCT  , )

														# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
														# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
														# see https://bugs.python.org/issue14243 and
														# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
														with tempfile.NamedTemporaryFile(delete=__lowerCAmelCase  ) as tmp_file:
																					dset.save_faiss_index("""vecs"""  , tmp_file.name  )
																					dset.load_faiss_index("""vecs2"""  , tmp_file.name  )
														os.unlink(tmp_file.name  )

														UpperCamelCase__     ,     UpperCamelCase__				     =			dset.get_nearest_examples("""vecs2"""  , np.ones(5  , dtype=np.floataa  )  )
														self.assertEqual(examples["""filename"""][0]  , """my_name-train_29"""  )






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			self._create_dummy_dataset()
														dset.add_faiss_index_from_external_arrays(
														    external_arrays=np.ones((30, 5)  ) * np.arange(30  ).reshape(-1  , 1  )  , index_name="""vecs"""  )
														dset.drop_index("""vecs"""  )
														self.assertRaises(__lowerCAmelCase  , partial(dset.get_nearest_examples  , """vecs2"""  , np.ones(5  , dtype=np.floataa  )  )  )







							def 				_lowerCamelCase		(						self  ):
														from elasticsearch import Elasticsearch

														UpperCamelCase__				     =			self._create_dummy_dataset()
														with patch("""elasticsearch.Elasticsearch.search"""  ) as mocked_search, patch(
														    """elasticsearch.client.IndicesClient.create"""  ) as mocked_index_create, patch("""elasticsearch.helpers.streaming_bulk"""  ) as mocked_bulk:
																					UpperCamelCase__				     =			{"""acknowledged""": True}
																					mocked_bulk.return_value([(True, None)] * 30  )
																					UpperCamelCase__				     =			{"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 29}]}}
																					UpperCamelCase__				     =			Elasticsearch()

																					dset.add_elasticsearch_index("""filename"""  , es_client=__lowerCAmelCase  )
																					UpperCamelCase__     ,     UpperCamelCase__				     =			dset.get_nearest_examples("""filename"""  , """my_name-train_29"""  )
																					self.assertEqual(examples["""filename"""][0]  , """my_name-train_29"""  )






@require_faiss
class        __SCREAMING_SNAKE_CASE					(     _a       ):






							def 				_lowerCamelCase		(						self  ):
														import faiss

														UpperCamelCase__				     =			FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT  )

														# add vectors
														index.add_vectors(np.eye(5  , dtype=np.floataa  )  )
														self.assertIsNotNone(index.faiss_index  )
														self.assertEqual(index.faiss_index.ntotal  , 5  )
														index.add_vectors(np.zeros((5, 5)  , dtype=np.floataa  )  )
														self.assertEqual(index.faiss_index.ntotal  , 10  )

														# single query
														UpperCamelCase__				     =			np.zeros(5  , dtype=np.floataa  )
														UpperCamelCase__				     =			1
														UpperCamelCase__     ,     UpperCamelCase__				     =			index.search(__lowerCAmelCase  )
														self.assertRaises(__lowerCAmelCase  , index.search  , query.reshape(-1  , 1  )  )
														self.assertGreater(scores[0]  , 0  )
														self.assertEqual(indices[0]  , 1  )

														# batched queries
														UpperCamelCase__				     =			np.eye(5  , dtype=np.floataa  )[::-1]
														UpperCamelCase__     ,     UpperCamelCase__				     =			index.search_batch(__lowerCAmelCase  )
														self.assertRaises(__lowerCAmelCase  , index.search_batch  , queries[0]  )
														UpperCamelCase__				     =			[scores[0] for scores in total_scores]
														UpperCamelCase__				     =			[indices[0] for indices in total_indices]
														self.assertGreater(np.min(__lowerCAmelCase  )  , 0  )
														self.assertListEqual([4, 3, 2, 1, 0]  , __lowerCAmelCase  )






							def 				_lowerCamelCase		(						self  ):
														import faiss

														UpperCamelCase__				     =			FaissIndex(string_factory="""Flat"""  )
														index.add_vectors(np.eye(5  , dtype=np.floataa  )  )
														self.assertIsInstance(index.faiss_index  , faiss.IndexFlat  )
														UpperCamelCase__				     =			FaissIndex(string_factory="""LSH"""  )
														index.add_vectors(np.eye(5  , dtype=np.floataa  )  )
														self.assertIsInstance(index.faiss_index  , faiss.IndexLSH  )
														with self.assertRaises(__lowerCAmelCase  ):
																					UpperCamelCase__				     =			FaissIndex(string_factory="""Flat"""  , custom_index=faiss.IndexFlat(5  )  )






							def 				_lowerCamelCase		(						self  ):
														import faiss

														UpperCamelCase__				     =			faiss.IndexFlat(5  )
														UpperCamelCase__				     =			FaissIndex(custom_index=__lowerCAmelCase  )
														index.add_vectors(np.eye(5  , dtype=np.floataa  )  )
														self.assertIsInstance(index.faiss_index  , faiss.IndexFlat  )







							def 				_lowerCamelCase		(						self  ):
														import faiss

														UpperCamelCase__				     =			FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT  )
														index.add_vectors(np.eye(5  , dtype=np.floataa  )  )

														# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
														# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
														# see https://bugs.python.org/issue14243 and
														# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
														with tempfile.NamedTemporaryFile(delete=__lowerCAmelCase  ) as tmp_file:
																					index.save(tmp_file.name  )
																					UpperCamelCase__				     =			FaissIndex.load(tmp_file.name  )
														os.unlink(tmp_file.name  )

														UpperCamelCase__				     =			np.zeros(5  , dtype=np.floataa  )
														UpperCamelCase__				     =			1
														UpperCamelCase__     ,     UpperCamelCase__				     =			index.search(__lowerCAmelCase  )
														self.assertGreater(scores[0]  , 0  )
														self.assertEqual(indices[0]  , 1  )






@require_faiss
def     _UpperCamelCase     (a__   :Dict       ):






							"""simple docstring"""





							import faiss

							UpperCamelCase__				     =			FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT       )
							index.add_vectors(np.eye(5   ,			dtype=np.floataa       )       )

							UpperCamelCase__				     =			"""index.faiss"""
							UpperCamelCase__				     =			f"""mock://{index_name}"""
							index.save(a__   ,			storage_options=mockfs.storage_options       )
							UpperCamelCase__				     =			FaissIndex.load(a__   ,			storage_options=mockfs.storage_options       )

							UpperCamelCase__				     =			np.zeros(5   ,			dtype=np.floataa       )
							UpperCamelCase__				     =			1
							UpperCamelCase__     ,     UpperCamelCase__				     =			index.search(a__       )
							assert scores[0] > 0
							assert indices[0] == 1

@require_elasticsearch
class        __SCREAMING_SNAKE_CASE					(     _a       ):






							def 				_lowerCamelCase		(						self  ):
														from elasticsearch import Elasticsearch

														with patch("""elasticsearch.Elasticsearch.search"""  ) as mocked_search, patch(
														    """elasticsearch.client.IndicesClient.create"""  ) as mocked_index_create, patch("""elasticsearch.helpers.streaming_bulk"""  ) as mocked_bulk:
																					UpperCamelCase__				     =			Elasticsearch()
																					UpperCamelCase__				     =			{"""acknowledged""": True}
																					UpperCamelCase__				     =			ElasticSearchIndex(es_client=__lowerCAmelCase  )
																					mocked_bulk.return_value([(True, None)] * 3  )
																					index.add_documents(["""foo""", """bar""", """foobar"""]  )

																					# single query
																					UpperCamelCase__				     =			"""foo"""
																					UpperCamelCase__				     =			{"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 0}]}}
																					UpperCamelCase__     ,     UpperCamelCase__				     =			index.search(__lowerCAmelCase  )
																					self.assertEqual(scores[0]  , 1  )
																					self.assertEqual(indices[0]  , 0  )

																					# single query with timeout
																					UpperCamelCase__				     =			"""foo"""
																					UpperCamelCase__				     =			{"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 0}]}}
																					UpperCamelCase__     ,     UpperCamelCase__				     =			index.search(__lowerCAmelCase  , request_timeout=30  )
																					self.assertEqual(scores[0]  , 1  )
																					self.assertEqual(indices[0]  , 0  )

																					# batched queries
																					UpperCamelCase__				     =			["""foo""", """bar""", """foobar"""]
																					UpperCamelCase__				     =			{"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 1}]}}
																					UpperCamelCase__     ,     UpperCamelCase__				     =			index.search_batch(__lowerCAmelCase  )
																					UpperCamelCase__				     =			[scores[0] for scores in total_scores]
																					UpperCamelCase__				     =			[indices[0] for indices in total_indices]
																					self.assertGreater(np.min(__lowerCAmelCase  )  , 0  )
																					self.assertListEqual([1, 1, 1]  , __lowerCAmelCase  )

																					# batched queries with timeout
																					UpperCamelCase__				     =			["""foo""", """bar""", """foobar"""]
																					UpperCamelCase__				     =			{"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 1}]}}
																					UpperCamelCase__     ,     UpperCamelCase__				     =			index.search_batch(__lowerCAmelCase  , request_timeout=30  )
																					UpperCamelCase__				     =			[scores[0] for scores in total_scores]
																					UpperCamelCase__				     =			[indices[0] for indices in total_indices]
																					self.assertGreater(np.min(__lowerCAmelCase  )  , 0  )
																					self.assertListEqual([1, 1, 1]  , __lowerCAmelCase  )








from typing import Any





def     _UpperCamelCase     (a__   :list       ):






							"""simple docstring"""





							if not input_list:
														return []
							UpperCamelCase__				     =			[input_list.count(a__       ) for value in input_list]
							UpperCamelCase__				     =			max(a__       )  # Gets the maximum count in the input list.
							# Gets values of modes
							return sorted({input_list[i] for i, value in enumerate(a__       ) if value == y}       )


if __name__ == "__main__":
					import doctest

					doctest.testmod()




