




"""simple docstring"""




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowerCAmelCase							:    Optional[int]  							=							{
    """configuration_upernet""": ["""UperNetConfig"""],
}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowerCAmelCase							:    List[str]  							=							[
					    """UperNetForSemanticSegmentation""",
					    """UperNetPreTrainedModel""",
					]


if TYPE_CHECKING:
					from .configuration_upernet import UperNetConfig

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_upernet import UperNetForSemanticSegmentation, UperNetPreTrainedModel


else:
					import sys

					lowerCAmelCase							:    List[Any]  							=							_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)









"""simple docstring"""




import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


lowerCAmelCase							:    List[Any]  							=							pytest.mark.integration






@require_faiss
class 					__magic_name__							( UpperCAmelCase__       ):






				'''simple docstring'''





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							lowerCamelCase       =      Dataset.from_dict({"""filename""": ["""my_name-train""" + """_""" + str(_a				) for x in np.arange(30				).tolist()]}				)
							return dset





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							import faiss

							lowerCamelCase       =      self._create_dummy_dataset()
							lowerCamelCase       =      dset.map(
							    lambda _a	,				_a				: {"vecs": i * np.ones(5	,				dtype=np.floataa				)}	,				with_indices=_a	,				keep_in_memory=_a				)
							lowerCamelCase       =      dset.add_faiss_index("""vecs"""	,				batch_size=100	,				metric_type=faiss.METRIC_INNER_PRODUCT				)
							lowerCamelCase   ,     lowerCamelCase       =      dset.get_nearest_examples("""vecs"""	,				np.ones(5	,				dtype=np.floataa				)				)
							self.assertEqual(examples["""filename"""][0]	,				"""my_name-train_29"""				)
							dset.drop_index("""vecs"""				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							import faiss

							lowerCamelCase       =      self._create_dummy_dataset()
							dset.add_faiss_index_from_external_arrays(
							    external_arrays=np.ones((30, 5)				) * np.arange(30				).reshape(-1	,				1				)	,				index_name="""vecs"""	,				batch_size=100	,				metric_type=faiss.METRIC_INNER_PRODUCT	,				)
							lowerCamelCase   ,     lowerCamelCase       =      dset.get_nearest_examples("""vecs"""	,				np.ones(5	,				dtype=np.floataa				)				)
							self.assertEqual(examples["""filename"""][0]	,				"""my_name-train_29"""				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							import faiss

							lowerCamelCase       =      self._create_dummy_dataset()
							dset.add_faiss_index_from_external_arrays(
							    external_arrays=np.ones((30, 5)				) * np.arange(30				).reshape(-1	,				1				)	,				index_name="""vecs"""	,				metric_type=faiss.METRIC_INNER_PRODUCT	,				)

							# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
							# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
							# see https://bugs.python.org/issue14243 and
							# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
							with tempfile.NamedTemporaryFile(delete=_a				) as tmp_file:
										dset.save_faiss_index("""vecs"""	,				tmp_file.name				)
										dset.load_faiss_index("""vecs2"""	,				tmp_file.name				)
							os.unlink(tmp_file.name				)

							lowerCamelCase   ,     lowerCamelCase       =      dset.get_nearest_examples("""vecs2"""	,				np.ones(5	,				dtype=np.floataa				)				)
							self.assertEqual(examples["""filename"""][0]	,				"""my_name-train_29"""				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							lowerCamelCase       =      self._create_dummy_dataset()
							dset.add_faiss_index_from_external_arrays(
							    external_arrays=np.ones((30, 5)				) * np.arange(30				).reshape(-1	,				1				)	,				index_name="""vecs"""				)
							dset.drop_index("""vecs"""				)
							self.assertRaises(_a	,				partial(dset.get_nearest_examples	,				"""vecs2"""	,				np.ones(5	,				dtype=np.floataa				)				)				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							from elasticsearch import Elasticsearch

							lowerCamelCase       =      self._create_dummy_dataset()
							with patch("""elasticsearch.Elasticsearch.search"""				) as mocked_search, patch(
							    """elasticsearch.client.IndicesClient.create"""				) as mocked_index_create, patch("""elasticsearch.helpers.streaming_bulk"""				) as mocked_bulk:
										lowerCamelCase       =      {"""acknowledged""": True}
										mocked_bulk.return_value([(True, None)] * 30				)
										lowerCamelCase       =      {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 29}]}}
										lowerCamelCase       =      Elasticsearch()

										dset.add_elasticsearch_index("""filename"""	,				es_client=_a				)
										lowerCamelCase   ,     lowerCamelCase       =      dset.get_nearest_examples("""filename"""	,				"""my_name-train_29"""				)
										self.assertEqual(examples["""filename"""][0]	,				"""my_name-train_29"""				)






@require_faiss
class 					__magic_name__							( UpperCAmelCase__       ):






				'''simple docstring'''





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							import faiss

							lowerCamelCase       =      FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT				)

							# add vectors
							index.add_vectors(np.eye(5	,				dtype=np.floataa				)				)
							self.assertIsNotNone(index.faiss_index				)
							self.assertEqual(index.faiss_index.ntotal	,				5				)
							index.add_vectors(np.zeros((5, 5)	,				dtype=np.floataa				)				)
							self.assertEqual(index.faiss_index.ntotal	,				10				)

							# single query
							lowerCamelCase       =      np.zeros(5	,				dtype=np.floataa				)
							lowerCamelCase       =      1
							lowerCamelCase   ,     lowerCamelCase       =      index.search(_a				)
							self.assertRaises(_a	,				index.search	,				query.reshape(-1	,				1				)				)
							self.assertGreater(scores[0]	,				0				)
							self.assertEqual(indices[0]	,				1				)

							# batched queries
							lowerCamelCase       =      np.eye(5	,				dtype=np.floataa				)[::-1]
							lowerCamelCase   ,     lowerCamelCase       =      index.search_batch(_a				)
							self.assertRaises(_a	,				index.search_batch	,				queries[0]				)
							lowerCamelCase       =      [scores[0] for scores in total_scores]
							lowerCamelCase       =      [indices[0] for indices in total_indices]
							self.assertGreater(np.min(_a				)	,				0				)
							self.assertListEqual([4, 3, 2, 1, 0]	,				_a				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							import faiss

							lowerCamelCase       =      FaissIndex(string_factory="""Flat"""				)
							index.add_vectors(np.eye(5	,				dtype=np.floataa				)				)
							self.assertIsInstance(index.faiss_index	,				faiss.IndexFlat				)
							lowerCamelCase       =      FaissIndex(string_factory="""LSH"""				)
							index.add_vectors(np.eye(5	,				dtype=np.floataa				)				)
							self.assertIsInstance(index.faiss_index	,				faiss.IndexLSH				)
							with self.assertRaises(_a				):
										lowerCamelCase       =      FaissIndex(string_factory="""Flat"""	,				custom_index=faiss.IndexFlat(5				)				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							import faiss

							lowerCamelCase       =      faiss.IndexFlat(5				)
							lowerCamelCase       =      FaissIndex(custom_index=_a				)
							index.add_vectors(np.eye(5	,				dtype=np.floataa				)				)
							self.assertIsInstance(index.faiss_index	,				faiss.IndexFlat				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							import faiss

							lowerCamelCase       =      FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT				)
							index.add_vectors(np.eye(5	,				dtype=np.floataa				)				)

							# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
							# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
							# see https://bugs.python.org/issue14243 and
							# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
							with tempfile.NamedTemporaryFile(delete=_a				) as tmp_file:
										index.save(tmp_file.name				)
										lowerCamelCase       =      FaissIndex.load(tmp_file.name				)
							os.unlink(tmp_file.name				)

							lowerCamelCase       =      np.zeros(5	,				dtype=np.floataa				)
							lowerCamelCase       =      1
							lowerCamelCase   ,     lowerCamelCase       =      index.search(_a				)
							self.assertGreater(scores[0]	,				0				)
							self.assertEqual(indices[0]	,				1				)




@require_faiss
def 					a__				(       snake_case__       )		->      Tuple:
			import faiss

			lowerCamelCase       =      FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT       )
			index.add_vectors(np.eye(5 ,				dtype=np.floataa       )       )

			lowerCamelCase       =      """index.faiss"""
			lowerCamelCase       =      F'mock://{index_name}'
			index.save(snake_case__ ,				storage_options=mockfs.storage_options       )
			lowerCamelCase       =      FaissIndex.load(snake_case__ ,				storage_options=mockfs.storage_options       )

			lowerCamelCase       =      np.zeros(5 ,				dtype=np.floataa       )
			lowerCamelCase       =      1
			lowerCamelCase   ,     lowerCamelCase       =      index.search(snake_case__       )
			assert scores[0] > 0
			assert indices[0] == 1






@require_elasticsearch
class 					__magic_name__							( UpperCAmelCase__       ):






				'''simple docstring'''





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							from elasticsearch import Elasticsearch

							with patch("""elasticsearch.Elasticsearch.search"""				) as mocked_search, patch(
							    """elasticsearch.client.IndicesClient.create"""				) as mocked_index_create, patch("""elasticsearch.helpers.streaming_bulk"""				) as mocked_bulk:
										lowerCamelCase       =      Elasticsearch()
										lowerCamelCase       =      {"""acknowledged""": True}
										lowerCamelCase       =      ElasticSearchIndex(es_client=_a				)
										mocked_bulk.return_value([(True, None)] * 3				)
										index.add_documents(["""foo""", """bar""", """foobar"""]				)

										# single query
										lowerCamelCase       =      """foo"""
										lowerCamelCase       =      {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 0}]}}
										lowerCamelCase   ,     lowerCamelCase       =      index.search(_a				)
										self.assertEqual(scores[0]	,				1				)
										self.assertEqual(indices[0]	,				0				)

										# single query with timeout
										lowerCamelCase       =      """foo"""
										lowerCamelCase       =      {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 0}]}}
										lowerCamelCase   ,     lowerCamelCase       =      index.search(_a	,				request_timeout=30				)
										self.assertEqual(scores[0]	,				1				)
										self.assertEqual(indices[0]	,				0				)

										# batched queries
										lowerCamelCase       =      ["""foo""", """bar""", """foobar"""]
										lowerCamelCase       =      {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 1}]}}
										lowerCamelCase   ,     lowerCamelCase       =      index.search_batch(_a				)
										lowerCamelCase       =      [scores[0] for scores in total_scores]
										lowerCamelCase       =      [indices[0] for indices in total_indices]
										self.assertGreater(np.min(_a				)	,				0				)
										self.assertListEqual([1, 1, 1]	,				_a				)

										# batched queries with timeout
										lowerCamelCase       =      ["""foo""", """bar""", """foobar"""]
										lowerCamelCase       =      {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 1}]}}
										lowerCamelCase   ,     lowerCamelCase       =      index.search_batch(_a	,				request_timeout=30				)
										lowerCamelCase       =      [scores[0] for scores in total_scores]
										lowerCamelCase       =      [indices[0] for indices in total_indices]
										self.assertGreater(np.min(_a				)	,				0				)
										self.assertListEqual([1, 1, 1]	,				_a				)



