





import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


A   : Tuple       =				pytest.mark.integration


@require_faiss
class   A      (__a   ):
			'''simple docstring'''




			def        a_	(      self       :   int       )     ->     Optional[Any]:







					"""simple docstring"""






					A__  				=       Dataset.from_dict({"""filename""": ["""my_name-train""" + """_""" + str(a__       ) for x in np.arange(30       ).tolist()]}       )
					return dset






			def        a_	(      self       :   List[str]       )     ->     Union[str, Any]:







					"""simple docstring"""






					import faiss

					A__  				=       self._create_dummy_dataset()
					A__  				=       dset.map(
					    lambda __lowerCAmelCase							,     __lowerCAmelCase       : {"vecs": i * np.ones(5							,     dtype=np.floataa       )}							,     with_indices=a__							,     keep_in_memory=a__       )
					A__  				=       dset.add_faiss_index("""vecs"""							,     batch_size=1_00							,     metric_type=faiss.METRIC_INNER_PRODUCT       )
					A__				,      A__  				=       dset.get_nearest_examples("""vecs"""							,     np.ones(5							,     dtype=np.floataa       )       )
					self.assertEqual(examples["""filename"""][0]							,     """my_name-train_29"""       )
					dset.drop_index("""vecs"""       )






			def        a_	(      self       :   str       )     ->     List[str]:







					"""simple docstring"""






					import faiss

					A__  				=       self._create_dummy_dataset()
					dset.add_faiss_index_from_external_arrays(
					    external_arrays=np.ones((30, 5)       ) * np.arange(30       ).reshape(-1							,     1       )							,     index_name="""vecs"""							,     batch_size=1_00							,     metric_type=faiss.METRIC_INNER_PRODUCT							,     )
					A__				,      A__  				=       dset.get_nearest_examples("""vecs"""							,     np.ones(5							,     dtype=np.floataa       )       )
					self.assertEqual(examples["""filename"""][0]							,     """my_name-train_29"""       )






			def        a_	(      self       :   Union[str, Any]       )     ->     int:







					"""simple docstring"""






					import faiss

					A__  				=       self._create_dummy_dataset()
					dset.add_faiss_index_from_external_arrays(
					    external_arrays=np.ones((30, 5)       ) * np.arange(30       ).reshape(-1							,     1       )							,     index_name="""vecs"""							,     metric_type=faiss.METRIC_INNER_PRODUCT							,     )

					# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
					# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
					# see https://bugs.python.org/issue14243 and
					# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
					with tempfile.NamedTemporaryFile(delete=a__       ) as tmp_file:
							dset.save_faiss_index("""vecs"""							,     tmp_file.name       )
							dset.load_faiss_index("""vecs2"""							,     tmp_file.name       )
					os.unlink(tmp_file.name       )

					A__				,      A__  				=       dset.get_nearest_examples("""vecs2"""							,     np.ones(5							,     dtype=np.floataa       )       )
					self.assertEqual(examples["""filename"""][0]							,     """my_name-train_29"""       )






			def        a_	(      self       :   Union[str, Any]       )     ->     int:







					"""simple docstring"""






					A__  				=       self._create_dummy_dataset()
					dset.add_faiss_index_from_external_arrays(
					    external_arrays=np.ones((30, 5)       ) * np.arange(30       ).reshape(-1							,     1       )							,     index_name="""vecs"""       )
					dset.drop_index("""vecs"""       )
					self.assertRaises(a__							,     partial(dset.get_nearest_examples							,     """vecs2"""							,     np.ones(5							,     dtype=np.floataa       )       )       )







			def        a_	(      self       :   List[str]       )     ->     List[Any]:







					"""simple docstring"""






					from elasticsearch import Elasticsearch

					A__  				=       self._create_dummy_dataset()
					with patch("""elasticsearch.Elasticsearch.search"""       ) as mocked_search, patch(
					    """elasticsearch.client.IndicesClient.create"""       ) as mocked_index_create, patch("""elasticsearch.helpers.streaming_bulk"""       ) as mocked_bulk:
							A__  				=       {"""acknowledged""": True}
							mocked_bulk.return_value([(True, None)] * 30       )
							A__  				=       {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 29}]}}
							A__  				=       Elasticsearch()

							dset.add_elasticsearch_index("""filename"""							,     es_client=a__       )
							A__				,      A__  				=       dset.get_nearest_examples("""filename"""							,     """my_name-train_29"""       )
							self.assertEqual(examples["""filename"""][0]							,     """my_name-train_29"""       )


@require_faiss
class   A      (__a   ):
			'''simple docstring'''




			def        a_	(      self       :   Union[str, Any]       )     ->     Any:







					"""simple docstring"""






					import faiss

					A__  				=       FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT       )

					# add vectors
					index.add_vectors(np.eye(5							,     dtype=np.floataa       )       )
					self.assertIsNotNone(index.faiss_index       )
					self.assertEqual(index.faiss_index.ntotal							,     5       )
					index.add_vectors(np.zeros((5, 5)							,     dtype=np.floataa       )       )
					self.assertEqual(index.faiss_index.ntotal							,     10       )

					# single query
					A__  				=       np.zeros(5							,     dtype=np.floataa       )
					A__  				=       1
					A__				,      A__  				=       index.search(a__       )
					self.assertRaises(a__							,     index.search							,     query.reshape(-1							,     1       )       )
					self.assertGreater(scores[0]							,     0       )
					self.assertEqual(indices[0]							,     1       )

					# batched queries
					A__  				=       np.eye(5							,     dtype=np.floataa       )[::-1]
					A__				,      A__  				=       index.search_batch(a__       )
					self.assertRaises(a__							,     index.search_batch							,     queries[0]       )
					A__  				=       [scores[0] for scores in total_scores]
					A__  				=       [indices[0] for indices in total_indices]
					self.assertGreater(np.min(a__       )							,     0       )
					self.assertListEqual([4, 3, 2, 1, 0]							,     a__       )






			def        a_	(      self       :   Optional[Any]       )     ->     Optional[int]:







					"""simple docstring"""






					import faiss

					A__  				=       FaissIndex(string_factory="""Flat"""       )
					index.add_vectors(np.eye(5							,     dtype=np.floataa       )       )
					self.assertIsInstance(index.faiss_index							,     faiss.IndexFlat       )
					A__  				=       FaissIndex(string_factory="""LSH"""       )
					index.add_vectors(np.eye(5							,     dtype=np.floataa       )       )
					self.assertIsInstance(index.faiss_index							,     faiss.IndexLSH       )
					with self.assertRaises(a__       ):
							A__  				=       FaissIndex(string_factory="""Flat"""							,     custom_index=faiss.IndexFlat(5       )       )






			def        a_	(      self       :   List[str]       )     ->     List[Any]:







					"""simple docstring"""






					import faiss

					A__  				=       faiss.IndexFlat(5       )
					A__  				=       FaissIndex(custom_index=a__       )
					index.add_vectors(np.eye(5							,     dtype=np.floataa       )       )
					self.assertIsInstance(index.faiss_index							,     faiss.IndexFlat       )







			def        a_	(      self       :   str       )     ->     Any:







					"""simple docstring"""






					import faiss

					A__  				=       FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT       )
					index.add_vectors(np.eye(5							,     dtype=np.floataa       )       )

					# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
					# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
					# see https://bugs.python.org/issue14243 and
					# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
					with tempfile.NamedTemporaryFile(delete=a__       ) as tmp_file:
							index.save(tmp_file.name       )
							A__  				=       FaissIndex.load(tmp_file.name       )
					os.unlink(tmp_file.name       )

					A__  				=       np.zeros(5							,     dtype=np.floataa       )
					A__  				=       1
					A__				,      A__  				=       index.search(a__       )
					self.assertGreater(scores[0]							,     0       )
					self.assertEqual(indices[0]							,     1       )







@require_faiss
def        __lowerCamelCase (							__a       :Optional[int]   )		->    Tuple:






		"""simple docstring"""

		import faiss

		A__  				=       FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT   )
		index.add_vectors(np.eye(5   ,					dtype=np.floataa   )   )

		A__  				=       """index.faiss"""
		A__  				=       F'mock://{index_name}'
		index.save(__a   ,					storage_options=mockfs.storage_options   )
		A__  				=       FaissIndex.load(__a   ,					storage_options=mockfs.storage_options   )

		A__  				=       np.zeros(5   ,					dtype=np.floataa   )
		A__  				=       1
		A__				,      A__  				=       index.search(__a   )
		assert scores[0] > 0
		assert indices[0] == 1






@require_elasticsearch
class   A      (__a   ):
			'''simple docstring'''




			def        a_	(      self       :   Any       )     ->     Tuple:







					"""simple docstring"""






					from elasticsearch import Elasticsearch

					with patch("""elasticsearch.Elasticsearch.search"""       ) as mocked_search, patch(
					    """elasticsearch.client.IndicesClient.create"""       ) as mocked_index_create, patch("""elasticsearch.helpers.streaming_bulk"""       ) as mocked_bulk:
							A__  				=       Elasticsearch()
							A__  				=       {"""acknowledged""": True}
							A__  				=       ElasticSearchIndex(es_client=a__       )
							mocked_bulk.return_value([(True, None)] * 3       )
							index.add_documents(["""foo""", """bar""", """foobar"""]       )

							# single query
							A__  				=       """foo"""
							A__  				=       {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 0}]}}
							A__				,      A__  				=       index.search(a__       )
							self.assertEqual(scores[0]							,     1       )
							self.assertEqual(indices[0]							,     0       )

							# single query with timeout
							A__  				=       """foo"""
							A__  				=       {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 0}]}}
							A__				,      A__  				=       index.search(a__							,     request_timeout=30       )
							self.assertEqual(scores[0]							,     1       )
							self.assertEqual(indices[0]							,     0       )

							# batched queries
							A__  				=       ["""foo""", """bar""", """foobar"""]
							A__  				=       {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 1}]}}
							A__				,      A__  				=       index.search_batch(a__       )
							A__  				=       [scores[0] for scores in total_scores]
							A__  				=       [indices[0] for indices in total_indices]
							self.assertGreater(np.min(a__       )							,     0       )
							self.assertListEqual([1, 1, 1]							,     a__       )

							# batched queries with timeout
							A__  				=       ["""foo""", """bar""", """foobar"""]
							A__  				=       {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 1}]}}
							A__				,      A__  				=       index.search_batch(a__							,     request_timeout=30       )
							A__  				=       [scores[0] for scores in total_scores]
							A__  				=       [indices[0] for indices in total_indices]
							self.assertGreater(np.min(a__       )							,     0       )
							self.assertListEqual([1, 1, 1]							,     a__       )










# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch

from accelerate import PartialState
from accelerate.utils.operations import broadcast, gather, gather_object, pad_across_processes, reduce







def        __lowerCamelCase (							__a       :List[str]   )		->    Tuple:






		"""simple docstring"""

		return (torch.arange(state.num_processes   ) + 1.0 + (state.num_processes * state.process_index)).to(state.device   )







def        __lowerCamelCase (							__a       :int   )		->    Optional[int]:






		"""simple docstring"""

		A__  				=       create_tensor(__a   )
		A__  				=       gather(__a   )
		assert gathered_tensor.tolist() == list(range(1   ,					state.num_processes**2 + 1   )   )







def        __lowerCamelCase (							__a       :Any   )		->    Any:






		"""simple docstring"""

		A__  				=       [state.process_index]
		A__  				=       gather_object(__a   )
		assert len(__a   ) == state.num_processes, F'{gathered_obj}, {len(__a   )} != {state.num_processes}'
		assert gathered_obj == list(range(state.num_processes   )   ), F'{gathered_obj} != {list(range(state.num_processes   )   )}'







def        __lowerCamelCase (							__a       :List[str]   )		->    List[str]:






		"""simple docstring"""

		A__  				=       create_tensor(__a   )
		A__  				=       broadcast(__a   )
		assert broadcasted_tensor.shape == torch.Size([state.num_processes]   )
		assert broadcasted_tensor.tolist() == list(range(1   ,					state.num_processes + 1   )   )







def        __lowerCamelCase (							__a       :Any   )		->    Any:






		"""simple docstring"""

		if state.is_main_process:
				A__  				=       torch.arange(state.num_processes + 1   ).to(state.device   )
		else:
				A__  				=       torch.arange(state.num_processes   ).to(state.device   )
		A__  				=       pad_across_processes(__a   )
		assert padded_tensor.shape == torch.Size([state.num_processes + 1]   )
		if not state.is_main_process:
				assert padded_tensor.tolist() == list(range(0   ,					state.num_processes   )   ) + [0]







def        __lowerCamelCase (							__a       :Union[str, Any]   )		->    str:






		"""simple docstring"""

		if state.num_processes != 2:
				return
		A__  				=       create_tensor(__a   )
		A__  				=       reduce(__a   ,					"""sum"""   )
		A__  				=       torch.tensor([4.0, 6]   ).to(state.device   )
		assert torch.allclose(__a   ,					__a   ), F'{reduced_tensor} != {truth_tensor}'







def        __lowerCamelCase (							__a       :List[Any]   )		->    List[str]:






		"""simple docstring"""

		if state.num_processes != 2:
				return
		A__  				=       create_tensor(__a   )
		A__  				=       reduce(__a   ,					"""mean"""   )
		A__  				=       torch.tensor([2.0, 3]   ).to(state.device   )
		assert torch.allclose(__a   ,					__a   ), F'{reduced_tensor} != {truth_tensor}'







def        __lowerCamelCase (							__a       :str   )		->    Optional[int]:






		"""simple docstring"""

		main()







def        __lowerCamelCase (							)		->    str:






		"""simple docstring"""

		A__  				=       PartialState()
		state.print(F'State: {state}'   )
		state.print("""testing gather"""   )
		test_gather(__a   )
		state.print("""testing gather_object"""   )
		test_gather_object(__a   )
		state.print("""testing broadcast"""   )
		test_broadcast(__a   )
		state.print("""testing pad_across_processes"""   )
		test_pad_across_processes(__a   )
		state.print("""testing reduce_sum"""   )
		test_reduce_sum(__a   )
		state.print("""testing reduce_mean"""   )
		test_reduce_mean(__a   )


if __name__ == "__main__":
					main()



