import json
import os
import pickle
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers import is_faiss_available
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bart.tokenization_bart import BartTokenizer
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.dpr.tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer
from transformers.models.rag.configuration_rag import RagConfig
from transformers.models.rag.retrieval_rag import CustomHFIndex, RagRetriever
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_sentencepiece, require_tokenizers, require_torch


if is_faiss_available():
      import faiss

@require_faiss
class    snake_case       (   UpperCamelCase_    ):




  def 						__lowercase(   self  :						List[str]					)->							Any:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__					:					Any      =		tempfile.mkdtemp()
       SCREAMING_SNAKE_CASE__					:					List[Any]      =		8

       # DPR tok
       SCREAMING_SNAKE_CASE__					:					Union[str, Any]      =		[
           '[UNK]',
           '[CLS]',
           '[SEP]',
           '[PAD]',
           '[MASK]',
           'want',
           '##want',
           '##ed',
           'wa',
           'un',
           'runn',
           '##ing',
           ',',
           'low',
           'lowest',
       ]
       SCREAMING_SNAKE_CASE__					:					Dict      =		os.path.join(self.tmpdirname  ,			'dpr_tokenizer'					)
       os.makedirs(a_  ,			exist_ok=a_					)
       SCREAMING_SNAKE_CASE__					:					Tuple      =		os.path.join(a_  ,			DPR_VOCAB_FILES_NAMES['vocab_file']					)
       with open(self.vocab_file  ,			'w'  ,			encoding='utf-8'					) as vocab_writer:
            vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)

       # BART tok
       SCREAMING_SNAKE_CASE__					:					List[str]      =		[
           'l',
           'o',
           'w',
           'e',
           'r',
           's',
           't',
           'i',
           'd',
           'n',
           '\u0120',
           '\u0120l',
           '\u0120n',
           '\u0120lo',
           '\u0120low',
           'er',
           '\u0120lowest',
           '\u0120newer',
           '\u0120wider',
           '<unk>',
       ]
       SCREAMING_SNAKE_CASE__					:					Any      =		dict(zip(a_  ,			range(len(a_					)					)					)					)
       SCREAMING_SNAKE_CASE__					:					Tuple      =		['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
       SCREAMING_SNAKE_CASE__					:					Union[str, Any]      =		{'unk_token': '<unk>'}

       SCREAMING_SNAKE_CASE__					:					List[str]      =		os.path.join(self.tmpdirname  ,			'bart_tokenizer'					)
       os.makedirs(a_  ,			exist_ok=a_					)
       SCREAMING_SNAKE_CASE__					:					Optional[Any]      =		os.path.join(a_  ,			BART_VOCAB_FILES_NAMES['vocab_file']					)
       SCREAMING_SNAKE_CASE__					:					List[str]      =		os.path.join(a_  ,			BART_VOCAB_FILES_NAMES['merges_file']					)
       with open(self.vocab_file  ,			'w'  ,			encoding='utf-8'					) as fp:
            fp.write(json.dumps(a_					) + '\n'					)
       with open(self.merges_file  ,			'w'  ,			encoding='utf-8'					) as fp:
            fp.write('\n'.join(a_					)					)




  def 						__lowercase(   self  :						Union[str, Any]					)->							DPRQuestionEncoderTokenizer:




       """simple docstring"""
       return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname  ,			'dpr_tokenizer'					)					)




  def 						__lowercase(   self  :						Union[str, Any]					)->							DPRContextEncoderTokenizer:




       """simple docstring"""
       return DPRContextEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname  ,			'dpr_tokenizer'					)					)




  def 						__lowercase(   self  :						List[Any]					)->							BartTokenizer:




       """simple docstring"""
       return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname  ,			'bart_tokenizer'					)					)




  def 						__lowercase(   self  :						str					)->							int:




       """simple docstring"""
       shutil.rmtree(self.tmpdirname					)




  def 						__lowercase(   self  :						str					)->							Dict:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__					:					Tuple      =		Dataset.from_dict(
           {
               'id': ['0', '1'],
               'text': ['foo', 'bar'],
               'title': ['Foo', 'Bar'],
               'embeddings': [np.ones(self.retrieval_vector_size					), 2 * np.ones(self.retrieval_vector_size					)],
           }					)
       dataset.add_faiss_index('embeddings'  ,			string_factory='Flat'  ,			metric_type=faiss.METRIC_INNER_PRODUCT					)
       return dataset




  def 						__lowercase(   self  :						List[str]					)->							Optional[Any]:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__					:					Union[str, Any]      =		self.get_dummy_dataset()
       SCREAMING_SNAKE_CASE__					:					Optional[int]      =		RagConfig(
           retrieval_vector_size=self.retrieval_vector_size  ,			question_encoder=DPRConfig().to_dict()  ,			generator=BartConfig().to_dict()  ,			)
       with patch('transformers.models.rag.retrieval_rag.load_dataset'					) as mock_load_dataset:
            SCREAMING_SNAKE_CASE__					:					Any      =		dataset
            SCREAMING_SNAKE_CASE__					:					int      =		RagRetriever(
                a_  ,			question_encoder_tokenizer=self.get_dpr_tokenizer()  ,			generator_tokenizer=self.get_bart_tokenizer()  ,			)
       return retriever




  def 						__lowercase(   self  :						Tuple  ,			a_  :						bool					)->							Union[str, Any]:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__					:					List[str]      =		self.get_dummy_dataset()
       SCREAMING_SNAKE_CASE__					:					Union[str, Any]      =		RagConfig(
           retrieval_vector_size=self.retrieval_vector_size  ,			question_encoder=DPRConfig().to_dict()  ,			generator=BartConfig().to_dict()  ,			index_name='custom'  ,			)
       if from_disk:
            SCREAMING_SNAKE_CASE__					:					Optional[Any]      =		os.path.join(self.tmpdirname  ,			'dataset'					)
            SCREAMING_SNAKE_CASE__					:					Optional[int]      =		os.path.join(self.tmpdirname  ,			'index.faiss'					)
            dataset.get_index('embeddings'					).save(os.path.join(self.tmpdirname  ,			'index.faiss'					)					)
            dataset.drop_index('embeddings'					)
            dataset.save_to_disk(os.path.join(self.tmpdirname  ,			'dataset'					)					)
            del dataset
            SCREAMING_SNAKE_CASE__					:					Union[str, Any]      =		RagRetriever(
                a_  ,			question_encoder_tokenizer=self.get_dpr_tokenizer()  ,			generator_tokenizer=self.get_bart_tokenizer()  ,			)
       else:
            SCREAMING_SNAKE_CASE__					:					Optional[int]      =		RagRetriever(
                a_  ,			question_encoder_tokenizer=self.get_dpr_tokenizer()  ,			generator_tokenizer=self.get_bart_tokenizer()  ,			index=CustomHFIndex(config.retrieval_vector_size  ,			a_					)  ,			)
       return retriever




  def 						__lowercase(   self  :						Any					)->							str:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__					:					int      =		Dataset.from_dict(
           {
               'id': ['0', '1'],
               'text': ['foo', 'bar'],
               'title': ['Foo', 'Bar'],
               'embeddings': [np.ones(self.retrieval_vector_size + 1					), 2 * np.ones(self.retrieval_vector_size + 1					)],
           }					)
       dataset.add_faiss_index('embeddings'  ,			string_factory='Flat'  ,			metric_type=faiss.METRIC_INNER_PRODUCT					)

       SCREAMING_SNAKE_CASE__					:					str      =		os.path.join(self.tmpdirname  ,			'hf_bert_base.hnswSQ8_correct_phi_128.c_index'					)
       dataset.save_faiss_index('embeddings'  ,			index_file_name + '.index.dpr'					)
       pickle.dump(dataset['id']  ,			open(index_file_name + '.index_meta.dpr'  ,			'wb'					)					)

       SCREAMING_SNAKE_CASE__					:					List[str]      =		os.path.join(self.tmpdirname  ,			'psgs_w100.tsv.pkl'					)
       SCREAMING_SNAKE_CASE__					:					List[str]      =		{sample['id']: [sample['text'], sample['title']] for sample in dataset}
       pickle.dump(a_  ,			open(a_  ,			'wb'					)					)

       SCREAMING_SNAKE_CASE__					:					Any      =		RagConfig(
           retrieval_vector_size=self.retrieval_vector_size  ,			question_encoder=DPRConfig().to_dict()  ,			generator=BartConfig().to_dict()  ,			index_name='legacy'  ,			index_path=self.tmpdirname  ,			)
       SCREAMING_SNAKE_CASE__					:					int      =		RagRetriever(
           a_  ,			question_encoder_tokenizer=self.get_dpr_tokenizer()  ,			generator_tokenizer=self.get_bart_tokenizer()					)
       return retriever




  def 						__lowercase(   self  :						Tuple					)->							Tuple:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__					:					int      =		1
       SCREAMING_SNAKE_CASE__					:					Optional[Any]      =		self.get_dummy_canonical_hf_index_retriever()
       SCREAMING_SNAKE_CASE__					:					Any      =		np.array(
           [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]  ,			dtype=np.floataa					)
       SCREAMING_SNAKE_CASE__      ,				SCREAMING_SNAKE_CASE__      ,				SCREAMING_SNAKE_CASE__					:					List[str]      =		retriever.retrieve(a_  ,			n_docs=a_					)
       self.assertEqual(retrieved_doc_embeds.shape  ,			(2, n_docs, self.retrieval_vector_size)					)
       self.assertEqual(len(a_					)  ,			2					)
       self.assertEqual(sorted(doc_dicts[0]					)  ,			['embeddings', 'id', 'text', 'title']					)
       self.assertEqual(len(doc_dicts[0]['id']					)  ,			a_					)
       self.assertEqual(doc_dicts[0]['id'][0]  ,			'1'					)  # max inner product is reached with second doc
       self.assertEqual(doc_dicts[1]['id'][0]  ,			'0'					)  # max inner product is reached with first doc
       self.assertListEqual(doc_ids.tolist()  ,			[[1], [0]]					)




  def 						__lowercase(   self  :						List[str]					)->							Optional[int]:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__					:					int      =		self.get_dummy_canonical_hf_index_retriever()
       with tempfile.TemporaryDirectory() as tmp_dirname:
            with patch('transformers.models.rag.retrieval_rag.load_dataset'					) as mock_load_dataset:
                 SCREAMING_SNAKE_CASE__					:					int      =		self.get_dummy_dataset()
                 retriever.save_pretrained(a_					)
                 SCREAMING_SNAKE_CASE__					:					Optional[Any]      =		RagRetriever.from_pretrained(a_					)
                 self.assertIsInstance(a_  ,			a_					)
                 SCREAMING_SNAKE_CASE__					:					Optional[int]      =		np.array(
                     [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]  ,			dtype=np.floataa					)
                 SCREAMING_SNAKE_CASE__					:					List[Any]      =		retriever.retrieve(a_  ,			n_docs=1					)
                 self.assertTrue(out is not None					)




  def 						__lowercase(   self  :						Optional[Any]					)->							Union[str, Any]:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__					:					Optional[int]      =		1
       SCREAMING_SNAKE_CASE__					:					int      =		self.get_dummy_custom_hf_index_retriever(from_disk=a_					)
       SCREAMING_SNAKE_CASE__					:					Optional[int]      =		np.array(
           [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]  ,			dtype=np.floataa					)
       SCREAMING_SNAKE_CASE__      ,				SCREAMING_SNAKE_CASE__      ,				SCREAMING_SNAKE_CASE__					:					Union[str, Any]      =		retriever.retrieve(a_  ,			n_docs=a_					)
       self.assertEqual(retrieved_doc_embeds.shape  ,			(2, n_docs, self.retrieval_vector_size)					)
       self.assertEqual(len(a_					)  ,			2					)
       self.assertEqual(sorted(doc_dicts[0]					)  ,			['embeddings', 'id', 'text', 'title']					)
       self.assertEqual(len(doc_dicts[0]['id']					)  ,			a_					)
       self.assertEqual(doc_dicts[0]['id'][0]  ,			'1'					)  # max inner product is reached with second doc
       self.assertEqual(doc_dicts[1]['id'][0]  ,			'0'					)  # max inner product is reached with first doc
       self.assertListEqual(doc_ids.tolist()  ,			[[1], [0]]					)




  def 						__lowercase(   self  :						Optional[int]					)->							Union[str, Any]:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__					:					Any      =		self.get_dummy_custom_hf_index_retriever(from_disk=a_					)
       with tempfile.TemporaryDirectory() as tmp_dirname:
            retriever.save_pretrained(a_					)
            SCREAMING_SNAKE_CASE__					:					List[str]      =		RagRetriever.from_pretrained(a_					)
            self.assertIsInstance(a_  ,			a_					)
            SCREAMING_SNAKE_CASE__					:					Tuple      =		np.array(
                [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]  ,			dtype=np.floataa					)
            SCREAMING_SNAKE_CASE__					:					Optional[Any]      =		retriever.retrieve(a_  ,			n_docs=1					)
            self.assertTrue(out is not None					)




  def 						__lowercase(   self  :						Union[str, Any]					)->							Optional[Any]:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__					:					Optional[Any]      =		1
       SCREAMING_SNAKE_CASE__					:					List[Any]      =		self.get_dummy_custom_hf_index_retriever(from_disk=a_					)
       SCREAMING_SNAKE_CASE__					:					Tuple      =		np.array(
           [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]  ,			dtype=np.floataa					)
       SCREAMING_SNAKE_CASE__      ,				SCREAMING_SNAKE_CASE__      ,				SCREAMING_SNAKE_CASE__					:					Dict      =		retriever.retrieve(a_  ,			n_docs=a_					)
       self.assertEqual(retrieved_doc_embeds.shape  ,			(2, n_docs, self.retrieval_vector_size)					)
       self.assertEqual(len(a_					)  ,			2					)
       self.assertEqual(sorted(doc_dicts[0]					)  ,			['embeddings', 'id', 'text', 'title']					)
       self.assertEqual(len(doc_dicts[0]['id']					)  ,			a_					)
       self.assertEqual(doc_dicts[0]['id'][0]  ,			'1'					)  # max inner product is reached with second doc
       self.assertEqual(doc_dicts[1]['id'][0]  ,			'0'					)  # max inner product is reached with first doc
       self.assertListEqual(doc_ids.tolist()  ,			[[1], [0]]					)




  def 						__lowercase(   self  :						Dict					)->							Tuple:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__					:					Optional[Any]      =		self.get_dummy_custom_hf_index_retriever(from_disk=a_					)
       with tempfile.TemporaryDirectory() as tmp_dirname:
            retriever.save_pretrained(a_					)
            SCREAMING_SNAKE_CASE__					:					Optional[Any]      =		RagRetriever.from_pretrained(a_					)
            self.assertIsInstance(a_  ,			a_					)
            SCREAMING_SNAKE_CASE__					:					Dict      =		np.array(
                [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]  ,			dtype=np.floataa					)
            SCREAMING_SNAKE_CASE__					:					Any      =		retriever.retrieve(a_  ,			n_docs=1					)
            self.assertTrue(out is not None					)




  def 						__lowercase(   self  :						str					)->							int:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__					:					Optional[Any]      =		1
       SCREAMING_SNAKE_CASE__					:					Tuple      =		self.get_dummy_legacy_index_retriever()
       SCREAMING_SNAKE_CASE__					:					Union[str, Any]      =		np.array(
           [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]  ,			dtype=np.floataa					)
       SCREAMING_SNAKE_CASE__      ,				SCREAMING_SNAKE_CASE__      ,				SCREAMING_SNAKE_CASE__					:					Optional[int]      =		retriever.retrieve(a_  ,			n_docs=a_					)
       self.assertEqual(retrieved_doc_embeds.shape  ,			(2, n_docs, self.retrieval_vector_size)					)
       self.assertEqual(len(a_					)  ,			2					)
       self.assertEqual(sorted(doc_dicts[0]					)  ,			['text', 'title']					)
       self.assertEqual(len(doc_dicts[0]['text']					)  ,			a_					)
       self.assertEqual(doc_dicts[0]['text'][0]  ,			'bar'					)  # max inner product is reached with second doc
       self.assertEqual(doc_dicts[1]['text'][0]  ,			'foo'					)  # max inner product is reached with first doc
       self.assertListEqual(doc_ids.tolist()  ,			[[1], [0]]					)




  def 						__lowercase(   self  :						Dict					)->							List[str]:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__					:					str      =		self.get_dummy_legacy_index_retriever()
       with tempfile.TemporaryDirectory() as tmp_dirname:
            retriever.save_pretrained(a_					)
            SCREAMING_SNAKE_CASE__					:					Optional[Any]      =		RagRetriever.from_pretrained(a_					)
            self.assertIsInstance(a_  ,			a_					)
            SCREAMING_SNAKE_CASE__					:					Any      =		np.array(
                [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]  ,			dtype=np.floataa					)
            SCREAMING_SNAKE_CASE__					:					Union[str, Any]      =		retriever.retrieve(a_  ,			n_docs=1					)
            self.assertTrue(out is not None					)




  @require_torch
  @require_tokenizers
  @require_sentencepiece
  def 						__lowercase(   self  :						int					)->							Tuple:




       """simple docstring"""
       import torch

       SCREAMING_SNAKE_CASE__					:					List[Any]      =		1
       SCREAMING_SNAKE_CASE__					:					Any      =		self.get_dummy_canonical_hf_index_retriever()
       SCREAMING_SNAKE_CASE__					:					Optional[Any]      =		[[5, 7], [10, 11]]
       SCREAMING_SNAKE_CASE__					:					str      =		np.array(
           [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]  ,			dtype=np.floataa					)
       SCREAMING_SNAKE_CASE__					:					Union[str, Any]      =		retriever(a_  ,			a_  ,			prefix=retriever.config.generator.prefix  ,			n_docs=a_					)
       SCREAMING_SNAKE_CASE__      ,				SCREAMING_SNAKE_CASE__      ,				SCREAMING_SNAKE_CASE__					:					List[str]      =		(
           out['context_input_ids'],
           out['context_attention_mask'],
           out['retrieved_doc_embeds'],
       )
       self.assertEqual(retrieved_doc_embeds.shape  ,			(2, n_docs, self.retrieval_vector_size)					)
       self.assertIsInstance(a_  ,			a_					)
       self.assertIsInstance(a_  ,			a_					)
       self.assertIsInstance(a_  ,			np.ndarray					)

       SCREAMING_SNAKE_CASE__					:					int      =		retriever(
           a_  ,			a_  ,			prefix=retriever.config.generator.prefix  ,			n_docs=a_  ,			return_tensors='pt'  ,			)
       SCREAMING_SNAKE_CASE__      ,				SCREAMING_SNAKE_CASE__      ,				SCREAMING_SNAKE_CASE__      ,				SCREAMING_SNAKE_CASE__					:					Tuple      =		(  # noqa: F841
           out['context_input_ids'],
           out['context_attention_mask'],
           out['retrieved_doc_embeds'],
           out['doc_ids'],
       )
       self.assertEqual(retrieved_doc_embeds.shape  ,			(2, n_docs, self.retrieval_vector_size)					)
       self.assertIsInstance(a_  ,			torch.Tensor					)
       self.assertIsInstance(a_  ,			torch.Tensor					)
       self.assertIsInstance(a_  ,			torch.Tensor					)




  @require_torch
  @require_tokenizers
  @require_sentencepiece
  def 						__lowercase(   self  :						List[Any]					)->							Dict:




       """simple docstring"""
       SCREAMING_SNAKE_CASE__					:					Optional[int]      =		self.get_dpr_ctx_encoder_tokenizer()
       SCREAMING_SNAKE_CASE__					:					Dict      =		1
       SCREAMING_SNAKE_CASE__					:					int      =		self.get_dummy_custom_hf_index_retriever(from_disk=a_					)
       retriever.set_ctx_encoder_tokenizer(a_					)

       SCREAMING_SNAKE_CASE__					:					Optional[Any]      =		[[5, 7], [10, 11]]
       SCREAMING_SNAKE_CASE__					:					List[Any]      =		np.array(
           [np.ones(self.retrieval_vector_size					), -np.ones(self.retrieval_vector_size					)]  ,			dtype=np.floataa					)
       SCREAMING_SNAKE_CASE__					:					int      =		retriever(a_  ,			a_  ,			prefix=retriever.config.generator.prefix  ,			n_docs=a_					)

       self.assertEqual(
           len(a_					)  ,			6					)  # check whether the retriever output consist of 6 attributes including tokenized docs
       self.assertEqual(
           all(k in out for k in ('tokenized_doc_ids', 'tokenized_doc_attention_mask')					)  ,			a_					)  # check for doc token related keys in dictionary.






import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


_lowerCamelCase      :					List[str]					    =      pytest.mark.integration





@require_faiss
class        lowerCAmelCase__  (					__magic_name__			):



       '''simple docstring'''







       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             __A    =Dataset.from_dict({'''filename''': ['''my_name-train''' + '''_''' + str(lowercase__     ) for x in np.arange(3_0     ).tolist()]}     )
             return dset

       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             import faiss

             __A    =self._create_dummy_dataset()
             __A    =dset.map(
                 lambda lowercase__ ,   lowercase__     : {"vecs": i * np.ones(5 ,   dtype=np.floataa     )} ,   with_indices=lowercase__ ,   keep_in_memory=lowercase__     )
             __A    =dset.add_faiss_index('''vecs''' ,   batch_size=1_0_0 ,   metric_type=faiss.METRIC_INNER_PRODUCT     )
             __A    ,		__A    =dset.get_nearest_examples('''vecs''' ,   np.ones(5 ,   dtype=np.floataa     )     )
             self.assertEqual(examples['''filename'''][0] ,   '''my_name-train_29'''     )
             dset.drop_index('''vecs'''     )

       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             import faiss

             __A    =self._create_dummy_dataset()
             dset.add_faiss_index_from_external_arrays(
                 external_arrays=np.ones((3_0, 5)     ) * np.arange(3_0     ).reshape(-1 ,   1     ) ,   index_name='''vecs''' ,   batch_size=1_0_0 ,   metric_type=faiss.METRIC_INNER_PRODUCT ,   )
             __A    ,		__A    =dset.get_nearest_examples('''vecs''' ,   np.ones(5 ,   dtype=np.floataa     )     )
             self.assertEqual(examples['''filename'''][0] ,   '''my_name-train_29'''     )

       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             import faiss

             __A    =self._create_dummy_dataset()
             dset.add_faiss_index_from_external_arrays(
                 external_arrays=np.ones((3_0, 5)     ) * np.arange(3_0     ).reshape(-1 ,   1     ) ,   index_name='''vecs''' ,   metric_type=faiss.METRIC_INNER_PRODUCT ,   )

             # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
             # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
             # see https://bugs.python.org/issue14243 and
             # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
             with tempfile.NamedTemporaryFile(delete=lowercase__     ) as tmp_file:
                   dset.save_faiss_index('''vecs''' ,   tmp_file.name     )
                   dset.load_faiss_index('''vecs2''' ,   tmp_file.name     )
             os.unlink(tmp_file.name     )

             __A    ,		__A    =dset.get_nearest_examples('''vecs2''' ,   np.ones(5 ,   dtype=np.floataa     )     )
             self.assertEqual(examples['''filename'''][0] ,   '''my_name-train_29'''     )

       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             __A    =self._create_dummy_dataset()
             dset.add_faiss_index_from_external_arrays(
                 external_arrays=np.ones((3_0, 5)     ) * np.arange(3_0     ).reshape(-1 ,   1     ) ,   index_name='''vecs'''     )
             dset.drop_index('''vecs'''     )
             self.assertRaises(lowercase__ ,   partial(dset.get_nearest_examples ,   '''vecs2''' ,   np.ones(5 ,   dtype=np.floataa     )     )     )


       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             from elasticsearch import Elasticsearch

             __A    =self._create_dummy_dataset()
             with patch('''elasticsearch.Elasticsearch.search'''     ) as mocked_search, patch(
                 '''elasticsearch.client.IndicesClient.create'''     ) as mocked_index_create, patch('''elasticsearch.helpers.streaming_bulk'''     ) as mocked_bulk:
                   __A    ={'''acknowledged''': True}
                   mocked_bulk.return_value([(True, None)] * 3_0     )
                   __A    ={'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 2_9}]}}
                   __A    =Elasticsearch()

                   dset.add_elasticsearch_index('''filename''' ,   es_client=lowercase__     )
                   __A    ,		__A    =dset.get_nearest_examples('''filename''' ,   '''my_name-train_29'''     )
                   self.assertEqual(examples['''filename'''][0] ,   '''my_name-train_29'''     )





@require_faiss
class        lowerCAmelCase__  (					__magic_name__			):



       '''simple docstring'''







       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             import faiss

             __A    =FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT     )

             # add vectors
             index.add_vectors(np.eye(5 ,   dtype=np.floataa     )     )
             self.assertIsNotNone(index.faiss_index     )
             self.assertEqual(index.faiss_index.ntotal ,   5     )
             index.add_vectors(np.zeros((5, 5) ,   dtype=np.floataa     )     )
             self.assertEqual(index.faiss_index.ntotal ,   1_0     )

             # single query
             __A    =np.zeros(5 ,   dtype=np.floataa     )
             __A    =1
             __A    ,		__A    =index.search(lowercase__     )
             self.assertRaises(lowercase__ ,   index.search ,   query.reshape(-1 ,   1     )     )
             self.assertGreater(scores[0] ,   0     )
             self.assertEqual(indices[0] ,   1     )

             # batched queries
             __A    =np.eye(5 ,   dtype=np.floataa     )[::-1]
             __A    ,		__A    =index.search_batch(lowercase__     )
             self.assertRaises(lowercase__ ,   index.search_batch ,   queries[0]     )
             __A    =[scores[0] for scores in total_scores]
             __A    =[indices[0] for indices in total_indices]
             self.assertGreater(np.min(lowercase__     ) ,   0     )
             self.assertListEqual([4, 3, 2, 1, 0] ,   lowercase__     )

       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             import faiss

             __A    =FaissIndex(string_factory='''Flat'''     )
             index.add_vectors(np.eye(5 ,   dtype=np.floataa     )     )
             self.assertIsInstance(index.faiss_index ,   faiss.IndexFlat     )
             __A    =FaissIndex(string_factory='''LSH'''     )
             index.add_vectors(np.eye(5 ,   dtype=np.floataa     )     )
             self.assertIsInstance(index.faiss_index ,   faiss.IndexLSH     )
             with self.assertRaises(lowercase__     ):
                   __A    =FaissIndex(string_factory='''Flat''' ,   custom_index=faiss.IndexFlat(5     )     )

       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             import faiss

             __A    =faiss.IndexFlat(5     )
             __A    =FaissIndex(custom_index=lowercase__     )
             index.add_vectors(np.eye(5 ,   dtype=np.floataa     )     )
             self.assertIsInstance(index.faiss_index ,   faiss.IndexFlat     )


       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             import faiss

             __A    =FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT     )
             index.add_vectors(np.eye(5 ,   dtype=np.floataa     )     )

             # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
             # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
             # see https://bugs.python.org/issue14243 and
             # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
             with tempfile.NamedTemporaryFile(delete=lowercase__     ) as tmp_file:
                   index.save(tmp_file.name     )
                   __A    =FaissIndex.load(tmp_file.name     )
             os.unlink(tmp_file.name     )

             __A    =np.zeros(5 ,   dtype=np.floataa     )
             __A    =1
             __A    ,		__A    =index.search(lowercase__     )
             self.assertGreater(scores[0] ,   0     )
             self.assertEqual(indices[0] ,   1     )





@require_faiss
def      A__					(			__A  :						Dict			)					->List[Any]:
      import faiss

      __A    =FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT			)
      index.add_vectors(np.eye(5     ,				dtype=np.floataa			)			)

      __A    ='''index.faiss'''
      __A    =F'''mock://{index_name}'''
      index.save(__A     ,				storage_options=mockfs.storage_options			)
      __A    =FaissIndex.load(__A     ,				storage_options=mockfs.storage_options			)

      __A    =np.zeros(5     ,				dtype=np.floataa			)
      __A    =1
      __A    ,		__A    =index.search(__A			)
      assert scores[0] > 0
      assert indices[0] == 1





@require_elasticsearch
class        lowerCAmelCase__  (					__magic_name__			):



       '''simple docstring'''







       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             from elasticsearch import Elasticsearch

             with patch('''elasticsearch.Elasticsearch.search'''     ) as mocked_search, patch(
                 '''elasticsearch.client.IndicesClient.create'''     ) as mocked_index_create, patch('''elasticsearch.helpers.streaming_bulk'''     ) as mocked_bulk:
                   __A    =Elasticsearch()
                   __A    ={'''acknowledged''': True}
                   __A    =ElasticSearchIndex(es_client=lowercase__     )
                   mocked_bulk.return_value([(True, None)] * 3     )
                   index.add_documents(['''foo''', '''bar''', '''foobar''']     )

                   # single query
                   __A    ='''foo'''
                   __A    ={'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 0}]}}
                   __A    ,		__A    =index.search(lowercase__     )
                   self.assertEqual(scores[0] ,   1     )
                   self.assertEqual(indices[0] ,   0     )

                   # single query with timeout
                   __A    ='''foo'''
                   __A    ={'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 0}]}}
                   __A    ,		__A    =index.search(lowercase__ ,   request_timeout=3_0     )
                   self.assertEqual(scores[0] ,   1     )
                   self.assertEqual(indices[0] ,   0     )

                   # batched queries
                   __A    =['''foo''', '''bar''', '''foobar''']
                   __A    ={'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 1}]}}
                   __A    ,		__A    =index.search_batch(lowercase__     )
                   __A    =[scores[0] for scores in total_scores]
                   __A    =[indices[0] for indices in total_indices]
                   self.assertGreater(np.min(lowercase__     ) ,   0     )
                   self.assertListEqual([1, 1, 1] ,   lowercase__     )

                   # batched queries with timeout
                   __A    =['''foo''', '''bar''', '''foobar''']
                   __A    ={'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 1}]}}
                   __A    ,		__A    =index.search_batch(lowercase__ ,   request_timeout=3_0     )
                   __A    =[scores[0] for scores in total_scores]
                   __A    =[indices[0] for indices in total_indices]
                   self.assertGreater(np.min(lowercase__     ) ,   0     )
                   self.assertListEqual([1, 1, 1] ,   lowercase__     )


