




'''simple docstring'''







import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


A									=pytest.mark.integration


@require_faiss
class     _a						(			__a				):
       def 					A   (			self		:			str       ):



           '''simple docstring'''

           UpperCAmelCase     =	Dataset.from_dict({'''filename''': ['''my_name-train''' + '''_''' + str(lowercase       ) for x in np.arange(30       ).tolist()]}       )
           return dset
       def 					A   (			self		:			Any       ):



           '''simple docstring'''

           import faiss

           UpperCAmelCase     =	self._create_dummy_dataset()
           UpperCAmelCase     =	dset.map(
               lambda lowercase     ,       lowercase       : {"vecs": i * np.ones(5     ,       dtype=np.floataa       )}     ,       with_indices=lowercase     ,       keep_in_memory=lowercase       )
           UpperCAmelCase     =	dset.add_faiss_index('''vecs'''     ,       batch_size=100     ,       metric_type=faiss.METRIC_INNER_PRODUCT       )
           UpperCAmelCase ,  UpperCAmelCase     =	dset.get_nearest_examples('''vecs'''     ,       np.ones(5     ,       dtype=np.floataa       )       )
           self.assertEqual(examples['''filename'''][0]     ,       '''my_name-train_29'''       )
           dset.drop_index('''vecs'''       )
       def 					A   (			self		:			int       ):



           '''simple docstring'''

           import faiss

           UpperCAmelCase     =	self._create_dummy_dataset()
           dset.add_faiss_index_from_external_arrays(
               external_arrays=np.ones((30, 5)       ) * np.arange(30       ).reshape(-1     ,       1       )     ,       index_name='''vecs'''     ,       batch_size=100     ,       metric_type=faiss.METRIC_INNER_PRODUCT     ,       )
           UpperCAmelCase ,  UpperCAmelCase     =	dset.get_nearest_examples('''vecs'''     ,       np.ones(5     ,       dtype=np.floataa       )       )
           self.assertEqual(examples['''filename'''][0]     ,       '''my_name-train_29'''       )
       def 					A   (			self		:			List[Any]       ):



           '''simple docstring'''

           import faiss

           UpperCAmelCase     =	self._create_dummy_dataset()
           dset.add_faiss_index_from_external_arrays(
               external_arrays=np.ones((30, 5)       ) * np.arange(30       ).reshape(-1     ,       1       )     ,       index_name='''vecs'''     ,       metric_type=faiss.METRIC_INNER_PRODUCT     ,       )

           # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
           # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
           # see https://bugs.python.org/issue14243 and
           # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
           with tempfile.NamedTemporaryFile(delete=lowercase       ) as tmp_file:
               dset.save_faiss_index('''vecs'''     ,       tmp_file.name       )
               dset.load_faiss_index('''vecs2'''     ,       tmp_file.name       )
           os.unlink(tmp_file.name       )

           UpperCAmelCase ,  UpperCAmelCase     =	dset.get_nearest_examples('''vecs2'''     ,       np.ones(5     ,       dtype=np.floataa       )       )
           self.assertEqual(examples['''filename'''][0]     ,       '''my_name-train_29'''       )
       def 					A   (			self		:			Dict       ):



           '''simple docstring'''

           UpperCAmelCase     =	self._create_dummy_dataset()
           dset.add_faiss_index_from_external_arrays(
               external_arrays=np.ones((30, 5)       ) * np.arange(30       ).reshape(-1     ,       1       )     ,       index_name='''vecs'''       )
           dset.drop_index('''vecs'''       )
           self.assertRaises(lowercase     ,       partial(dset.get_nearest_examples     ,       '''vecs2'''     ,       np.ones(5     ,       dtype=np.floataa       )       )       )







       def 					A   (			self		:			List[str]       ):



           '''simple docstring'''

           from elasticsearch import Elasticsearch

           UpperCAmelCase     =	self._create_dummy_dataset()
           with patch('''elasticsearch.Elasticsearch.search'''       ) as mocked_search, patch(
               '''elasticsearch.client.IndicesClient.create'''       ) as mocked_index_create, patch('''elasticsearch.helpers.streaming_bulk'''       ) as mocked_bulk:
               UpperCAmelCase     =	{'''acknowledged''': True}
               mocked_bulk.return_value([(True, None)] * 30       )
               UpperCAmelCase     =	{'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 29}]}}
               UpperCAmelCase     =	Elasticsearch()

               dset.add_elasticsearch_index('''filename'''     ,       es_client=lowercase       )
               UpperCAmelCase ,  UpperCAmelCase     =	dset.get_nearest_examples('''filename'''     ,       '''my_name-train_29'''       )
               self.assertEqual(examples['''filename'''][0]     ,       '''my_name-train_29'''       )






@require_faiss
class     _a						(			__a				):
       def 					A   (			self		:			Any       ):



           '''simple docstring'''

           import faiss

           UpperCAmelCase     =	FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT       )

           # add vectors
           index.add_vectors(np.eye(5     ,       dtype=np.floataa       )       )
           self.assertIsNotNone(index.faiss_index       )
           self.assertEqual(index.faiss_index.ntotal     ,       5       )
           index.add_vectors(np.zeros((5, 5)     ,       dtype=np.floataa       )       )
           self.assertEqual(index.faiss_index.ntotal     ,       10       )

           # single query
           UpperCAmelCase     =	np.zeros(5     ,       dtype=np.floataa       )
           UpperCAmelCase     =	1
           UpperCAmelCase ,  UpperCAmelCase     =	index.search(lowercase       )
           self.assertRaises(lowercase     ,       index.search     ,       query.reshape(-1     ,       1       )       )
           self.assertGreater(scores[0]     ,       0       )
           self.assertEqual(indices[0]     ,       1       )

           # batched queries
           UpperCAmelCase     =	np.eye(5     ,       dtype=np.floataa       )[::-1]
           UpperCAmelCase ,  UpperCAmelCase     =	index.search_batch(lowercase       )
           self.assertRaises(lowercase     ,       index.search_batch     ,       queries[0]       )
           UpperCAmelCase     =	[scores[0] for scores in total_scores]
           UpperCAmelCase     =	[indices[0] for indices in total_indices]
           self.assertGreater(np.min(lowercase       )     ,       0       )
           self.assertListEqual([4, 3, 2, 1, 0]     ,       lowercase       )
       def 					A   (			self		:			Optional[int]       ):



           '''simple docstring'''

           import faiss

           UpperCAmelCase     =	FaissIndex(string_factory='''Flat'''       )
           index.add_vectors(np.eye(5     ,       dtype=np.floataa       )       )
           self.assertIsInstance(index.faiss_index     ,       faiss.IndexFlat       )
           UpperCAmelCase     =	FaissIndex(string_factory='''LSH'''       )
           index.add_vectors(np.eye(5     ,       dtype=np.floataa       )       )
           self.assertIsInstance(index.faiss_index     ,       faiss.IndexLSH       )
           with self.assertRaises(lowercase       ):
               UpperCAmelCase     =	FaissIndex(string_factory='''Flat'''     ,       custom_index=faiss.IndexFlat(5       )       )
       def 					A   (			self		:			Optional[Any]       ):



           '''simple docstring'''

           import faiss

           UpperCAmelCase     =	faiss.IndexFlat(5       )
           UpperCAmelCase     =	FaissIndex(custom_index=lowercase       )
           index.add_vectors(np.eye(5     ,       dtype=np.floataa       )       )
           self.assertIsInstance(index.faiss_index     ,       faiss.IndexFlat       )







       def 					A   (			self		:			Union[str, Any]       ):



           '''simple docstring'''

           import faiss

           UpperCAmelCase     =	FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT       )
           index.add_vectors(np.eye(5     ,       dtype=np.floataa       )       )

           # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
           # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
           # see https://bugs.python.org/issue14243 and
           # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
           with tempfile.NamedTemporaryFile(delete=lowercase       ) as tmp_file:
               index.save(tmp_file.name       )
               UpperCAmelCase     =	FaissIndex.load(tmp_file.name       )
           os.unlink(tmp_file.name       )

           UpperCAmelCase     =	np.zeros(5     ,       dtype=np.floataa       )
           UpperCAmelCase     =	1
           UpperCAmelCase ,  UpperCAmelCase     =	index.search(lowercase       )
           self.assertGreater(scores[0]     ,       0       )
           self.assertEqual(indices[0]     ,       1       )






@require_faiss
def 		snake_case_							(_a			:							Any  ):
    import faiss

    UpperCAmelCase     =	FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT  )
    index.add_vectors(np.eye(5					,     dtype=np.floataa  )  )

    UpperCAmelCase     =	'''index.faiss'''
    UpperCAmelCase     =	F"mock://{index_name}"
    index.save(_a					,     storage_options=mockfs.storage_options  )
    UpperCAmelCase     =	FaissIndex.load(_a					,     storage_options=mockfs.storage_options  )

    UpperCAmelCase     =	np.zeros(5					,     dtype=np.floataa  )
    UpperCAmelCase     =	1
    UpperCAmelCase ,  UpperCAmelCase     =	index.search(_a  )
    assert scores[0] > 0
    assert indices[0] == 1







@require_elasticsearch
class     _a						(			__a				):
       def 					A   (			self		:			Any       ):



           '''simple docstring'''

           from elasticsearch import Elasticsearch

           with patch('''elasticsearch.Elasticsearch.search'''       ) as mocked_search, patch(
               '''elasticsearch.client.IndicesClient.create'''       ) as mocked_index_create, patch('''elasticsearch.helpers.streaming_bulk'''       ) as mocked_bulk:
               UpperCAmelCase     =	Elasticsearch()
               UpperCAmelCase     =	{'''acknowledged''': True}
               UpperCAmelCase     =	ElasticSearchIndex(es_client=lowercase       )
               mocked_bulk.return_value([(True, None)] * 3       )
               index.add_documents(['''foo''', '''bar''', '''foobar''']       )

               # single query
               UpperCAmelCase     =	'''foo'''
               UpperCAmelCase     =	{'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 0}]}}
               UpperCAmelCase ,  UpperCAmelCase     =	index.search(lowercase       )
               self.assertEqual(scores[0]     ,       1       )
               self.assertEqual(indices[0]     ,       0       )

               # single query with timeout
               UpperCAmelCase     =	'''foo'''
               UpperCAmelCase     =	{'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 0}]}}
               UpperCAmelCase ,  UpperCAmelCase     =	index.search(lowercase     ,       request_timeout=30       )
               self.assertEqual(scores[0]     ,       1       )
               self.assertEqual(indices[0]     ,       0       )

               # batched queries
               UpperCAmelCase     =	['''foo''', '''bar''', '''foobar''']
               UpperCAmelCase     =	{'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 1}]}}
               UpperCAmelCase ,  UpperCAmelCase     =	index.search_batch(lowercase       )
               UpperCAmelCase     =	[scores[0] for scores in total_scores]
               UpperCAmelCase     =	[indices[0] for indices in total_indices]
               self.assertGreater(np.min(lowercase       )     ,       0       )
               self.assertListEqual([1, 1, 1]     ,       lowercase       )

               # batched queries with timeout
               UpperCAmelCase     =	['''foo''', '''bar''', '''foobar''']
               UpperCAmelCase     =	{'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 1}]}}
               UpperCAmelCase ,  UpperCAmelCase     =	index.search_batch(lowercase     ,       request_timeout=30       )
               UpperCAmelCase     =	[scores[0] for scores in total_scores]
               UpperCAmelCase     =	[indices[0] for indices in total_indices]
               self.assertGreater(np.min(lowercase       )     ,       0       )
               self.assertListEqual([1, 1, 1]     ,       lowercase       )










'''simple docstring'''







from typing import Union

from ..utils import add_end_docstrings, is_torch_available, is_vision_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_vision_available():
   from PIL import Image

   from ..image_utils import load_image

if is_torch_available():
   from ..models.auto.modeling_auto import MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING

A									=logging.get_logger(__name__)


@add_end_docstrings(__a				)
class     _a						(			__a				):
       def __init__(			self		:			Optional[int]     ,       *lowercase		:			Any     ,       **lowercase		:			Optional[int]       ):



           '''simple docstring'''

           super().__init__(*lowercase     ,       **lowercase       )
           self.check_model_type(lowercase       )
       def 					A   (			self		:			List[str]     ,       lowercase		:			str=None     ,       lowercase		:			List[str]=None     ,       lowercase		:			List[str]=None     ,       **lowercase		:			List[str]       ):



           '''simple docstring'''

           UpperCAmelCase ,  UpperCAmelCase     =	{}, {}
           if padding is not None:
               UpperCAmelCase     =	padding
           if truncation is not None:
               UpperCAmelCase     =	truncation
           if top_k is not None:
               UpperCAmelCase     =	top_k
           return preprocess_params, {}, postprocess_params
       def __call__(			self		:			Optional[Any]     ,       lowercase		:			Union["Image.Image", str]     ,       lowercase		:			str = None     ,       **lowercase		:			Optional[int]       ):



           '''simple docstring'''

           if isinstance(lowercase     ,       (Image.Image, str)       ) and isinstance(lowercase     ,       lowercase       ):
               UpperCAmelCase     =	{'''image''': image, '''question''': question}
           else:
               UpperCAmelCase     =	image
           UpperCAmelCase     =	super().__call__(lowercase     ,       **lowercase       )
           return results
       def 					A   (			self		:			List[Any]     ,       lowercase		:			List[str]     ,       lowercase		:			Any=False     ,       lowercase		:			Any=False       ):



           '''simple docstring'''

           UpperCAmelCase     =	load_image(inputs['''image''']       )
           UpperCAmelCase     =	self.tokenizer(
               inputs['''question''']     ,       return_tensors=self.framework     ,       padding=lowercase     ,       truncation=lowercase       )
           UpperCAmelCase     =	self.image_processor(images=lowercase     ,       return_tensors=self.framework       )
           model_inputs.update(lowercase       )
           return model_inputs
       def 					A   (			self		:			Dict     ,       lowercase		:			Optional[Any]       ):



           '''simple docstring'''

           UpperCAmelCase     =	self.model(**lowercase       )
           return model_outputs







       def 					A   (			self		:			Union[str, Any]     ,       lowercase		:			int     ,       lowercase		:			Optional[Any]=5       ):



           '''simple docstring'''

           if top_k > self.model.config.num_labels:
               UpperCAmelCase     =	self.model.config.num_labels

           if self.framework == "pt":
               UpperCAmelCase     =	model_outputs.logits.sigmoid()[0]
               UpperCAmelCase ,  UpperCAmelCase     =	probs.topk(lowercase       )
           else:
               raise ValueError(f"Unsupported framework: {self.framework}"       )

           UpperCAmelCase     =	scores.tolist()
           UpperCAmelCase     =	ids.tolist()
           return [{"score": score, "answer": self.model.config.idalabel[_id]} for score, _id in zip(lowercase     ,       lowercase       )]




