



"""simple docstring"""







import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


lowercase__		:    Optional[int]				   =     pytest.mark.integration







@require_faiss
class 				UpperCamelCase__       (     lowercase_  ):




      """simple docstring"""




      def 					SCREAMING_SNAKE_CASE__			(				self    :				Union[str, Any]	):
        lowerCAmelCase_       :    str								=			Dataset.from_dict({'filename': ['my_name-train' + '_' + str(SCREAMING_SNAKE_CASE_	) for x in np.arange(3_0	).tolist()]}	)
        return dset



      def 					SCREAMING_SNAKE_CASE__			(				self    :				List[Any]	):
        import faiss

        lowerCAmelCase_       :    Dataset								=			self._create_dummy_dataset()
        lowerCAmelCase_       :    Any								=			dset.map(
            lambda SCREAMING_SNAKE_CASE_   ,      SCREAMING_SNAKE_CASE_	: {"vecs": i * np.ones(5   ,      dtype=np.floataa	)}   ,      with_indices=SCREAMING_SNAKE_CASE_   ,      keep_in_memory=SCREAMING_SNAKE_CASE_	)
        lowerCAmelCase_       :    List[Any]								=			dset.add_faiss_index('vecs'   ,      batch_size=1_0_0   ,      metric_type=faiss.METRIC_INNER_PRODUCT	)
        lowerCAmelCase_       :    str								=			dset.get_nearest_examples('vecs'   ,      np.ones(5   ,      dtype=np.floataa	)	)
        self.assertEqual(examples['filename'][0]   ,      'my_name-train_29'	)
        dset.drop_index('vecs'	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Optional[int]	):
        import faiss

        lowerCAmelCase_       :    Dataset								=			self._create_dummy_dataset()
        dset.add_faiss_index_from_external_arrays(
            external_arrays=np.ones((3_0, 5)	) * np.arange(3_0	).reshape(-1   ,      1	)   ,      index_name='vecs'   ,      batch_size=1_0_0   ,      metric_type=faiss.METRIC_INNER_PRODUCT   ,      )
        lowerCAmelCase_       :    Optional[Any]								=			dset.get_nearest_examples('vecs'   ,      np.ones(5   ,      dtype=np.floataa	)	)
        self.assertEqual(examples['filename'][0]   ,      'my_name-train_29'	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Union[str, Any]	):
        import faiss

        lowerCAmelCase_       :    Dataset								=			self._create_dummy_dataset()
        dset.add_faiss_index_from_external_arrays(
            external_arrays=np.ones((3_0, 5)	) * np.arange(3_0	).reshape(-1   ,      1	)   ,      index_name='vecs'   ,      metric_type=faiss.METRIC_INNER_PRODUCT   ,      )

        # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
        # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
        # see https://bugs.python.org/issue14243 and
        # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
        with tempfile.NamedTemporaryFile(delete=SCREAMING_SNAKE_CASE_	) as tmp_file:
          dset.save_faiss_index('vecs'   ,      tmp_file.name	)
          dset.load_faiss_index('vecs2'   ,      tmp_file.name	)
        os.unlink(tmp_file.name	)

        lowerCAmelCase_       :    int								=			dset.get_nearest_examples('vecs2'   ,      np.ones(5   ,      dtype=np.floataa	)	)
        self.assertEqual(examples['filename'][0]   ,      'my_name-train_29'	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Any	):
        lowerCAmelCase_       :    Dataset								=			self._create_dummy_dataset()
        dset.add_faiss_index_from_external_arrays(
            external_arrays=np.ones((3_0, 5)	) * np.arange(3_0	).reshape(-1   ,      1	)   ,      index_name='vecs'	)
        dset.drop_index('vecs'	)
        self.assertRaises(SCREAMING_SNAKE_CASE_   ,      partial(dset.get_nearest_examples   ,      'vecs2'   ,      np.ones(5   ,      dtype=np.floataa	)	)	)





      def 					SCREAMING_SNAKE_CASE__			(				self    :				Dict	):
        from elasticsearch import Elasticsearch

        lowerCAmelCase_       :    Dataset								=			self._create_dummy_dataset()
        with patch('elasticsearch.Elasticsearch.search'	) as mocked_search, patch(
            'elasticsearch.client.IndicesClient.create'	) as mocked_index_create, patch('elasticsearch.helpers.streaming_bulk'	) as mocked_bulk:
          lowerCAmelCase_       :    Any								=			{'acknowledged': True}
          mocked_bulk.return_value([(True, None)] * 3_0	)
          lowerCAmelCase_       :    List[str]								=			{'hits': {'hits': [{'_score': 1, '_id': 2_9}]}}
          lowerCAmelCase_       :    Union[str, Any]								=			Elasticsearch()

          dset.add_elasticsearch_index('filename'   ,      es_client=SCREAMING_SNAKE_CASE_	)
          lowerCAmelCase_       :    Union[str, Any]								=			dset.get_nearest_examples('filename'   ,      'my_name-train_29'	)
          self.assertEqual(examples['filename'][0]   ,      'my_name-train_29'	)







@require_faiss
class 				UpperCamelCase__       (     lowercase_  ):




      """simple docstring"""




      def 					SCREAMING_SNAKE_CASE__			(				self    :				str	):
        import faiss

        lowerCAmelCase_       :    Tuple								=			FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT	)

        # add vectors
        index.add_vectors(np.eye(5   ,      dtype=np.floataa	)	)
        self.assertIsNotNone(index.faiss_index	)
        self.assertEqual(index.faiss_index.ntotal   ,      5	)
        index.add_vectors(np.zeros((5, 5)   ,      dtype=np.floataa	)	)
        self.assertEqual(index.faiss_index.ntotal   ,      1_0	)

        # single query
        lowerCAmelCase_       :    Optional[int]								=			np.zeros(5   ,      dtype=np.floataa	)
        lowerCAmelCase_       :    List[Any]								=			1
        lowerCAmelCase_       :    int								=			index.search(SCREAMING_SNAKE_CASE_	)
        self.assertRaises(SCREAMING_SNAKE_CASE_   ,      index.search   ,      query.reshape(-1   ,      1	)	)
        self.assertGreater(scores[0]   ,      0	)
        self.assertEqual(indices[0]   ,      1	)

        # batched queries
        lowerCAmelCase_       :    int								=			np.eye(5   ,      dtype=np.floataa	)[::-1]
        lowerCAmelCase_       :    int								=			index.search_batch(SCREAMING_SNAKE_CASE_	)
        self.assertRaises(SCREAMING_SNAKE_CASE_   ,      index.search_batch   ,      queries[0]	)
        lowerCAmelCase_       :    List[str]								=			[scores[0] for scores in total_scores]
        lowerCAmelCase_       :    int								=			[indices[0] for indices in total_indices]
        self.assertGreater(np.min(SCREAMING_SNAKE_CASE_	)   ,      0	)
        self.assertListEqual([4, 3, 2, 1, 0]   ,      SCREAMING_SNAKE_CASE_	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Union[str, Any]	):
        import faiss

        lowerCAmelCase_       :    Tuple								=			FaissIndex(string_factory='Flat'	)
        index.add_vectors(np.eye(5   ,      dtype=np.floataa	)	)
        self.assertIsInstance(index.faiss_index   ,      faiss.IndexFlat	)
        lowerCAmelCase_       :    Optional[Any]								=			FaissIndex(string_factory='LSH'	)
        index.add_vectors(np.eye(5   ,      dtype=np.floataa	)	)
        self.assertIsInstance(index.faiss_index   ,      faiss.IndexLSH	)
        with self.assertRaises(SCREAMING_SNAKE_CASE_	):
          lowerCAmelCase_       :    Dict								=			FaissIndex(string_factory='Flat'   ,      custom_index=faiss.IndexFlat(5	)	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Dict	):
        import faiss

        lowerCAmelCase_       :    List[str]								=			faiss.IndexFlat(5	)
        lowerCAmelCase_       :    int								=			FaissIndex(custom_index=SCREAMING_SNAKE_CASE_	)
        index.add_vectors(np.eye(5   ,      dtype=np.floataa	)	)
        self.assertIsInstance(index.faiss_index   ,      faiss.IndexFlat	)





      def 					SCREAMING_SNAKE_CASE__			(				self    :				str	):
        import faiss

        lowerCAmelCase_       :    List[str]								=			FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT	)
        index.add_vectors(np.eye(5   ,      dtype=np.floataa	)	)

        # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
        # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
        # see https://bugs.python.org/issue14243 and
        # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
        with tempfile.NamedTemporaryFile(delete=SCREAMING_SNAKE_CASE_	) as tmp_file:
          index.save(tmp_file.name	)
          lowerCAmelCase_       :    List[Any]								=			FaissIndex.load(tmp_file.name	)
        os.unlink(tmp_file.name	)

        lowerCAmelCase_       :    Optional[int]								=			np.zeros(5   ,      dtype=np.floataa	)
        lowerCAmelCase_       :    str								=			1
        lowerCAmelCase_       :    List[Any]								=			index.search(SCREAMING_SNAKE_CASE_	)
        self.assertGreater(scores[0]   ,      0	)
        self.assertEqual(indices[0]   ,      1	)




@require_faiss
def     UpperCamelCase_   (   lowerCAmelCase__				:    int						)						->       Tuple:

  """simple docstring"""




  import faiss

  lowerCAmelCase_       :    Any								=			FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT						)
  index.add_vectors(np.eye(5   ,       dtype=np.floataa						)						)

  lowerCAmelCase_       :    List[str]								=			'index.faiss'
  lowerCAmelCase_       :    Union[str, Any]								=			f"mock://{index_name}"
  index.save(lowerCAmelCase__   ,       storage_options=mockfs.storage_options						)
  lowerCAmelCase_       :    int								=			FaissIndex.load(lowerCAmelCase__   ,       storage_options=mockfs.storage_options						)

  lowerCAmelCase_       :    str								=			np.zeros(5   ,       dtype=np.floataa						)
  lowerCAmelCase_       :    List[str]								=			1
  lowerCAmelCase_       :    int								=			index.search(lowerCAmelCase__						)
  assert scores[0] > 0
  assert indices[0] == 1







@require_elasticsearch
class 				UpperCamelCase__       (     lowercase_  ):




      """simple docstring"""




      def 					SCREAMING_SNAKE_CASE__			(				self    :				Dict	):
        from elasticsearch import Elasticsearch

        with patch('elasticsearch.Elasticsearch.search'	) as mocked_search, patch(
            'elasticsearch.client.IndicesClient.create'	) as mocked_index_create, patch('elasticsearch.helpers.streaming_bulk'	) as mocked_bulk:
          lowerCAmelCase_       :    Union[str, Any]								=			Elasticsearch()
          lowerCAmelCase_       :    Dict								=			{'acknowledged': True}
          lowerCAmelCase_       :    str								=			ElasticSearchIndex(es_client=SCREAMING_SNAKE_CASE_	)
          mocked_bulk.return_value([(True, None)] * 3	)
          index.add_documents(['foo', 'bar', 'foobar']	)

          # single query
          lowerCAmelCase_       :    int								=			'foo'
          lowerCAmelCase_       :    Optional[int]								=			{'hits': {'hits': [{'_score': 1, '_id': 0}]}}
          lowerCAmelCase_       :    int								=			index.search(SCREAMING_SNAKE_CASE_	)
          self.assertEqual(scores[0]   ,      1	)
          self.assertEqual(indices[0]   ,      0	)

          # single query with timeout
          lowerCAmelCase_       :    Dict								=			'foo'
          lowerCAmelCase_       :    Union[str, Any]								=			{'hits': {'hits': [{'_score': 1, '_id': 0}]}}
          lowerCAmelCase_       :    Dict								=			index.search(SCREAMING_SNAKE_CASE_   ,      request_timeout=3_0	)
          self.assertEqual(scores[0]   ,      1	)
          self.assertEqual(indices[0]   ,      0	)

          # batched queries
          lowerCAmelCase_       :    int								=			['foo', 'bar', 'foobar']
          lowerCAmelCase_       :    Any								=			{'hits': {'hits': [{'_score': 1, '_id': 1}]}}
          lowerCAmelCase_       :    Union[str, Any]								=			index.search_batch(SCREAMING_SNAKE_CASE_	)
          lowerCAmelCase_       :    int								=			[scores[0] for scores in total_scores]
          lowerCAmelCase_       :    Dict								=			[indices[0] for indices in total_indices]
          self.assertGreater(np.min(SCREAMING_SNAKE_CASE_	)   ,      0	)
          self.assertListEqual([1, 1, 1]   ,      SCREAMING_SNAKE_CASE_	)

          # batched queries with timeout
          lowerCAmelCase_       :    Tuple								=			['foo', 'bar', 'foobar']
          lowerCAmelCase_       :    Optional[int]								=			{'hits': {'hits': [{'_score': 1, '_id': 1}]}}
          lowerCAmelCase_       :    Any								=			index.search_batch(SCREAMING_SNAKE_CASE_   ,      request_timeout=3_0	)
          lowerCAmelCase_       :    Optional[int]								=			[scores[0] for scores in total_scores]
          lowerCAmelCase_       :    int								=			[indices[0] for indices in total_indices]
          self.assertGreater(np.min(SCREAMING_SNAKE_CASE_	)   ,      0	)
          self.assertListEqual([1, 1, 1]   ,      SCREAMING_SNAKE_CASE_	)










"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


lowercase__		:    Dict				   =     {
    """configuration_falcon""": ["""FALCON_PRETRAINED_CONFIG_ARCHIVE_MAP""", """FalconConfig"""],
}

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__		:    str				   =     [
         """FALCON_PRETRAINED_MODEL_ARCHIVE_LIST""",
         """FalconForCausalLM""",
         """FalconModel""",
         """FalconPreTrainedModel""",
         """FalconForSequenceClassification""",
         """FalconForTokenClassification""",
         """FalconForQuestionAnswering""",
     ]


if TYPE_CHECKING:
     from .configuration_falcon import FALCON_PRETRAINED_CONFIG_ARCHIVE_MAP, FalconConfig

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_falcon import (
              FALCON_PRETRAINED_MODEL_ARCHIVE_LIST,
              FalconForCausalLM,
              FalconForQuestionAnswering,
              FalconForSequenceClassification,
              FalconForTokenClassification,
              FalconModel,
              FalconPreTrainedModel,
          )


else:
     import sys

     lowercase__		:    Dict				   =     _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





