
"""simple docstring"""




A__   :					int							=   {
    'Pillow': 'Pillow<10.0.0',
    'accelerate': 'accelerate>=0.20.3',
    'av': 'av==9.2.0',
    'beautifulsoup4': 'beautifulsoup4',
    'black': 'black~=23.1',
    'codecarbon': 'codecarbon==1.2.0',
    'cookiecutter': 'cookiecutter==1.7.3',
    'dataclasses': 'dataclasses',
    'datasets': 'datasets!=2.5.0',
    'decord': 'decord==0.6.0',
    'deepspeed': 'deepspeed>=0.9.3',
    'diffusers': 'diffusers',
    'dill': 'dill<0.3.5',
    'evaluate': 'evaluate>=0.2.0',
    'fairscale': 'fairscale>0.3',
    'faiss-cpu': 'faiss-cpu',
    'fastapi': 'fastapi',
    'filelock': 'filelock',
    'flax': 'flax>=0.4.1,<=0.7.0',
    'ftfy': 'ftfy',
    'fugashi': 'fugashi>=1.0',
    'GitPython': 'GitPython<3.1.19',
    'hf-doc-builder': 'hf-doc-builder>=0.3.0',
    'huggingface-hub': 'huggingface-hub>=0.14.1,<1.0',
    'importlib_metadata': 'importlib_metadata',
    'ipadic': 'ipadic>=1.0.0,<2.0',
    'isort': 'isort>=5.5.4',
    'jax': 'jax>=0.2.8,!=0.3.2,<=0.4.13',
    'jaxlib': 'jaxlib>=0.1.65,<=0.4.13',
    'jieba': 'jieba',
    'kenlm': 'kenlm',
    'keras-nlp': 'keras-nlp>=0.3.1',
    'librosa': 'librosa',
    'nltk': 'nltk',
    'natten': 'natten>=0.14.6',
    'numpy': 'numpy>=1.17',
    'onnxconverter-common': 'onnxconverter-common',
    'onnxruntime-tools': 'onnxruntime-tools>=1.4.2',
    'onnxruntime': 'onnxruntime>=1.4.0',
    'opencv-python': 'opencv-python',
    'optuna': 'optuna',
    'optax': 'optax>=0.0.8,<=0.1.4',
    'packaging': 'packaging>=20.0',
    'parameterized': 'parameterized',
    'phonemizer': 'phonemizer',
    'protobuf': 'protobuf',
    'psutil': 'psutil',
    'pyyaml': 'pyyaml>=5.1',
    'pydantic': 'pydantic<2',
    'pytest': 'pytest>=7.2.0',
    'pytest-timeout': 'pytest-timeout',
    'pytest-xdist': 'pytest-xdist',
    'python': 'python>=3.8.0',
    'ray[tune]': 'ray[tune]',
    'regex': 'regex!=2019.12.17',
    'requests': 'requests',
    'rhoknp': 'rhoknp>=1.1.0,<1.3.1',
    'rjieba': 'rjieba',
    'rouge-score': 'rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1',
    'ruff': 'ruff>=0.0.241,<=0.0.259',
    'sacrebleu': 'sacrebleu>=1.4.12,<2.0.0',
    'sacremoses': 'sacremoses',
    'safetensors': 'safetensors>=0.3.1',
    'sagemaker': 'sagemaker>=2.31.0',
    'scikit-learn': 'scikit-learn',
    'sentencepiece': 'sentencepiece>=0.1.91,!=0.1.92',
    'sigopt': 'sigopt',
    'starlette': 'starlette',
    'sudachipy': 'sudachipy>=0.6.6',
    'sudachidict_core': 'sudachidict_core>=20220729',
    'tensorflow-cpu': 'tensorflow-cpu>=2.6,<2.14',
    'tensorflow': 'tensorflow>=2.6,<2.14',
    'tensorflow-text': 'tensorflow-text<2.14',
    'tf2onnx': 'tf2onnx',
    'timeout-decorator': 'timeout-decorator',
    'timm': 'timm',
    'tokenizers': 'tokenizers>=0.11.1,!=0.11.3,<0.14',
    'torch': 'torch>=1.9,!=1.12.0',
    'torchaudio': 'torchaudio',
    'torchvision': 'torchvision',
    'pyctcdecode': 'pyctcdecode>=0.4.0',
    'tqdm': 'tqdm>=4.27',
    'unidic': 'unidic>=1.0.2',
    'unidic_lite': 'unidic_lite>=1.0.7',
    'urllib3': 'urllib3<2.0.0',
    'uvicorn': 'uvicorn',
}


"""simple docstring"""




import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


A__   :					Any							=   pytest.mark.integration


@require_faiss
class 			__magic_name__       (       SCREAMING_SNAKE_CASE__ ):
       def 							lowercase_							(     self       )     ->	List[Any]:





         """simple docstring"""




         _lowercase:			str       =     Dataset.from_dict({'''filename''': ['''my_name-train''' + '''_''' + str(A_       ) for x in np.arange(30       ).tolist()]}       )
         return dset
       def 							lowercase_							(     self       )     ->	int:





         """simple docstring"""




         import faiss

         _lowercase:			Dataset       =     self._create_dummy_dataset()
         _lowercase:			List[str]       =     dset.map(
             lambda A_ ,				A_       : {"vecs": i * np.ones(5 ,				dtype=np.floataa       )} ,				with_indices=A_ ,				keep_in_memory=A_       )
         _lowercase:			Dict       =     dset.add_faiss_index('''vecs''' ,				batch_size=100 ,				metric_type=faiss.METRIC_INNER_PRODUCT       )
         _lowercase ,				_lowercase:			Union[str, Any]       =     dset.get_nearest_examples('''vecs''' ,				np.ones(5 ,				dtype=np.floataa       )       )
         self.assertEqual(examples['''filename'''][0] ,				'''my_name-train_29'''       )
         dset.drop_index('''vecs'''       )
       def 							lowercase_							(     self       )     ->	List[Any]:





         """simple docstring"""




         import faiss

         _lowercase:			Dataset       =     self._create_dummy_dataset()
         dset.add_faiss_index_from_external_arrays(
             external_arrays=np.ones((30, 5)       ) * np.arange(30       ).reshape(-1 ,				1       ) ,				index_name='''vecs''' ,				batch_size=100 ,				metric_type=faiss.METRIC_INNER_PRODUCT ,				)
         _lowercase ,				_lowercase:			Dict       =     dset.get_nearest_examples('''vecs''' ,				np.ones(5 ,				dtype=np.floataa       )       )
         self.assertEqual(examples['''filename'''][0] ,				'''my_name-train_29'''       )
       def 							lowercase_							(     self       )     ->	List[str]:





         """simple docstring"""




         import faiss

         _lowercase:			Dataset       =     self._create_dummy_dataset()
         dset.add_faiss_index_from_external_arrays(
             external_arrays=np.ones((30, 5)       ) * np.arange(30       ).reshape(-1 ,				1       ) ,				index_name='''vecs''' ,				metric_type=faiss.METRIC_INNER_PRODUCT ,				)

         # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
         # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
         # see https://bugs.python.org/issue14243 and
         # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
         with tempfile.NamedTemporaryFile(delete=A_       ) as tmp_file:
           dset.save_faiss_index('''vecs''' ,				tmp_file.name       )
           dset.load_faiss_index('''vecs2''' ,				tmp_file.name       )
         os.unlink(tmp_file.name       )

         _lowercase ,				_lowercase:			Dict       =     dset.get_nearest_examples('''vecs2''' ,				np.ones(5 ,				dtype=np.floataa       )       )
         self.assertEqual(examples['''filename'''][0] ,				'''my_name-train_29'''       )
       def 							lowercase_							(     self       )     ->	int:





         """simple docstring"""




         _lowercase:			Dataset       =     self._create_dummy_dataset()
         dset.add_faiss_index_from_external_arrays(
             external_arrays=np.ones((30, 5)       ) * np.arange(30       ).reshape(-1 ,				1       ) ,				index_name='''vecs'''       )
         dset.drop_index('''vecs'''       )
         self.assertRaises(A_ ,				partial(dset.get_nearest_examples ,				'''vecs2''' ,				np.ones(5 ,				dtype=np.floataa       )       )       )



       def 							lowercase_							(     self       )     ->	List[Any]:





         """simple docstring"""




         from elasticsearch import Elasticsearch

         _lowercase:			Dataset       =     self._create_dummy_dataset()
         with patch('''elasticsearch.Elasticsearch.search'''       ) as mocked_search, patch(
             '''elasticsearch.client.IndicesClient.create'''       ) as mocked_index_create, patch('''elasticsearch.helpers.streaming_bulk'''       ) as mocked_bulk:
           _lowercase:			List[Any]       =     {'''acknowledged''': True}
           mocked_bulk.return_value([(True, None)] * 30       )
           _lowercase:			List[str]       =     {'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 29}]}}
           _lowercase:			int       =     Elasticsearch()

           dset.add_elasticsearch_index('''filename''' ,				es_client=A_       )
           _lowercase ,				_lowercase:			Dict       =     dset.get_nearest_examples('''filename''' ,				'''my_name-train_29'''       )
           self.assertEqual(examples['''filename'''][0] ,				'''my_name-train_29'''       )


@require_faiss
class 			__magic_name__       (       SCREAMING_SNAKE_CASE__ ):
       def 							lowercase_							(     self       )     ->	Any:





         """simple docstring"""




         import faiss

         _lowercase:			str       =     FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT       )

         # add vectors
         index.add_vectors(np.eye(5 ,				dtype=np.floataa       )       )
         self.assertIsNotNone(index.faiss_index       )
         self.assertEqual(index.faiss_index.ntotal ,				5       )
         index.add_vectors(np.zeros((5, 5) ,				dtype=np.floataa       )       )
         self.assertEqual(index.faiss_index.ntotal ,				10       )

         # single query
         _lowercase:			List[Any]       =     np.zeros(5 ,				dtype=np.floataa       )
         _lowercase:			int       =     1
         _lowercase ,				_lowercase:			Optional[Any]       =     index.search(A_       )
         self.assertRaises(A_ ,				index.search ,				query.reshape(-1 ,				1       )       )
         self.assertGreater(scores[0] ,				0       )
         self.assertEqual(indices[0] ,				1       )

         # batched queries
         _lowercase:			Tuple       =     np.eye(5 ,				dtype=np.floataa       )[::-1]
         _lowercase ,				_lowercase:			str       =     index.search_batch(A_       )
         self.assertRaises(A_ ,				index.search_batch ,				queries[0]       )
         _lowercase:			Tuple       =     [scores[0] for scores in total_scores]
         _lowercase:			Union[str, Any]       =     [indices[0] for indices in total_indices]
         self.assertGreater(np.min(A_       ) ,				0       )
         self.assertListEqual([4, 3, 2, 1, 0] ,				A_       )
       def 							lowercase_							(     self       )     ->	Union[str, Any]:





         """simple docstring"""




         import faiss

         _lowercase:			Union[str, Any]       =     FaissIndex(string_factory='''Flat'''       )
         index.add_vectors(np.eye(5 ,				dtype=np.floataa       )       )
         self.assertIsInstance(index.faiss_index ,				faiss.IndexFlat       )
         _lowercase:			Dict       =     FaissIndex(string_factory='''LSH'''       )
         index.add_vectors(np.eye(5 ,				dtype=np.floataa       )       )
         self.assertIsInstance(index.faiss_index ,				faiss.IndexLSH       )
         with self.assertRaises(A_       ):
           _lowercase:			List[str]       =     FaissIndex(string_factory='''Flat''' ,				custom_index=faiss.IndexFlat(5       )       )
       def 							lowercase_							(     self       )     ->	List[str]:





         """simple docstring"""




         import faiss

         _lowercase:			Any       =     faiss.IndexFlat(5       )
         _lowercase:			List[Any]       =     FaissIndex(custom_index=A_       )
         index.add_vectors(np.eye(5 ,				dtype=np.floataa       )       )
         self.assertIsInstance(index.faiss_index ,				faiss.IndexFlat       )



       def 							lowercase_							(     self       )     ->	str:





         """simple docstring"""




         import faiss

         _lowercase:			Tuple       =     FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT       )
         index.add_vectors(np.eye(5 ,				dtype=np.floataa       )       )

         # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
         # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
         # see https://bugs.python.org/issue14243 and
         # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
         with tempfile.NamedTemporaryFile(delete=A_       ) as tmp_file:
           index.save(tmp_file.name       )
           _lowercase:			Optional[Any]       =     FaissIndex.load(tmp_file.name       )
         os.unlink(tmp_file.name       )

         _lowercase:			Optional[Any]       =     np.zeros(5 ,				dtype=np.floataa       )
         _lowercase:			Union[str, Any]       =     1
         _lowercase ,				_lowercase:			Tuple       =     index.search(A_       )
         self.assertGreater(scores[0] ,				0       )
         self.assertEqual(indices[0] ,				1       )




@require_faiss
def  _lowerCAmelCase		(		_UpperCamelCase						):






  """simple docstring"""



  import faiss

  _lowercase:			Dict       =     FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT						)
  index.add_vectors(np.eye(5 ,				dtype=np.floataa						)						)

  _lowercase:			Tuple       =     '''index.faiss'''
  _lowercase:			str       =     f'''mock://{index_name}'''
  index.save(_UpperCamelCase ,				storage_options=mockfs.storage_options						)
  _lowercase:			List[Any]       =     FaissIndex.load(_UpperCamelCase ,				storage_options=mockfs.storage_options						)

  _lowercase:			Union[str, Any]       =     np.zeros(5 ,				dtype=np.floataa						)
  _lowercase:			Dict       =     1
  _lowercase ,				_lowercase:			str       =     index.search(_UpperCamelCase						)
  assert scores[0] > 0
  assert indices[0] == 1







@require_elasticsearch
class 			__magic_name__       (       SCREAMING_SNAKE_CASE__ ):
       def 							lowercase_							(     self       )     ->	int:





         """simple docstring"""




         from elasticsearch import Elasticsearch

         with patch('''elasticsearch.Elasticsearch.search'''       ) as mocked_search, patch(
             '''elasticsearch.client.IndicesClient.create'''       ) as mocked_index_create, patch('''elasticsearch.helpers.streaming_bulk'''       ) as mocked_bulk:
           _lowercase:			int       =     Elasticsearch()
           _lowercase:			Tuple       =     {'''acknowledged''': True}
           _lowercase:			Tuple       =     ElasticSearchIndex(es_client=A_       )
           mocked_bulk.return_value([(True, None)] * 3       )
           index.add_documents(['''foo''', '''bar''', '''foobar''']       )

           # single query
           _lowercase:			Dict       =     '''foo'''
           _lowercase:			Union[str, Any]       =     {'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 0}]}}
           _lowercase ,				_lowercase:			Optional[Any]       =     index.search(A_       )
           self.assertEqual(scores[0] ,				1       )
           self.assertEqual(indices[0] ,				0       )

           # single query with timeout
           _lowercase:			Optional[int]       =     '''foo'''
           _lowercase:			Union[str, Any]       =     {'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 0}]}}
           _lowercase ,				_lowercase:			List[Any]       =     index.search(A_ ,				request_timeout=30       )
           self.assertEqual(scores[0] ,				1       )
           self.assertEqual(indices[0] ,				0       )

           # batched queries
           _lowercase:			Union[str, Any]       =     ['''foo''', '''bar''', '''foobar''']
           _lowercase:			str       =     {'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 1}]}}
           _lowercase ,				_lowercase:			Optional[int]       =     index.search_batch(A_       )
           _lowercase:			Any       =     [scores[0] for scores in total_scores]
           _lowercase:			List[str]       =     [indices[0] for indices in total_indices]
           self.assertGreater(np.min(A_       ) ,				0       )
           self.assertListEqual([1, 1, 1] ,				A_       )

           # batched queries with timeout
           _lowercase:			List[str]       =     ['''foo''', '''bar''', '''foobar''']
           _lowercase:			Dict       =     {'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 1}]}}
           _lowercase ,				_lowercase:			Optional[int]       =     index.search_batch(A_ ,				request_timeout=30       )
           _lowercase:			Optional[Any]       =     [scores[0] for scores in total_scores]
           _lowercase:			Dict       =     [indices[0] for indices in total_indices]
           self.assertGreater(np.min(A_       ) ,				0       )
           self.assertListEqual([1, 1, 1] ,				A_       )
