





"""simple docstring"""
import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


__UpperCamelCase     :						Union[str, Any]						       =  datasets.utils.logging.get_logger(__name__)







@dataclass
class       a							(   datasets.BuilderConfig  ):
 snake_case__             =       1_0_0_0_0
 snake_case__             =       None
 snake_case__             =       None







class       a							(   datasets.ArrowBasedBuilder  ):
 snake_case__             =       ParquetConfig



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  return datasets.DatasetInfo(features=self.config.features     )



 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  if not self.config.data_files:
   raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}'     )
  lowerCAmelCase    				= dl_manager.download_and_extract(self.config.data_files     )
  if isinstance(_snake_case				,       (str, list, tuple)     ):
   lowerCAmelCase    				= data_files
   if isinstance(_snake_case				,       _snake_case     ):
    lowerCAmelCase    				= [files]
   # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
   lowerCAmelCase    				= [dl_manager.iter_files(_snake_case     ) for file in files]
   return [datasets.SplitGenerator(name=datasets.Split.TRAIN				,       gen_kwargs={'files': files}     )]
  lowerCAmelCase    				= []
  for split_name, files in data_files.items():
   if isinstance(_snake_case				,       _snake_case     ):
    lowerCAmelCase    				= [files]
   # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
   lowerCAmelCase    				= [dl_manager.iter_files(_snake_case     ) for file in files]
   # Infer features is they are stoed in the arrow schema
   if self.info.features is None:
    for file in itertools.chain.from_iterable(_snake_case     ):
     with open(_snake_case				,       'rb'     ) as f:
      lowerCAmelCase    				= datasets.Features.from_arrow_schema(pq.read_schema(_snake_case     )     )
     break
   splits.append(datasets.SplitGenerator(name=_snake_case				,       gen_kwargs={'files': files}     )     )
  return splits



 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  if self.info.features is not None:
   # more expensive cast to support nested features with keys in a different order
   # allows str <-> int/float or str to Audio for example
   lowerCAmelCase    				= table_cast(_snake_case				,       self.info.features.arrow_schema     )
  return pa_table



 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  lowerCAmelCase    				= self.info.features.arrow_schema if self.info.features is not None else None
  if self.info.features is not None and self.config.columns is not None:
   if sorted(field.name for field in schema     ) != sorted(self.config.columns     ):
    raise ValueError(
        F'Tried to load parquet data with columns \'{self.config.columns}\' with mismatching features \'{self.info.features}\''     )
  for file_idx, file in enumerate(itertools.chain.from_iterable(_snake_case     )     ):
   with open(_snake_case				,       'rb'     ) as f:
    lowerCAmelCase    				= pq.ParquetFile(_snake_case     )
    try:
     for batch_idx, record_batch in enumerate(
         parquet_file.iter_batches(batch_size=self.config.batch_size				,       columns=self.config.columns     )     ):
      lowerCAmelCase    				= pa.Table.from_batches([record_batch]     )
      # Uncomment for debugging (will print the Arrow table size and elements)
      # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
      # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
      yield F'{file_idx}_{batch_idx}', self._cast_table(_snake_case     )
    except ValueError as e:
     logger.error(F'Failed to read file \'{file}\' with error {type(_snake_case     )}: {e}'     )
     raise












"""simple docstring"""
import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


__UpperCamelCase     :						Optional[int]						       =  pytest.mark.integration







@require_faiss
class       a							(   a__  ):



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  lowerCAmelCase    				= Dataset.from_dict({'filename': ['my_name-train' + '_' + str(_snake_case     ) for x in np.arange(30     ).tolist()]}     )
  return dset



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  import faiss

  lowerCAmelCase    				= self._create_dummy_dataset()
  lowerCAmelCase    				= dset.map(
      lambda _snake_case				,       _snake_case     : {"vecs": i * np.ones(5				,       dtype=np.floataa     )}				,       with_indices=_snake_case				,       keep_in_memory=_snake_case     )
  lowerCAmelCase    				= dset.add_faiss_index('vecs'				,       batch_size=1_00				,       metric_type=faiss.METRIC_INNER_PRODUCT     )
  lowerCAmelCase				,lowerCAmelCase    				= dset.get_nearest_examples('vecs'				,       np.ones(5				,       dtype=np.floataa     )     )
  self.assertEqual(examples['filename'][0]				,       'my_name-train_29'     )
  dset.drop_index('vecs'     )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  import faiss

  lowerCAmelCase    				= self._create_dummy_dataset()
  dset.add_faiss_index_from_external_arrays(
      external_arrays=np.ones((30, 5)     ) * np.arange(30     ).reshape(-1				,       1     )				,       index_name='vecs'				,       batch_size=1_00				,       metric_type=faiss.METRIC_INNER_PRODUCT				,       )
  lowerCAmelCase				,lowerCAmelCase    				= dset.get_nearest_examples('vecs'				,       np.ones(5				,       dtype=np.floataa     )     )
  self.assertEqual(examples['filename'][0]				,       'my_name-train_29'     )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  import faiss

  lowerCAmelCase    				= self._create_dummy_dataset()
  dset.add_faiss_index_from_external_arrays(
      external_arrays=np.ones((30, 5)     ) * np.arange(30     ).reshape(-1				,       1     )				,       index_name='vecs'				,       metric_type=faiss.METRIC_INNER_PRODUCT				,       )

  # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
  # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
  # see https://bugs.python.org/issue14243 and
  # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
  with tempfile.NamedTemporaryFile(delete=_snake_case     ) as tmp_file:
   dset.save_faiss_index('vecs'				,       tmp_file.name     )
   dset.load_faiss_index('vecs2'				,       tmp_file.name     )
  os.unlink(tmp_file.name     )

  lowerCAmelCase				,lowerCAmelCase    				= dset.get_nearest_examples('vecs2'				,       np.ones(5				,       dtype=np.floataa     )     )
  self.assertEqual(examples['filename'][0]				,       'my_name-train_29'     )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  lowerCAmelCase    				= self._create_dummy_dataset()
  dset.add_faiss_index_from_external_arrays(
      external_arrays=np.ones((30, 5)     ) * np.arange(30     ).reshape(-1				,       1     )				,       index_name='vecs'     )
  dset.drop_index('vecs'     )
  self.assertRaises(_snake_case				,       partial(dset.get_nearest_examples				,       'vecs2'				,       np.ones(5				,       dtype=np.floataa     )     )     )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  from elasticsearch import Elasticsearch

  lowerCAmelCase    				= self._create_dummy_dataset()
  with patch('elasticsearch.Elasticsearch.search'     ) as mocked_search, patch(
      'elasticsearch.client.IndicesClient.create'     ) as mocked_index_create, patch('elasticsearch.helpers.streaming_bulk'     ) as mocked_bulk:
   lowerCAmelCase    				= {'acknowledged': True}
   mocked_bulk.return_value([(True, None)] * 30     )
   lowerCAmelCase    				= {'hits': {'hits': [{'_score': 1, '_id': 29}]}}
   lowerCAmelCase    				= Elasticsearch()

   dset.add_elasticsearch_index('filename'				,       es_client=_snake_case     )
   lowerCAmelCase				,lowerCAmelCase    				= dset.get_nearest_examples('filename'				,       'my_name-train_29'     )
   self.assertEqual(examples['filename'][0]				,       'my_name-train_29'     )







@require_faiss
class       a							(   a__  ):



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  import faiss

  lowerCAmelCase    				= FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT     )

  # add vectors
  index.add_vectors(np.eye(5				,       dtype=np.floataa     )     )
  self.assertIsNotNone(index.faiss_index     )
  self.assertEqual(index.faiss_index.ntotal				,       5     )
  index.add_vectors(np.zeros((5, 5)				,       dtype=np.floataa     )     )
  self.assertEqual(index.faiss_index.ntotal				,       10     )

  # single query
  lowerCAmelCase    				= np.zeros(5				,       dtype=np.floataa     )
  lowerCAmelCase    				= 1
  lowerCAmelCase				,lowerCAmelCase    				= index.search(_snake_case     )
  self.assertRaises(_snake_case				,       index.search				,       query.reshape(-1				,       1     )     )
  self.assertGreater(scores[0]				,       0     )
  self.assertEqual(indices[0]				,       1     )

  # batched queries
  lowerCAmelCase    				= np.eye(5				,       dtype=np.floataa     )[::-1]
  lowerCAmelCase				,lowerCAmelCase    				= index.search_batch(_snake_case     )
  self.assertRaises(_snake_case				,       index.search_batch				,       queries[0]     )
  lowerCAmelCase    				= [scores[0] for scores in total_scores]
  lowerCAmelCase    				= [indices[0] for indices in total_indices]
  self.assertGreater(np.min(_snake_case     )				,       0     )
  self.assertListEqual([4, 3, 2, 1, 0]				,       _snake_case     )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  import faiss

  lowerCAmelCase    				= FaissIndex(string_factory='Flat'     )
  index.add_vectors(np.eye(5				,       dtype=np.floataa     )     )
  self.assertIsInstance(index.faiss_index				,       faiss.IndexFlat     )
  lowerCAmelCase    				= FaissIndex(string_factory='LSH'     )
  index.add_vectors(np.eye(5				,       dtype=np.floataa     )     )
  self.assertIsInstance(index.faiss_index				,       faiss.IndexLSH     )
  with self.assertRaises(_snake_case     ):
   lowerCAmelCase    				= FaissIndex(string_factory='Flat'				,       custom_index=faiss.IndexFlat(5     )     )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  import faiss

  lowerCAmelCase    				= faiss.IndexFlat(5     )
  lowerCAmelCase    				= FaissIndex(custom_index=_snake_case     )
  index.add_vectors(np.eye(5				,       dtype=np.floataa     )     )
  self.assertIsInstance(index.faiss_index				,       faiss.IndexFlat     )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  import faiss

  lowerCAmelCase    				= FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT     )
  index.add_vectors(np.eye(5				,       dtype=np.floataa     )     )

  # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
  # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
  # see https://bugs.python.org/issue14243 and
  # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
  with tempfile.NamedTemporaryFile(delete=_snake_case     ) as tmp_file:
   index.save(tmp_file.name     )
   lowerCAmelCase    				= FaissIndex.load(tmp_file.name     )
  os.unlink(tmp_file.name     )

  lowerCAmelCase    				= np.zeros(5				,       dtype=np.floataa     )
  lowerCAmelCase    				= 1
  lowerCAmelCase				,lowerCAmelCase    				= index.search(_snake_case     )
  self.assertGreater(scores[0]				,       0     )
  self.assertEqual(indices[0]				,       1     )


@require_faiss
def 		_SCREAMING_SNAKE_CASE			(_UpperCAmelCase					:    Dict			):
 import faiss

 lowerCAmelCase    				= FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT			)
 index.add_vectors(np.eye(5  ,     dtype=np.floataa			)			)

 lowerCAmelCase    				= 'index.faiss'
 lowerCAmelCase    				= F'mock://{index_name}'
 index.save(_UpperCAmelCase  ,     storage_options=mockfs.storage_options			)
 lowerCAmelCase    				= FaissIndex.load(_UpperCAmelCase  ,     storage_options=mockfs.storage_options			)

 lowerCAmelCase    				= np.zeros(5  ,     dtype=np.floataa			)
 lowerCAmelCase    				= 1
 lowerCAmelCase				,lowerCAmelCase    				= index.search(_UpperCAmelCase			)
 assert scores[0] > 0
 assert indices[0] == 1







@require_elasticsearch
class       a							(   a__  ):



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  from elasticsearch import Elasticsearch

  with patch('elasticsearch.Elasticsearch.search'     ) as mocked_search, patch(
      'elasticsearch.client.IndicesClient.create'     ) as mocked_index_create, patch('elasticsearch.helpers.streaming_bulk'     ) as mocked_bulk:
   lowerCAmelCase    				= Elasticsearch()
   lowerCAmelCase    				= {'acknowledged': True}
   lowerCAmelCase    				= ElasticSearchIndex(es_client=_snake_case     )
   mocked_bulk.return_value([(True, None)] * 3     )
   index.add_documents(['foo', 'bar', 'foobar']     )

   # single query
   lowerCAmelCase    				= 'foo'
   lowerCAmelCase    				= {'hits': {'hits': [{'_score': 1, '_id': 0}]}}
   lowerCAmelCase				,lowerCAmelCase    				= index.search(_snake_case     )
   self.assertEqual(scores[0]				,       1     )
   self.assertEqual(indices[0]				,       0     )

   # single query with timeout
   lowerCAmelCase    				= 'foo'
   lowerCAmelCase    				= {'hits': {'hits': [{'_score': 1, '_id': 0}]}}
   lowerCAmelCase				,lowerCAmelCase    				= index.search(_snake_case				,       request_timeout=30     )
   self.assertEqual(scores[0]				,       1     )
   self.assertEqual(indices[0]				,       0     )

   # batched queries
   lowerCAmelCase    				= ['foo', 'bar', 'foobar']
   lowerCAmelCase    				= {'hits': {'hits': [{'_score': 1, '_id': 1}]}}
   lowerCAmelCase				,lowerCAmelCase    				= index.search_batch(_snake_case     )
   lowerCAmelCase    				= [scores[0] for scores in total_scores]
   lowerCAmelCase    				= [indices[0] for indices in total_indices]
   self.assertGreater(np.min(_snake_case     )				,       0     )
   self.assertListEqual([1, 1, 1]				,       _snake_case     )

   # batched queries with timeout
   lowerCAmelCase    				= ['foo', 'bar', 'foobar']
   lowerCAmelCase    				= {'hits': {'hits': [{'_score': 1, '_id': 1}]}}
   lowerCAmelCase				,lowerCAmelCase    				= index.search_batch(_snake_case				,       request_timeout=30     )
   lowerCAmelCase    				= [scores[0] for scores in total_scores]
   lowerCAmelCase    				= [indices[0] for indices in total_indices]
   self.assertGreater(np.min(_snake_case     )				,       0     )
   self.assertListEqual([1, 1, 1]				,       _snake_case     )





