






'''simple docstring'''



import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


SCREAMING_SNAKE_CASE							=						pytest.mark.integration
@require_faiss
class UpperCAmelCase_       (     __A					):




      """simple docstring"""




      def 					A__   (			self      :		Tuple		)	->	Tuple:


       '''simple docstring'''

       lowercase       : Any						=Dataset.from_dict({'''filename''': ['''my_name-train''' + '''_''' + str(UpperCAmelCase		) for x in np.arange(30		).tolist()]}		)
       return dset






      def 					A__   (			self      :		str		)	->	str:


       '''simple docstring'''

       import faiss

       lowercase       : Dataset						=self._create_dummy_dataset()
       lowercase       : Any						=dset.map(
           lambda UpperCAmelCase  ,   UpperCAmelCase		: {"vecs": i * np.ones(5  ,   dtype=np.floataa		)}  ,   with_indices=UpperCAmelCase  ,   keep_in_memory=UpperCAmelCase		)
       lowercase       : List[str]						=dset.add_faiss_index('''vecs'''  ,   batch_size=100  ,   metric_type=faiss.METRIC_INNER_PRODUCT		)
       lowercase	,		lowercase       : List[Any]						=dset.get_nearest_examples('''vecs'''  ,   np.ones(5  ,   dtype=np.floataa		)		)
       self.assertEqual(examples['''filename'''][0]  ,   '''my_name-train_29'''		)
       dset.drop_index('''vecs'''		)






      def 					A__   (			self      :		Union[str, Any]		)	->	List[Any]:


       '''simple docstring'''

       import faiss

       lowercase       : Dataset						=self._create_dummy_dataset()
       dset.add_faiss_index_from_external_arrays(
           external_arrays=np.ones((30, 5)		) * np.arange(30		).reshape(-1  ,   1		)  ,   index_name='''vecs'''  ,   batch_size=100  ,   metric_type=faiss.METRIC_INNER_PRODUCT  ,   )
       lowercase	,		lowercase       : List[str]						=dset.get_nearest_examples('''vecs'''  ,   np.ones(5  ,   dtype=np.floataa		)		)
       self.assertEqual(examples['''filename'''][0]  ,   '''my_name-train_29'''		)






      def 					A__   (			self      :		Union[str, Any]		)	->	Tuple:


       '''simple docstring'''

       import faiss

       lowercase       : Dataset						=self._create_dummy_dataset()
       dset.add_faiss_index_from_external_arrays(
           external_arrays=np.ones((30, 5)		) * np.arange(30		).reshape(-1  ,   1		)  ,   index_name='''vecs'''  ,   metric_type=faiss.METRIC_INNER_PRODUCT  ,   )

       # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
       # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
       # see https://bugs.python.org/issue14243 and
       # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
       with tempfile.NamedTemporaryFile(delete=UpperCAmelCase		) as tmp_file:
        dset.save_faiss_index('''vecs'''  ,   tmp_file.name		)
        dset.load_faiss_index('''vecs2'''  ,   tmp_file.name		)
       os.unlink(tmp_file.name		)

       lowercase	,		lowercase       : Optional[Any]						=dset.get_nearest_examples('''vecs2'''  ,   np.ones(5  ,   dtype=np.floataa		)		)
       self.assertEqual(examples['''filename'''][0]  ,   '''my_name-train_29'''		)






      def 					A__   (			self      :		int		)	->	Tuple:


       '''simple docstring'''

       lowercase       : Dataset						=self._create_dummy_dataset()
       dset.add_faiss_index_from_external_arrays(
           external_arrays=np.ones((30, 5)		) * np.arange(30		).reshape(-1  ,   1		)  ,   index_name='''vecs'''		)
       dset.drop_index('''vecs'''		)
       self.assertRaises(UpperCAmelCase  ,   partial(dset.get_nearest_examples  ,   '''vecs2'''  ,   np.ones(5  ,   dtype=np.floataa		)		)		)






      def 					A__   (			self      :		Any		)	->	Any:


       '''simple docstring'''

       from elasticsearch import Elasticsearch

       lowercase       : Dataset						=self._create_dummy_dataset()
       with patch('''elasticsearch.Elasticsearch.search'''		) as mocked_search, patch(
           '''elasticsearch.client.IndicesClient.create'''		) as mocked_index_create, patch('''elasticsearch.helpers.streaming_bulk'''		) as mocked_bulk:
        lowercase       : Optional[int]						={'''acknowledged''': True}
        mocked_bulk.return_value([(True, None)] * 30		)
        lowercase       : Optional[Any]						={'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 29}]}}
        lowercase       : Dict						=Elasticsearch()

        dset.add_elasticsearch_index('''filename'''  ,   es_client=UpperCAmelCase		)
        lowercase	,		lowercase       : Dict						=dset.get_nearest_examples('''filename'''  ,   '''my_name-train_29'''		)
        self.assertEqual(examples['''filename'''][0]  ,   '''my_name-train_29'''		)





@require_faiss
class UpperCAmelCase_       (     __A					):




      """simple docstring"""




      def 					A__   (			self      :		Any		)	->	Tuple:


       '''simple docstring'''

       import faiss

       lowercase       : Any						=FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT		)

       # add vectors
       index.add_vectors(np.eye(5  ,   dtype=np.floataa		)		)
       self.assertIsNotNone(index.faiss_index		)
       self.assertEqual(index.faiss_index.ntotal  ,   5		)
       index.add_vectors(np.zeros((5, 5)  ,   dtype=np.floataa		)		)
       self.assertEqual(index.faiss_index.ntotal  ,   10		)

       # single query
       lowercase       : Optional[int]						=np.zeros(5  ,   dtype=np.floataa		)
       lowercase       : str						=1
       lowercase	,		lowercase       : Tuple						=index.search(UpperCAmelCase		)
       self.assertRaises(UpperCAmelCase  ,   index.search  ,   query.reshape(-1  ,   1		)		)
       self.assertGreater(scores[0]  ,   0		)
       self.assertEqual(indices[0]  ,   1		)

       # batched queries
       lowercase       : List[str]						=np.eye(5  ,   dtype=np.floataa		)[::-1]
       lowercase	,		lowercase       : int						=index.search_batch(UpperCAmelCase		)
       self.assertRaises(UpperCAmelCase  ,   index.search_batch  ,   queries[0]		)
       lowercase       : Optional[int]						=[scores[0] for scores in total_scores]
       lowercase       : Optional[int]						=[indices[0] for indices in total_indices]
       self.assertGreater(np.min(UpperCAmelCase		)  ,   0		)
       self.assertListEqual([4, 3, 2, 1, 0]  ,   UpperCAmelCase		)






      def 					A__   (			self      :		Dict		)	->	Tuple:


       '''simple docstring'''

       import faiss

       lowercase       : Dict						=FaissIndex(string_factory='''Flat'''		)
       index.add_vectors(np.eye(5  ,   dtype=np.floataa		)		)
       self.assertIsInstance(index.faiss_index  ,   faiss.IndexFlat		)
       lowercase       : List[str]						=FaissIndex(string_factory='''LSH'''		)
       index.add_vectors(np.eye(5  ,   dtype=np.floataa		)		)
       self.assertIsInstance(index.faiss_index  ,   faiss.IndexLSH		)
       with self.assertRaises(UpperCAmelCase		):
        lowercase       : Any						=FaissIndex(string_factory='''Flat'''  ,   custom_index=faiss.IndexFlat(5		)		)






      def 					A__   (			self      :		str		)	->	str:


       '''simple docstring'''

       import faiss

       lowercase       : Optional[Any]						=faiss.IndexFlat(5		)
       lowercase       : Tuple						=FaissIndex(custom_index=UpperCAmelCase		)
       index.add_vectors(np.eye(5  ,   dtype=np.floataa		)		)
       self.assertIsInstance(index.faiss_index  ,   faiss.IndexFlat		)






      def 					A__   (			self      :		Optional[int]		)	->	Tuple:


       '''simple docstring'''

       import faiss

       lowercase       : List[str]						=FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT		)
       index.add_vectors(np.eye(5  ,   dtype=np.floataa		)		)

       # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
       # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
       # see https://bugs.python.org/issue14243 and
       # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
       with tempfile.NamedTemporaryFile(delete=UpperCAmelCase		) as tmp_file:
        index.save(tmp_file.name		)
        lowercase       : str						=FaissIndex.load(tmp_file.name		)
       os.unlink(tmp_file.name		)

       lowercase       : List[str]						=np.zeros(5  ,   dtype=np.floataa		)
       lowercase       : Optional[Any]						=1
       lowercase	,		lowercase       : Union[str, Any]						=index.search(UpperCAmelCase		)
       self.assertGreater(scores[0]  ,   0		)
       self.assertEqual(indices[0]  ,   1		)





@require_faiss
def 				lowercase_					(     __A			:							Union[str, Any]      )							->   List[Any]:



 """simple docstring"""




 import faiss

 lowercase       : int						=FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT      )
 index.add_vectors(np.eye(5       ,   dtype=np.floataa      )      )

 lowercase       : Dict						='''index.faiss'''
 lowercase       : str						=F'mock://{index_name}'
 index.save(__A       ,   storage_options=mockfs.storage_options      )
 lowercase       : str						=FaissIndex.load(__A       ,   storage_options=mockfs.storage_options      )

 lowercase       : List[str]						=np.zeros(5       ,   dtype=np.floataa      )
 lowercase       : str						=1
 lowercase	,		lowercase       : Union[str, Any]						=index.search(__A      )
 assert scores[0] > 0
 assert indices[0] == 1





@require_elasticsearch
class UpperCAmelCase_       (     __A					):




      """simple docstring"""




      def 					A__   (			self      :		Union[str, Any]		)	->	int:


       '''simple docstring'''

       from elasticsearch import Elasticsearch

       with patch('''elasticsearch.Elasticsearch.search'''		) as mocked_search, patch(
           '''elasticsearch.client.IndicesClient.create'''		) as mocked_index_create, patch('''elasticsearch.helpers.streaming_bulk'''		) as mocked_bulk:
        lowercase       : Optional[int]						=Elasticsearch()
        lowercase       : Dict						={'''acknowledged''': True}
        lowercase       : List[str]						=ElasticSearchIndex(es_client=UpperCAmelCase		)
        mocked_bulk.return_value([(True, None)] * 3		)
        index.add_documents(['''foo''', '''bar''', '''foobar''']		)

        # single query
        lowercase       : Tuple						='''foo'''
        lowercase       : Dict						={'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 0}]}}
        lowercase	,		lowercase       : Dict						=index.search(UpperCAmelCase		)
        self.assertEqual(scores[0]  ,   1		)
        self.assertEqual(indices[0]  ,   0		)

        # single query with timeout
        lowercase       : List[Any]						='''foo'''
        lowercase       : Dict						={'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 0}]}}
        lowercase	,		lowercase       : Optional[Any]						=index.search(UpperCAmelCase  ,   request_timeout=30		)
        self.assertEqual(scores[0]  ,   1		)
        self.assertEqual(indices[0]  ,   0		)

        # batched queries
        lowercase       : List[Any]						=['''foo''', '''bar''', '''foobar''']
        lowercase       : str						={'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 1}]}}
        lowercase	,		lowercase       : Optional[int]						=index.search_batch(UpperCAmelCase		)
        lowercase       : Optional[Any]						=[scores[0] for scores in total_scores]
        lowercase       : Tuple						=[indices[0] for indices in total_indices]
        self.assertGreater(np.min(UpperCAmelCase		)  ,   0		)
        self.assertListEqual([1, 1, 1]  ,   UpperCAmelCase		)

        # batched queries with timeout
        lowercase       : Optional[int]						=['''foo''', '''bar''', '''foobar''']
        lowercase       : str						={'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 1}]}}
        lowercase	,		lowercase       : Union[str, Any]						=index.search_batch(UpperCAmelCase  ,   request_timeout=30		)
        lowercase       : str						=[scores[0] for scores in total_scores]
        lowercase       : int						=[indices[0] for indices in total_indices]
        self.assertGreater(np.min(UpperCAmelCase		)  ,   0		)
        self.assertListEqual([1, 1, 1]  ,   UpperCAmelCase		)











'''simple docstring'''



import importlib.util
import os
import platform
from argparse import ArgumentParser

import huggingface_hub

from .. import __version__ as version
from ..utils import (
    is_accelerate_available,
    is_flax_available,
    is_safetensors_available,
    is_tf_available,
    is_torch_available,
)
from . import BaseTransformersCLICommand


def 				lowercase_					(     __A			:							List[Any]      )							->   Union[str, Any]:



 """simple docstring"""




 return EnvironmentCommand()


def 				lowercase_					(     __A			:							int      )							->   Dict:



 """simple docstring"""




 return EnvironmentCommand(args.accelerate_config_file      )





class UpperCAmelCase_       (     __A					):




      """simple docstring"""




      @staticmethod
      def 					A__   (			UpperCAmelCase      :		ArgumentParser		)	->	List[str]:


       '''simple docstring'''

       lowercase       : Optional[int]						=parser.add_parser('''env'''		)
       download_parser.set_defaults(func=UpperCAmelCase		)
       download_parser.add_argument(
           '''--accelerate-config_file'''  ,   default=UpperCAmelCase  ,   help='''The accelerate config file to use for the default values in the launching script.'''  ,   )
       download_parser.set_defaults(func=UpperCAmelCase		)






      def __init__(			self      :		Dict  ,   UpperCAmelCase      :		Tuple  ,   *UpperCAmelCase      :		Tuple		)	->	None:


       '''simple docstring'''

       lowercase       : List[str]						=accelerate_config_file






      def 					A__   (			self      :		Tuple		)	->	int:


       '''simple docstring'''

       lowercase       : Optional[Any]						='''not installed'''
       if is_safetensors_available():
        import safetensors

        lowercase       : List[Any]						=safetensors.__version__
       elif importlib.util.find_spec('''safetensors'''		) is not None:
        import safetensors

        lowercase       : List[str]						=f'{safetensors.__version__} but is ignored because of PyTorch version too old.'

       lowercase       : List[str]						='''not installed'''
       lowercase       : str						='''not found'''
       if is_accelerate_available():
        import accelerate
        from accelerate.commands.config import default_config_file, load_config_from_file

        lowercase       : List[str]						=accelerate.__version__
        # Get the default from the config file.
        if self._accelerate_config_file is not None or os.path.isfile(UpperCAmelCase		):
         lowercase       : Dict						=load_config_from_file(self._accelerate_config_file		).to_dict()

        lowercase       : Optional[int]						=(
            '''\n'''.join([f'\t- {prop}: {val}' for prop, val in accelerate_config.items()]		)
            if isinstance(UpperCAmelCase  ,   UpperCAmelCase		)
            else f'\t{accelerate_config}'
        )

       lowercase       : Optional[int]						='''not installed'''
       lowercase       : Optional[Any]						='''NA'''
       if is_torch_available():
        import torch

        lowercase       : List[Any]						=torch.__version__
        lowercase       : Dict						=torch.cuda.is_available()

       lowercase       : Optional[Any]						='''not installed'''
       lowercase       : List[str]						='''NA'''
       if is_tf_available():
        import tensorflow as tf

        lowercase       : List[Any]						=tf.__version__
        try:
         # deprecated in v2.1
         lowercase       : Optional[Any]						=tf.test.is_gpu_available()
        except AttributeError:
         # returns list of devices, convert to bool
         lowercase       : Optional[int]						=bool(tf.config.list_physical_devices('''GPU'''		)		)

       lowercase       : Optional[Any]						='''not installed'''
       lowercase       : Optional[Any]						='''not installed'''
       lowercase       : Optional[Any]						='''not installed'''
       lowercase       : List[str]						='''NA'''
       if is_flax_available():
        import flax
        import jax
        import jaxlib

        lowercase       : Dict						=flax.__version__
        lowercase       : List[Any]						=jax.__version__
        lowercase       : Any						=jaxlib.__version__
        lowercase       : Tuple						=jax.lib.xla_bridge.get_backend().platform

       lowercase       : str						={
           '''`transformers` version''': version,
           '''Platform''': platform.platform(),
           '''Python version''': platform.python_version(),
           '''Huggingface_hub version''': huggingface_hub.__version__,
           '''Safetensors version''': f'{safetensors_version}',
           '''Accelerate version''': f'{accelerate_version}',
           '''Accelerate config''': f'{accelerate_config_str}',
           '''PyTorch version (GPU?)''': f'{pt_version} ({pt_cuda_available})',
           '''Tensorflow version (GPU?)''': f'{tf_version} ({tf_cuda_available})',
           '''Flax version (CPU?/GPU?/TPU?)''': f'{flax_version} ({jax_backend})',
           '''Jax version''': f'{jax_version}',
           '''JaxLib version''': f'{jaxlib_version}',
           '''Using GPU in script?''': '''<fill in>''',
           '''Using distributed or parallel set-up in script?''': '''<fill in>''',
       }

       print('''\nCopy-and-paste the text below in your GitHub issue and FILL OUT the two last points.\n'''		)
       print(self.format_dict(UpperCAmelCase		)		)

       return info






      @staticmethod
      def 					A__   (			UpperCAmelCase      :		Any		)	->	Any:


       '''simple docstring'''

       return "\n".join([f'- {prop}: {val}' for prop, val in d.items()]		) + "\n"



