



"""simple docstring"""




import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
					import torch

if is_vision_available():
					from PIL import Image

					from transformers import ChineseCLIPImageProcessor






class 					__magic_name__							( unittest.TestCase       ):






				'''simple docstring'''





				def __init__( self	,				_a	,				_a=7	,				_a=3	,				_a=18	,				_a=30	,				_a=400	,				_a=True	,				_a=None	,				_a=True	,				_a=None	,				_a=True	,				_a=[0.48_145_466, 0.4_578_275, 0.40_821_073]	,				_a=[0.26_862_954, 0.26_130_258, 0.27_577_711]	,				_a=True	,				):





							"""simple docstring"""
							lowerCamelCase       =      size if size is not None else {'''height''': 224, '''width''': 224}
							lowerCamelCase       =      crop_size if crop_size is not None else {'''height''': 18, '''width''': 18}
							lowerCamelCase       =      parent
							lowerCamelCase       =      batch_size
							lowerCamelCase       =      num_channels
							lowerCamelCase       =      image_size
							lowerCamelCase       =      min_resolution
							lowerCamelCase       =      max_resolution
							lowerCamelCase       =      do_resize
							lowerCamelCase       =      size
							lowerCamelCase       =      do_center_crop
							lowerCamelCase       =      crop_size
							lowerCamelCase       =      do_normalize
							lowerCamelCase       =      image_mean
							lowerCamelCase       =      image_std
							lowerCamelCase       =      do_convert_rgb





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							return {
							    "do_resize": self.do_resize,
							    "size": self.size,
							    "do_center_crop": self.do_center_crop,
							    "crop_size": self.crop_size,
							    "do_normalize": self.do_normalize,
							    "image_mean": self.image_mean,
							    "image_std": self.image_std,
							    "do_convert_rgb": self.do_convert_rgb,
							}





				def      _lowerCAmelCase       ( self	,				_a=False	,				_a=False	,				_a=False				):





							"""simple docstring"""
							assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

							if equal_resolution:
										lowerCamelCase       =      []
										for i in range(self.batch_size				):
													image_inputs.append(
													    np.random.randint(
													        255	,				size=(self.num_channels, self.max_resolution, self.max_resolution)	,				dtype=np.uinta				)				)
							else:
										lowerCamelCase       =      []
										for i in range(self.batch_size				):
													lowerCamelCase       =      np.random.choice(np.arange(self.min_resolution	,				self.max_resolution				)	,				2				)
													image_inputs.append(np.random.randint(255	,				size=(self.num_channels, width, height)	,				dtype=np.uinta				)				)

							if not numpify and not torchify:
										# PIL expects the channel dimension as last dimension
										lowerCamelCase       =      [Image.fromarray(np.moveaxis(UpperCamelCase__	,				0	,				-1				)				) for x in image_inputs]

							if torchify:
										lowerCamelCase       =      [torch.from_numpy(UpperCamelCase__				) for x in image_inputs]

							return image_inputs






@require_torch
@require_vision
class 					__magic_name__							( SCREAMING_SNAKE_CASE_		,      unittest.TestCase       ):






				'''simple docstring'''
				__UpperCamelCase    						=		ChineseCLIPImageProcessor if is_vision_available() else None





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							lowerCamelCase       =      ChineseCLIPImageProcessingTester(self	,				do_center_crop=UpperCamelCase__				)





				@property
				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							return self.image_processor_tester.prepare_image_processor_dict()





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							lowerCamelCase       =      self.image_processing_class(**self.image_processor_dict				)
							self.assertTrue(hasattr(UpperCamelCase__	,				"""do_resize"""				)				)
							self.assertTrue(hasattr(UpperCamelCase__	,				"""size"""				)				)
							self.assertTrue(hasattr(UpperCamelCase__	,				"""do_center_crop"""				)				)
							self.assertTrue(hasattr(UpperCamelCase__	,				"""center_crop"""				)				)
							self.assertTrue(hasattr(UpperCamelCase__	,				"""do_normalize"""				)				)
							self.assertTrue(hasattr(UpperCamelCase__	,				"""image_mean"""				)				)
							self.assertTrue(hasattr(UpperCamelCase__	,				"""image_std"""				)				)
							self.assertTrue(hasattr(UpperCamelCase__	,				"""do_convert_rgb"""				)				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							lowerCamelCase       =      self.image_processing_class.from_dict(self.image_processor_dict				)
							self.assertEqual(image_processor.size	,				{"""height""": 224, """width""": 224}				)
							self.assertEqual(image_processor.crop_size	,				{"""height""": 18, """width""": 18}				)

							lowerCamelCase       =      self.image_processing_class.from_dict(self.image_processor_dict	,				size=42	,				crop_size=84				)
							self.assertEqual(image_processor.size	,				{"""shortest_edge""": 42}				)
							self.assertEqual(image_processor.crop_size	,				{"""height""": 84, """width""": 84}				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							pass





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							# Initialize image_processing
							lowerCamelCase       =      self.image_processing_class(**self.image_processor_dict				)
							# create random PIL images
							lowerCamelCase       =      self.image_processor_tester.prepare_inputs(equal_resolution=UpperCamelCase__				)
							for image in image_inputs:
										self.assertIsInstance(UpperCamelCase__	,				Image.Image				)

							# Test not batched input
							lowerCamelCase       =      image_processing(image_inputs[0]	,				return_tensors="""pt"""				).pixel_values
							self.assertEqual(
							    encoded_images.shape	,				(
							        1,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size["""height"""],
							        self.image_processor_tester.crop_size["""width"""],
							    )	,				)

							# Test batched
							lowerCamelCase       =      image_processing(UpperCamelCase__	,				return_tensors="""pt"""				).pixel_values
							self.assertEqual(
							    encoded_images.shape	,				(
							        self.image_processor_tester.batch_size,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size["""height"""],
							        self.image_processor_tester.crop_size["""width"""],
							    )	,				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							# Initialize image_processing
							lowerCamelCase       =      self.image_processing_class(**self.image_processor_dict				)
							# create random numpy tensors
							lowerCamelCase       =      self.image_processor_tester.prepare_inputs(equal_resolution=UpperCamelCase__	,				numpify=UpperCamelCase__				)
							for image in image_inputs:
										self.assertIsInstance(UpperCamelCase__	,				np.ndarray				)

							# Test not batched input
							lowerCamelCase       =      image_processing(image_inputs[0]	,				return_tensors="""pt"""				).pixel_values
							self.assertEqual(
							    encoded_images.shape	,				(
							        1,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size["""height"""],
							        self.image_processor_tester.crop_size["""width"""],
							    )	,				)

							# Test batched
							lowerCamelCase       =      image_processing(UpperCamelCase__	,				return_tensors="""pt"""				).pixel_values
							self.assertEqual(
							    encoded_images.shape	,				(
							        self.image_processor_tester.batch_size,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size["""height"""],
							        self.image_processor_tester.crop_size["""width"""],
							    )	,				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							# Initialize image_processing
							lowerCamelCase       =      self.image_processing_class(**self.image_processor_dict				)
							# create random PyTorch tensors
							lowerCamelCase       =      self.image_processor_tester.prepare_inputs(equal_resolution=UpperCamelCase__	,				torchify=UpperCamelCase__				)
							for image in image_inputs:
										self.assertIsInstance(UpperCamelCase__	,				torch.Tensor				)

							# Test not batched input
							lowerCamelCase       =      image_processing(image_inputs[0]	,				return_tensors="""pt"""				).pixel_values
							self.assertEqual(
							    encoded_images.shape	,				(
							        1,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size["""height"""],
							        self.image_processor_tester.crop_size["""width"""],
							    )	,				)

							# Test batched
							lowerCamelCase       =      image_processing(UpperCamelCase__	,				return_tensors="""pt"""				).pixel_values
							self.assertEqual(
							    encoded_images.shape	,				(
							        self.image_processor_tester.batch_size,
							        self.image_processor_tester.num_channels,
							        self.image_processor_tester.crop_size["""height"""],
							        self.image_processor_tester.crop_size["""width"""],
							    )	,				)






@require_torch
@require_vision
class 					__magic_name__							( SCREAMING_SNAKE_CASE_		,      unittest.TestCase       ):






				'''simple docstring'''
				__UpperCamelCase    						=		ChineseCLIPImageProcessor if is_vision_available() else None





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							lowerCamelCase       =      ChineseCLIPImageProcessingTester(self	,				num_channels=4	,				do_center_crop=UpperCamelCase__				)
							lowerCamelCase       =      3





				@property
				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							return self.image_processor_tester.prepare_image_processor_dict()





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							lowerCamelCase       =      self.image_processing_class(**self.image_processor_dict				)
							self.assertTrue(hasattr(UpperCamelCase__	,				"""do_resize"""				)				)
							self.assertTrue(hasattr(UpperCamelCase__	,				"""size"""				)				)
							self.assertTrue(hasattr(UpperCamelCase__	,				"""do_center_crop"""				)				)
							self.assertTrue(hasattr(UpperCamelCase__	,				"""center_crop"""				)				)
							self.assertTrue(hasattr(UpperCamelCase__	,				"""do_normalize"""				)				)
							self.assertTrue(hasattr(UpperCamelCase__	,				"""image_mean"""				)				)
							self.assertTrue(hasattr(UpperCamelCase__	,				"""image_std"""				)				)
							self.assertTrue(hasattr(UpperCamelCase__	,				"""do_convert_rgb"""				)				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							pass





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							# Initialize image_processing
							lowerCamelCase       =      self.image_processing_class(**self.image_processor_dict				)
							# create random PIL images
							lowerCamelCase       =      self.image_processor_tester.prepare_inputs(equal_resolution=UpperCamelCase__				)
							for image in image_inputs:
										self.assertIsInstance(UpperCamelCase__	,				Image.Image				)

							# Test not batched input
							lowerCamelCase       =      image_processing(image_inputs[0]	,				return_tensors="""pt"""				).pixel_values
							self.assertEqual(
							    encoded_images.shape	,				(
							        1,
							        self.expected_encoded_image_num_channels,
							        self.image_processor_tester.crop_size["""height"""],
							        self.image_processor_tester.crop_size["""width"""],
							    )	,				)

							# Test batched
							lowerCamelCase       =      image_processing(UpperCamelCase__	,				return_tensors="""pt"""				).pixel_values
							self.assertEqual(
							    encoded_images.shape	,				(
							        self.image_processor_tester.batch_size,
							        self.expected_encoded_image_num_channels,
							        self.image_processor_tester.crop_size["""height"""],
							        self.image_processor_tester.crop_size["""width"""],
							    )	,				)









"""simple docstring"""




import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


lowerCAmelCase							:    List[Any]  							=							pytest.mark.integration






@require_faiss
class 					__magic_name__							( UpperCAmelCase__       ):






				'''simple docstring'''





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							lowerCamelCase       =      Dataset.from_dict({"""filename""": ["""my_name-train""" + """_""" + str(_a				) for x in np.arange(30				).tolist()]}				)
							return dset





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							import faiss

							lowerCamelCase       =      self._create_dummy_dataset()
							lowerCamelCase       =      dset.map(
							    lambda _a	,				_a				: {"vecs": i * np.ones(5	,				dtype=np.floataa				)}	,				with_indices=_a	,				keep_in_memory=_a				)
							lowerCamelCase       =      dset.add_faiss_index("""vecs"""	,				batch_size=100	,				metric_type=faiss.METRIC_INNER_PRODUCT				)
							lowerCamelCase   ,     lowerCamelCase       =      dset.get_nearest_examples("""vecs"""	,				np.ones(5	,				dtype=np.floataa				)				)
							self.assertEqual(examples["""filename"""][0]	,				"""my_name-train_29"""				)
							dset.drop_index("""vecs"""				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							import faiss

							lowerCamelCase       =      self._create_dummy_dataset()
							dset.add_faiss_index_from_external_arrays(
							    external_arrays=np.ones((30, 5)				) * np.arange(30				).reshape(-1	,				1				)	,				index_name="""vecs"""	,				batch_size=100	,				metric_type=faiss.METRIC_INNER_PRODUCT	,				)
							lowerCamelCase   ,     lowerCamelCase       =      dset.get_nearest_examples("""vecs"""	,				np.ones(5	,				dtype=np.floataa				)				)
							self.assertEqual(examples["""filename"""][0]	,				"""my_name-train_29"""				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							import faiss

							lowerCamelCase       =      self._create_dummy_dataset()
							dset.add_faiss_index_from_external_arrays(
							    external_arrays=np.ones((30, 5)				) * np.arange(30				).reshape(-1	,				1				)	,				index_name="""vecs"""	,				metric_type=faiss.METRIC_INNER_PRODUCT	,				)

							# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
							# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
							# see https://bugs.python.org/issue14243 and
							# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
							with tempfile.NamedTemporaryFile(delete=_a				) as tmp_file:
										dset.save_faiss_index("""vecs"""	,				tmp_file.name				)
										dset.load_faiss_index("""vecs2"""	,				tmp_file.name				)
							os.unlink(tmp_file.name				)

							lowerCamelCase   ,     lowerCamelCase       =      dset.get_nearest_examples("""vecs2"""	,				np.ones(5	,				dtype=np.floataa				)				)
							self.assertEqual(examples["""filename"""][0]	,				"""my_name-train_29"""				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							lowerCamelCase       =      self._create_dummy_dataset()
							dset.add_faiss_index_from_external_arrays(
							    external_arrays=np.ones((30, 5)				) * np.arange(30				).reshape(-1	,				1				)	,				index_name="""vecs"""				)
							dset.drop_index("""vecs"""				)
							self.assertRaises(_a	,				partial(dset.get_nearest_examples	,				"""vecs2"""	,				np.ones(5	,				dtype=np.floataa				)				)				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							from elasticsearch import Elasticsearch

							lowerCamelCase       =      self._create_dummy_dataset()
							with patch("""elasticsearch.Elasticsearch.search"""				) as mocked_search, patch(
							    """elasticsearch.client.IndicesClient.create"""				) as mocked_index_create, patch("""elasticsearch.helpers.streaming_bulk"""				) as mocked_bulk:
										lowerCamelCase       =      {"""acknowledged""": True}
										mocked_bulk.return_value([(True, None)] * 30				)
										lowerCamelCase       =      {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 29}]}}
										lowerCamelCase       =      Elasticsearch()

										dset.add_elasticsearch_index("""filename"""	,				es_client=_a				)
										lowerCamelCase   ,     lowerCamelCase       =      dset.get_nearest_examples("""filename"""	,				"""my_name-train_29"""				)
										self.assertEqual(examples["""filename"""][0]	,				"""my_name-train_29"""				)






@require_faiss
class 					__magic_name__							( UpperCAmelCase__       ):






				'''simple docstring'''





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							import faiss

							lowerCamelCase       =      FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT				)

							# add vectors
							index.add_vectors(np.eye(5	,				dtype=np.floataa				)				)
							self.assertIsNotNone(index.faiss_index				)
							self.assertEqual(index.faiss_index.ntotal	,				5				)
							index.add_vectors(np.zeros((5, 5)	,				dtype=np.floataa				)				)
							self.assertEqual(index.faiss_index.ntotal	,				10				)

							# single query
							lowerCamelCase       =      np.zeros(5	,				dtype=np.floataa				)
							lowerCamelCase       =      1
							lowerCamelCase   ,     lowerCamelCase       =      index.search(_a				)
							self.assertRaises(_a	,				index.search	,				query.reshape(-1	,				1				)				)
							self.assertGreater(scores[0]	,				0				)
							self.assertEqual(indices[0]	,				1				)

							# batched queries
							lowerCamelCase       =      np.eye(5	,				dtype=np.floataa				)[::-1]
							lowerCamelCase   ,     lowerCamelCase       =      index.search_batch(_a				)
							self.assertRaises(_a	,				index.search_batch	,				queries[0]				)
							lowerCamelCase       =      [scores[0] for scores in total_scores]
							lowerCamelCase       =      [indices[0] for indices in total_indices]
							self.assertGreater(np.min(_a				)	,				0				)
							self.assertListEqual([4, 3, 2, 1, 0]	,				_a				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							import faiss

							lowerCamelCase       =      FaissIndex(string_factory="""Flat"""				)
							index.add_vectors(np.eye(5	,				dtype=np.floataa				)				)
							self.assertIsInstance(index.faiss_index	,				faiss.IndexFlat				)
							lowerCamelCase       =      FaissIndex(string_factory="""LSH"""				)
							index.add_vectors(np.eye(5	,				dtype=np.floataa				)				)
							self.assertIsInstance(index.faiss_index	,				faiss.IndexLSH				)
							with self.assertRaises(_a				):
										lowerCamelCase       =      FaissIndex(string_factory="""Flat"""	,				custom_index=faiss.IndexFlat(5				)				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							import faiss

							lowerCamelCase       =      faiss.IndexFlat(5				)
							lowerCamelCase       =      FaissIndex(custom_index=_a				)
							index.add_vectors(np.eye(5	,				dtype=np.floataa				)				)
							self.assertIsInstance(index.faiss_index	,				faiss.IndexFlat				)





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							import faiss

							lowerCamelCase       =      FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT				)
							index.add_vectors(np.eye(5	,				dtype=np.floataa				)				)

							# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
							# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
							# see https://bugs.python.org/issue14243 and
							# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
							with tempfile.NamedTemporaryFile(delete=_a				) as tmp_file:
										index.save(tmp_file.name				)
										lowerCamelCase       =      FaissIndex.load(tmp_file.name				)
							os.unlink(tmp_file.name				)

							lowerCamelCase       =      np.zeros(5	,				dtype=np.floataa				)
							lowerCamelCase       =      1
							lowerCamelCase   ,     lowerCamelCase       =      index.search(_a				)
							self.assertGreater(scores[0]	,				0				)
							self.assertEqual(indices[0]	,				1				)




@require_faiss
def 					a__				(       snake_case__       )		->      Tuple:
			import faiss

			lowerCamelCase       =      FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT       )
			index.add_vectors(np.eye(5 ,				dtype=np.floataa       )       )

			lowerCamelCase       =      """index.faiss"""
			lowerCamelCase       =      F'mock://{index_name}'
			index.save(snake_case__ ,				storage_options=mockfs.storage_options       )
			lowerCamelCase       =      FaissIndex.load(snake_case__ ,				storage_options=mockfs.storage_options       )

			lowerCamelCase       =      np.zeros(5 ,				dtype=np.floataa       )
			lowerCamelCase       =      1
			lowerCamelCase   ,     lowerCamelCase       =      index.search(snake_case__       )
			assert scores[0] > 0
			assert indices[0] == 1






@require_elasticsearch
class 					__magic_name__							( UpperCAmelCase__       ):






				'''simple docstring'''





				def      _lowerCAmelCase       ( self				):





							"""simple docstring"""
							from elasticsearch import Elasticsearch

							with patch("""elasticsearch.Elasticsearch.search"""				) as mocked_search, patch(
							    """elasticsearch.client.IndicesClient.create"""				) as mocked_index_create, patch("""elasticsearch.helpers.streaming_bulk"""				) as mocked_bulk:
										lowerCamelCase       =      Elasticsearch()
										lowerCamelCase       =      {"""acknowledged""": True}
										lowerCamelCase       =      ElasticSearchIndex(es_client=_a				)
										mocked_bulk.return_value([(True, None)] * 3				)
										index.add_documents(["""foo""", """bar""", """foobar"""]				)

										# single query
										lowerCamelCase       =      """foo"""
										lowerCamelCase       =      {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 0}]}}
										lowerCamelCase   ,     lowerCamelCase       =      index.search(_a				)
										self.assertEqual(scores[0]	,				1				)
										self.assertEqual(indices[0]	,				0				)

										# single query with timeout
										lowerCamelCase       =      """foo"""
										lowerCamelCase       =      {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 0}]}}
										lowerCamelCase   ,     lowerCamelCase       =      index.search(_a	,				request_timeout=30				)
										self.assertEqual(scores[0]	,				1				)
										self.assertEqual(indices[0]	,				0				)

										# batched queries
										lowerCamelCase       =      ["""foo""", """bar""", """foobar"""]
										lowerCamelCase       =      {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 1}]}}
										lowerCamelCase   ,     lowerCamelCase       =      index.search_batch(_a				)
										lowerCamelCase       =      [scores[0] for scores in total_scores]
										lowerCamelCase       =      [indices[0] for indices in total_indices]
										self.assertGreater(np.min(_a				)	,				0				)
										self.assertListEqual([1, 1, 1]	,				_a				)

										# batched queries with timeout
										lowerCamelCase       =      ["""foo""", """bar""", """foobar"""]
										lowerCamelCase       =      {"""hits""": {"""hits""": [{"""_score""": 1, """_id""": 1}]}}
										lowerCamelCase   ,     lowerCamelCase       =      index.search_batch(_a	,				request_timeout=30				)
										lowerCamelCase       =      [scores[0] for scores in total_scores]
										lowerCamelCase       =      [indices[0] for indices in total_indices]
										self.assertGreater(np.min(_a				)	,				0				)
										self.assertListEqual([1, 1, 1]	,				_a				)



