
"""simple docstring"""



from typing import Dict, List, Optional, Union

import numpy as np

from transformers.utils import is_vision_available
from transformers.utils.generic import TensorType

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_valid_image,
    to_numpy_array,
    valid_images,
)
from ...utils import logging


if is_vision_available():
 import PIL

__lowercase						:     Tuple							      =		logging.get_logger(__name__)


def lowerCamelCase_				(     _lowerCamelCase			:	List[Any]				):
       if isinstance(__SCREAMING_SNAKE_CASE					,				(list, tuple)				) and isinstance(videos[0]					,				(list, tuple)				) and is_valid_image(videos[0][0]				):
              return videos

       elif isinstance(__SCREAMING_SNAKE_CASE					,				(list, tuple)				) and is_valid_image(videos[0]				):
              return [videos]

       elif is_valid_image(__SCREAMING_SNAKE_CASE				):
              return [[videos]]

       raise ValueError(F"""Could not make batched video from {videos}"""				)





class 					lowerCAmelCase			(				lowerCAmelCase_					):
     """simple docstring"""







     __lowercase						:Union[str, Any]          =    ["pixel_values"]




     def __init__(      self ,		UpperCamelCase__ = True ,		UpperCamelCase__ = None ,		UpperCamelCase__ = PILImageResampling.BILINEAR ,		UpperCamelCase__ = True ,		UpperCamelCase__ = None ,		UpperCamelCase__ = True ,		UpperCamelCase__ = 1 / 255 ,		UpperCamelCase__ = True ,		UpperCamelCase__ = True ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		**UpperCamelCase__ ,		)					->  List[str]:







            '''simple docstring'''





            super().__init__(**__lowerCamelCase							)
            lowerCamelCase_       							=		size if size is not None else {'shortest_edge': 256}
            lowerCamelCase_       							=		get_size_dict(__lowerCamelCase ,		default_to_square=__lowerCamelCase							)
            lowerCamelCase_       							=		crop_size if crop_size is not None else {'height': 224, 'width': 224}
            lowerCamelCase_       							=		get_size_dict(__lowerCamelCase ,		param_name='''crop_size'''							)

            lowerCamelCase_       							=		do_resize
            lowerCamelCase_       							=		size
            lowerCamelCase_       							=		do_center_crop
            lowerCamelCase_       							=		crop_size
            lowerCamelCase_       							=		resample
            lowerCamelCase_       							=		do_rescale
            lowerCamelCase_       							=		rescale_factor
            lowerCamelCase_       							=		offset
            lowerCamelCase_       							=		do_normalize
            lowerCamelCase_       							=		image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
            lowerCamelCase_       							=		image_std if image_std is not None else IMAGENET_STANDARD_STD




     def    _lowerCAmelCase		(      self ,		UpperCamelCase__ ,		UpperCamelCase__ ,		UpperCamelCase__ = PILImageResampling.BILINEAR ,		UpperCamelCase__ = None ,		**UpperCamelCase__ ,		)					->  Tuple:







            '''simple docstring'''





            lowerCamelCase_       							=		get_size_dict(__lowerCamelCase ,		default_to_square=__lowerCamelCase							)
            if "shortest_edge" in size:
                   lowerCamelCase_       							=		get_resize_output_image_size(__lowerCamelCase ,		size['''shortest_edge'''] ,		default_to_square=__lowerCamelCase							)
            elif "height" in size and "width" in size:
                   lowerCamelCase_       							=		(size['height'], size['width'])
            else:
                   raise ValueError(F"""Size must have 'height' and 'width' or 'shortest_edge' as keys. Got {size.keys()}"""							)
            return resize(__lowerCamelCase ,		size=__lowerCamelCase ,		resample=__lowerCamelCase ,		data_format=__lowerCamelCase ,		**__lowerCamelCase							)




     def    _lowerCAmelCase		(      self ,		UpperCamelCase__ ,		UpperCamelCase__ ,		UpperCamelCase__ = None ,		**UpperCamelCase__ ,		)					->  Any:







            '''simple docstring'''





            lowerCamelCase_       							=		get_size_dict(__lowerCamelCase							)
            if "height" not in size or "width" not in size:
                   raise ValueError(F"""Size must have 'height' and 'width' as keys. Got {size.keys()}"""							)
            return center_crop(__lowerCamelCase ,		size=(size['''height'''], size['''width''']) ,		data_format=__lowerCamelCase ,		**__lowerCamelCase							)




     def    _lowerCAmelCase		(      self ,		UpperCamelCase__ ,		UpperCamelCase__ ,		UpperCamelCase__ = True ,		UpperCamelCase__ = None ,		**UpperCamelCase__ ,		)					->  Union[str, Any]:







            '''simple docstring'''





            lowerCamelCase_       							=		image.astype(np.floataa							)
            if offset:
                   lowerCamelCase_       							=		image - (scale / 2)
            return rescale(__lowerCamelCase ,		scale=__lowerCamelCase ,		data_format=__lowerCamelCase ,		**__lowerCamelCase							)




     def    _lowerCAmelCase		(      self ,		UpperCamelCase__ ,		UpperCamelCase__ ,		UpperCamelCase__ ,		UpperCamelCase__ = None ,		**UpperCamelCase__ ,		)					->  Tuple:







            '''simple docstring'''





            return normalize(__lowerCamelCase ,		mean=__lowerCamelCase ,		std=__lowerCamelCase ,		data_format=__lowerCamelCase ,		**__lowerCamelCase							)




     def    _lowerCAmelCase		(      self ,		UpperCamelCase__ ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = ChannelDimension.FIRST ,		)					->  Union[str, Any]:







            '''simple docstring'''





            if do_resize and size is None or resample is None:
                   raise ValueError('''Size and resample must be specified if do_resize is True.'''							)

            if do_center_crop and crop_size is None:
                   raise ValueError('''Crop size must be specified if do_center_crop is True.'''							)

            if do_rescale and rescale_factor is None:
                   raise ValueError('''Rescale factor must be specified if do_rescale is True.'''							)

            if do_normalize and (image_mean is None or image_std is None):
                   raise ValueError('''Image mean and std must be specified if do_normalize is True.'''							)

            if offset and not do_rescale:
                   raise ValueError('''For offset, do_rescale must also be set to True.'''							)

            # All transformations expect numpy arrays.
            lowerCamelCase_       							=		to_numpy_array(__lowerCamelCase							)

            if do_resize:
                   lowerCamelCase_       							=		self.resize(image=__lowerCamelCase ,		size=__lowerCamelCase ,		resample=__lowerCamelCase							)

            if do_center_crop:
                   lowerCamelCase_       							=		self.center_crop(__lowerCamelCase ,		size=__lowerCamelCase							)

            if do_rescale:
                   lowerCamelCase_       							=		self.rescale(image=__lowerCamelCase ,		scale=__lowerCamelCase ,		offset=__lowerCamelCase							)

            if do_normalize:
                   lowerCamelCase_       							=		self.normalize(image=__lowerCamelCase ,		mean=__lowerCamelCase ,		std=__lowerCamelCase							)

            lowerCamelCase_       							=		to_channel_dimension_format(__lowerCamelCase ,		__lowerCamelCase							)
            return image




     def    _lowerCAmelCase		(      self ,		UpperCamelCase__ ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = None ,		UpperCamelCase__ = ChannelDimension.FIRST ,		**UpperCamelCase__ ,		)					->  List[str]:







            '''simple docstring'''





            lowerCamelCase_       							=		do_resize if do_resize is not None else self.do_resize
            lowerCamelCase_       							=		resample if resample is not None else self.resample
            lowerCamelCase_       							=		do_center_crop if do_center_crop is not None else self.do_center_crop
            lowerCamelCase_       							=		do_rescale if do_rescale is not None else self.do_rescale
            lowerCamelCase_       							=		rescale_factor if rescale_factor is not None else self.rescale_factor
            lowerCamelCase_       							=		offset if offset is not None else self.offset
            lowerCamelCase_       							=		do_normalize if do_normalize is not None else self.do_normalize
            lowerCamelCase_       							=		image_mean if image_mean is not None else self.image_mean
            lowerCamelCase_       							=		image_std if image_std is not None else self.image_std

            lowerCamelCase_       							=		size if size is not None else self.size
            lowerCamelCase_       							=		get_size_dict(__lowerCamelCase ,		default_to_square=__lowerCamelCase							)
            lowerCamelCase_       							=		crop_size if crop_size is not None else self.crop_size
            lowerCamelCase_       							=		get_size_dict(__lowerCamelCase ,		param_name='''crop_size'''							)

            if not valid_images(__lowerCamelCase							):
                   raise ValueError(
                       '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
                       '''torch.Tensor, tf.Tensor or jax.ndarray.'''							)

            lowerCamelCase_       							=		make_batched(__lowerCamelCase							)

            lowerCamelCase_       							=		[
                [
                    self._preprocess_image(
                        image=__lowerCamelCase ,		do_resize=__lowerCamelCase ,		size=__lowerCamelCase ,		resample=__lowerCamelCase ,		do_center_crop=__lowerCamelCase ,		crop_size=__lowerCamelCase ,		do_rescale=__lowerCamelCase ,		rescale_factor=__lowerCamelCase ,		offset=__lowerCamelCase ,		do_normalize=__lowerCamelCase ,		image_mean=__lowerCamelCase ,		image_std=__lowerCamelCase ,		data_format=__lowerCamelCase ,		)
                    for img in video
                ]
                for video in videos
            ]

            lowerCamelCase_       							=		{'pixel_values': videos}
            return BatchFeature(data=__lowerCamelCase ,		tensor_type=__lowerCamelCase							)





'''simple docstring'''






import json
import os
import pickle
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers import is_faiss_available
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bart.tokenization_bart import BartTokenizer
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.dpr.tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer
from transformers.models.rag.configuration_rag import RagConfig
from transformers.models.rag.retrieval_rag import CustomHFIndex, RagRetriever
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_sentencepiece, require_tokenizers, require_torch


if is_faiss_available():
      import faiss






@require_faiss
class        lowercase_      (						lowerCAmelCase_							):





     def 			_lowerCAmelCase      (     self     :					Optional[int]			):
           snake_case__       :   Tuple	      =    tempfile.mkdtemp()
           snake_case__       :   Dict	      =    8

           # DPR tok
           snake_case__       :   Any	      =    [
               '[UNK]',
               '[CLS]',
               '[SEP]',
               '[PAD]',
               '[MASK]',
               'want',
               '##want',
               '##ed',
               'wa',
               'un',
               'runn',
               '##ing',
               ',',
               'low',
               'lowest',
           ]
           snake_case__       :   Optional[int]	      =    os.path.join(self.tmpdirname     ,							'dpr_tokenizer'			)
           os.makedirs(__lowerCamelCase     ,							exist_ok=__lowerCamelCase			)
           snake_case__       :   Tuple	      =    os.path.join(__lowerCamelCase     ,							DPR_VOCAB_FILES_NAMES['vocab_file']			)
           with open(self.vocab_file     ,							'w'     ,							encoding='utf-8'			) as vocab_writer:
                 vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]			)			)

           # BART tok
           snake_case__       :   Dict	      =    [
               'l',
               'o',
               'w',
               'e',
               'r',
               's',
               't',
               'i',
               'd',
               'n',
               '\u0120',
               '\u0120l',
               '\u0120n',
               '\u0120lo',
               '\u0120low',
               'er',
               '\u0120lowest',
               '\u0120newer',
               '\u0120wider',
               '<unk>',
           ]
           snake_case__       :   str	      =    dict(zip(__lowerCamelCase     ,							range(len(__lowerCamelCase			)			)			)			)
           snake_case__       :   Union[str, Any]	      =    ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
           snake_case__       :   Any	      =    {'unk_token': '<unk>'}

           snake_case__       :   List[str]	      =    os.path.join(self.tmpdirname     ,							'bart_tokenizer'			)
           os.makedirs(__lowerCamelCase     ,							exist_ok=__lowerCamelCase			)
           snake_case__       :   Dict	      =    os.path.join(__lowerCamelCase     ,							BART_VOCAB_FILES_NAMES['vocab_file']			)
           snake_case__       :   int	      =    os.path.join(__lowerCamelCase     ,							BART_VOCAB_FILES_NAMES['merges_file']			)
           with open(self.vocab_file     ,							'w'     ,							encoding='utf-8'			) as fp:
                 fp.write(json.dumps(__lowerCamelCase			) + '\n'			)
           with open(self.merges_file     ,							'w'     ,							encoding='utf-8'			) as fp:
                 fp.write('\n'.join(__lowerCamelCase			)			)





     def 			_lowerCAmelCase      (     self     :					Optional[int]			):
           return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname     ,							'dpr_tokenizer'			)			)





     def 			_lowerCAmelCase      (     self     :					Union[str, Any]			):
           return DPRContextEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname     ,							'dpr_tokenizer'			)			)





     def 			_lowerCAmelCase      (     self     :					Any			):
           return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname     ,							'bart_tokenizer'			)			)





     def 			_lowerCAmelCase      (     self     :					int			):
           shutil.rmtree(self.tmpdirname			)





     def 			_lowerCAmelCase      (     self     :					str			):
           snake_case__       :   int	      =    Dataset.from_dict(
               {
                   'id': ['0', '1'],
                   'text': ['foo', 'bar'],
                   'title': ['Foo', 'Bar'],
                   'embeddings': [np.ones(self.retrieval_vector_size			), 2 * np.ones(self.retrieval_vector_size			)],
               }			)
           dataset.add_faiss_index('embeddings'     ,							string_factory='Flat'     ,							metric_type=faiss.METRIC_INNER_PRODUCT			)
           return dataset





     def 			_lowerCAmelCase      (     self     :					Any			):
           snake_case__       :   List[str]	      =    self.get_dummy_dataset()
           snake_case__       :   Dict	      =    RagConfig(
               retrieval_vector_size=self.retrieval_vector_size     ,							question_encoder=DPRConfig().to_dict()     ,							generator=BartConfig().to_dict()     ,							)
           with patch('transformers.models.rag.retrieval_rag.load_dataset'			) as mock_load_dataset:
                 snake_case__       :   Optional[Any]	      =    dataset
                 snake_case__       :   List[Any]	      =    RagRetriever(
                     __lowerCamelCase     ,							question_encoder_tokenizer=self.get_dpr_tokenizer()     ,							generator_tokenizer=self.get_bart_tokenizer()     ,							)
           return retriever





     def 			_lowerCAmelCase      (     self     :					List[Any]     ,							__lowerCamelCase     :					bool			):
           snake_case__       :   Union[str, Any]	      =    self.get_dummy_dataset()
           snake_case__       :   Dict	      =    RagConfig(
               retrieval_vector_size=self.retrieval_vector_size     ,							question_encoder=DPRConfig().to_dict()     ,							generator=BartConfig().to_dict()     ,							index_name='custom'     ,							)
           if from_disk:
                 snake_case__       :   List[Any]	      =    os.path.join(self.tmpdirname     ,							'dataset'			)
                 snake_case__       :   List[str]	      =    os.path.join(self.tmpdirname     ,							'index.faiss'			)
                 dataset.get_index('embeddings'			).save(os.path.join(self.tmpdirname     ,							'index.faiss'			)			)
                 dataset.drop_index('embeddings'			)
                 dataset.save_to_disk(os.path.join(self.tmpdirname     ,							'dataset'			)			)
                 del dataset
                 snake_case__       :   Optional[int]	      =    RagRetriever(
                     __lowerCamelCase     ,							question_encoder_tokenizer=self.get_dpr_tokenizer()     ,							generator_tokenizer=self.get_bart_tokenizer()     ,							)
           else:
                 snake_case__       :   Union[str, Any]	      =    RagRetriever(
                     __lowerCamelCase     ,							question_encoder_tokenizer=self.get_dpr_tokenizer()     ,							generator_tokenizer=self.get_bart_tokenizer()     ,							index=CustomHFIndex(config.retrieval_vector_size     ,							__lowerCamelCase			)     ,							)
           return retriever





     def 			_lowerCAmelCase      (     self     :					Tuple			):
           snake_case__       :   Dict	      =    Dataset.from_dict(
               {
                   'id': ['0', '1'],
                   'text': ['foo', 'bar'],
                   'title': ['Foo', 'Bar'],
                   'embeddings': [np.ones(self.retrieval_vector_size + 1			), 2 * np.ones(self.retrieval_vector_size + 1			)],
               }			)
           dataset.add_faiss_index('embeddings'     ,							string_factory='Flat'     ,							metric_type=faiss.METRIC_INNER_PRODUCT			)

           snake_case__       :   List[str]	      =    os.path.join(self.tmpdirname     ,							'hf_bert_base.hnswSQ8_correct_phi_128.c_index'			)
           dataset.save_faiss_index('embeddings'     ,							index_file_name + '.index.dpr'			)
           pickle.dump(dataset['id']     ,							open(index_file_name + '.index_meta.dpr'     ,							'wb'			)			)

           snake_case__       :   Optional[int]	      =    os.path.join(self.tmpdirname     ,							'psgs_w100.tsv.pkl'			)
           snake_case__       :   Optional[Any]	      =    {sample['id']: [sample['text'], sample['title']] for sample in dataset}
           pickle.dump(__lowerCamelCase     ,							open(__lowerCamelCase     ,							'wb'			)			)

           snake_case__       :   Union[str, Any]	      =    RagConfig(
               retrieval_vector_size=self.retrieval_vector_size     ,							question_encoder=DPRConfig().to_dict()     ,							generator=BartConfig().to_dict()     ,							index_name='legacy'     ,							index_path=self.tmpdirname     ,							)
           snake_case__       :   List[str]	      =    RagRetriever(
               __lowerCamelCase     ,							question_encoder_tokenizer=self.get_dpr_tokenizer()     ,							generator_tokenizer=self.get_bart_tokenizer()			)
           return retriever





     def 			_lowerCAmelCase      (     self     :					List[Any]			):
           snake_case__       :   List[str]	      =    1
           snake_case__       :   int	      =    self.get_dummy_canonical_hf_index_retriever()
           snake_case__       :   Tuple	      =    np.array(
               [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)]     ,							dtype=np.floataa			)
           snake_case__      ,							snake_case__      ,							snake_case__       :   int	      =    retriever.retrieve(__lowerCamelCase     ,							n_docs=__lowerCamelCase			)
           self.assertEqual(retrieved_doc_embeds.shape     ,							(2, n_docs, self.retrieval_vector_size)			)
           self.assertEqual(len(__lowerCamelCase			)     ,							2			)
           self.assertEqual(sorted(doc_dicts[0]			)     ,							['embeddings', 'id', 'text', 'title']			)
           self.assertEqual(len(doc_dicts[0]['id']			)     ,							__lowerCamelCase			)
           self.assertEqual(doc_dicts[0]['id'][0]     ,							'1'			)  # max inner product is reached with second doc
           self.assertEqual(doc_dicts[1]['id'][0]     ,							'0'			)  # max inner product is reached with first doc
           self.assertListEqual(doc_ids.tolist()     ,							[[1], [0]]			)





     def 			_lowerCAmelCase      (     self     :					List[str]			):
           snake_case__       :   str	      =    self.get_dummy_canonical_hf_index_retriever()
           with tempfile.TemporaryDirectory() as tmp_dirname:
                 with patch('transformers.models.rag.retrieval_rag.load_dataset'			) as mock_load_dataset:
                       snake_case__       :   Union[str, Any]	      =    self.get_dummy_dataset()
                       retriever.save_pretrained(__lowerCamelCase			)
                       snake_case__       :   List[Any]	      =    RagRetriever.from_pretrained(__lowerCamelCase			)
                       self.assertIsInstance(__lowerCamelCase     ,							__lowerCamelCase			)
                       snake_case__       :   str	      =    np.array(
                           [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)]     ,							dtype=np.floataa			)
                       snake_case__       :   Tuple	      =    retriever.retrieve(__lowerCamelCase     ,							n_docs=1			)
                       self.assertTrue(out is not None			)





     def 			_lowerCAmelCase      (     self     :					int			):
           snake_case__       :   Any	      =    1
           snake_case__       :   Tuple	      =    self.get_dummy_custom_hf_index_retriever(from_disk=__lowerCamelCase			)
           snake_case__       :   Dict	      =    np.array(
               [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)]     ,							dtype=np.floataa			)
           snake_case__      ,							snake_case__      ,							snake_case__       :   List[str]	      =    retriever.retrieve(__lowerCamelCase     ,							n_docs=__lowerCamelCase			)
           self.assertEqual(retrieved_doc_embeds.shape     ,							(2, n_docs, self.retrieval_vector_size)			)
           self.assertEqual(len(__lowerCamelCase			)     ,							2			)
           self.assertEqual(sorted(doc_dicts[0]			)     ,							['embeddings', 'id', 'text', 'title']			)
           self.assertEqual(len(doc_dicts[0]['id']			)     ,							__lowerCamelCase			)
           self.assertEqual(doc_dicts[0]['id'][0]     ,							'1'			)  # max inner product is reached with second doc
           self.assertEqual(doc_dicts[1]['id'][0]     ,							'0'			)  # max inner product is reached with first doc
           self.assertListEqual(doc_ids.tolist()     ,							[[1], [0]]			)





     def 			_lowerCAmelCase      (     self     :					int			):
           snake_case__       :   str	      =    self.get_dummy_custom_hf_index_retriever(from_disk=__lowerCamelCase			)
           with tempfile.TemporaryDirectory() as tmp_dirname:
                 retriever.save_pretrained(__lowerCamelCase			)
                 snake_case__       :   int	      =    RagRetriever.from_pretrained(__lowerCamelCase			)
                 self.assertIsInstance(__lowerCamelCase     ,							__lowerCamelCase			)
                 snake_case__       :   Tuple	      =    np.array(
                     [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)]     ,							dtype=np.floataa			)
                 snake_case__       :   List[str]	      =    retriever.retrieve(__lowerCamelCase     ,							n_docs=1			)
                 self.assertTrue(out is not None			)





     def 			_lowerCAmelCase      (     self     :					List[str]			):
           snake_case__       :   Any	      =    1
           snake_case__       :   Optional[int]	      =    self.get_dummy_custom_hf_index_retriever(from_disk=__lowerCamelCase			)
           snake_case__       :   str	      =    np.array(
               [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)]     ,							dtype=np.floataa			)
           snake_case__      ,							snake_case__      ,							snake_case__       :   List[Any]	      =    retriever.retrieve(__lowerCamelCase     ,							n_docs=__lowerCamelCase			)
           self.assertEqual(retrieved_doc_embeds.shape     ,							(2, n_docs, self.retrieval_vector_size)			)
           self.assertEqual(len(__lowerCamelCase			)     ,							2			)
           self.assertEqual(sorted(doc_dicts[0]			)     ,							['embeddings', 'id', 'text', 'title']			)
           self.assertEqual(len(doc_dicts[0]['id']			)     ,							__lowerCamelCase			)
           self.assertEqual(doc_dicts[0]['id'][0]     ,							'1'			)  # max inner product is reached with second doc
           self.assertEqual(doc_dicts[1]['id'][0]     ,							'0'			)  # max inner product is reached with first doc
           self.assertListEqual(doc_ids.tolist()     ,							[[1], [0]]			)





     def 			_lowerCAmelCase      (     self     :					Optional[int]			):
           snake_case__       :   Optional[int]	      =    self.get_dummy_custom_hf_index_retriever(from_disk=__lowerCamelCase			)
           with tempfile.TemporaryDirectory() as tmp_dirname:
                 retriever.save_pretrained(__lowerCamelCase			)
                 snake_case__       :   int	      =    RagRetriever.from_pretrained(__lowerCamelCase			)
                 self.assertIsInstance(__lowerCamelCase     ,							__lowerCamelCase			)
                 snake_case__       :   Optional[int]	      =    np.array(
                     [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)]     ,							dtype=np.floataa			)
                 snake_case__       :   List[Any]	      =    retriever.retrieve(__lowerCamelCase     ,							n_docs=1			)
                 self.assertTrue(out is not None			)





     def 			_lowerCAmelCase      (     self     :					int			):
           snake_case__       :   Tuple	      =    1
           snake_case__       :   Tuple	      =    self.get_dummy_legacy_index_retriever()
           snake_case__       :   str	      =    np.array(
               [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)]     ,							dtype=np.floataa			)
           snake_case__      ,							snake_case__      ,							snake_case__       :   List[str]	      =    retriever.retrieve(__lowerCamelCase     ,							n_docs=__lowerCamelCase			)
           self.assertEqual(retrieved_doc_embeds.shape     ,							(2, n_docs, self.retrieval_vector_size)			)
           self.assertEqual(len(__lowerCamelCase			)     ,							2			)
           self.assertEqual(sorted(doc_dicts[0]			)     ,							['text', 'title']			)
           self.assertEqual(len(doc_dicts[0]['text']			)     ,							__lowerCamelCase			)
           self.assertEqual(doc_dicts[0]['text'][0]     ,							'bar'			)  # max inner product is reached with second doc
           self.assertEqual(doc_dicts[1]['text'][0]     ,							'foo'			)  # max inner product is reached with first doc
           self.assertListEqual(doc_ids.tolist()     ,							[[1], [0]]			)





     def 			_lowerCAmelCase      (     self     :					Union[str, Any]			):
           snake_case__       :   Any	      =    self.get_dummy_legacy_index_retriever()
           with tempfile.TemporaryDirectory() as tmp_dirname:
                 retriever.save_pretrained(__lowerCamelCase			)
                 snake_case__       :   Dict	      =    RagRetriever.from_pretrained(__lowerCamelCase			)
                 self.assertIsInstance(__lowerCamelCase     ,							__lowerCamelCase			)
                 snake_case__       :   Any	      =    np.array(
                     [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)]     ,							dtype=np.floataa			)
                 snake_case__       :   List[Any]	      =    retriever.retrieve(__lowerCamelCase     ,							n_docs=1			)
                 self.assertTrue(out is not None			)





     @require_torch
     @require_tokenizers
     @require_sentencepiece
     def 			_lowerCAmelCase      (     self     :					Any			):
           import torch

           snake_case__       :   Optional[Any]	      =    1
           snake_case__       :   int	      =    self.get_dummy_canonical_hf_index_retriever()
           snake_case__       :   List[Any]	      =    [[5, 7], [10, 11]]
           snake_case__       :   List[Any]	      =    np.array(
               [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)]     ,							dtype=np.floataa			)
           snake_case__       :   Any	      =    retriever(__lowerCamelCase     ,							__lowerCamelCase     ,							prefix=retriever.config.generator.prefix     ,							n_docs=__lowerCamelCase			)
           snake_case__      ,							snake_case__      ,							snake_case__       :   Any	      =    (
               out['context_input_ids'],
               out['context_attention_mask'],
               out['retrieved_doc_embeds'],
           )
           self.assertEqual(retrieved_doc_embeds.shape     ,							(2, n_docs, self.retrieval_vector_size)			)
           self.assertIsInstance(__lowerCamelCase     ,							__lowerCamelCase			)
           self.assertIsInstance(__lowerCamelCase     ,							__lowerCamelCase			)
           self.assertIsInstance(__lowerCamelCase     ,							np.ndarray			)

           snake_case__       :   Dict	      =    retriever(
               __lowerCamelCase     ,							__lowerCamelCase     ,							prefix=retriever.config.generator.prefix     ,							n_docs=__lowerCamelCase     ,							return_tensors='pt'     ,							)
           snake_case__      ,							snake_case__      ,							snake_case__      ,							snake_case__       :   Optional[Any]	      =    (  # noqa: F841
               out['context_input_ids'],
               out['context_attention_mask'],
               out['retrieved_doc_embeds'],
               out['doc_ids'],
           )
           self.assertEqual(retrieved_doc_embeds.shape     ,							(2, n_docs, self.retrieval_vector_size)			)
           self.assertIsInstance(__lowerCamelCase     ,							torch.Tensor			)
           self.assertIsInstance(__lowerCamelCase     ,							torch.Tensor			)
           self.assertIsInstance(__lowerCamelCase     ,							torch.Tensor			)





     @require_torch
     @require_tokenizers
     @require_sentencepiece
     def 			_lowerCAmelCase      (     self     :					Optional[Any]			):
           snake_case__       :   List[Any]	      =    self.get_dpr_ctx_encoder_tokenizer()
           snake_case__       :   List[Any]	      =    1
           snake_case__       :   Any	      =    self.get_dummy_custom_hf_index_retriever(from_disk=__lowerCamelCase			)
           retriever.set_ctx_encoder_tokenizer(__lowerCamelCase			)

           snake_case__       :   Optional[int]	      =    [[5, 7], [10, 11]]
           snake_case__       :   Tuple	      =    np.array(
               [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)]     ,							dtype=np.floataa			)
           snake_case__       :   Tuple	      =    retriever(__lowerCamelCase     ,							__lowerCamelCase     ,							prefix=retriever.config.generator.prefix     ,							n_docs=__lowerCamelCase			)

           self.assertEqual(
               len(__lowerCamelCase			)     ,							6			)  # check whether the retriever output consist of 6 attributes including tokenized docs
           self.assertEqual(
               all(k in out for k in ('tokenized_doc_ids', 'tokenized_doc_attention_mask')			)     ,							__lowerCamelCase			)  # check for doc token related keys in dictionary.
