


import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
							from cookiecutter.main import cookiecutter

							a       	=  True
except ImportError:
							a       	=  False

a       	=  logging.get_logger(__name__)  # pylint: disable=invalid-name


def       UpperCAmelCase_					(   UpperCAmelCase__	):
					return AddNewModelCommand(args.testing    ,				args.testing_file    ,				path=args.path	)







class 							UpperCamelCase__ (						lowercase_ ):

					@staticmethod
					def 	UpperCAmelCase__							(  UpperCamelCase__							: str							):






										'''simple docstring'''






										lowercase_					=      parser.add_parser("""add-new-model"""							)
										add_new_model_parser.add_argument("""--testing"""				,       action="""store_true"""				,       help="""If in testing mode."""							)
										add_new_model_parser.add_argument("""--testing_file"""				,       type=UpperCamelCase__				,       help="""Configuration file on which to run."""							)
										add_new_model_parser.add_argument(
										    """--path"""				,       type=UpperCamelCase__				,       help="""Path to cookiecutter. Should only be used for testing purposes."""							)
										add_new_model_parser.set_defaults(func=UpperCamelCase__							)

					def __init__(  self							: Optional[Any]				,       UpperCamelCase__							: Optional[int]				,       UpperCamelCase__							: Dict				,       UpperCamelCase__							: List[str]=None				,       *UpperCamelCase__							: Optional[int]							):






										'''simple docstring'''






										lowercase_					=      testing
										lowercase_					=      testing_file
										lowercase_					=      path







					def 	UpperCAmelCase__							(  self							: str							):






										'''simple docstring'''






										warnings.warn(
										    """The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. """
										    """It is not actively maintained anymore, so might give a result that won\'t pass all tests and quality """
										    """checks, you should use `transformers-cli add-new-model-like` instead."""							)
										if not _has_cookiecutter:
															raise ImportError(
															    """Model creation dependencies are required to use the `add_new_model` command. Install them by running """
															    """the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n"""							)
										# Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
										lowercase_					=      [directory for directory in os.listdir() if """cookiecutter-template-""" == directory[:22]]
										if len(UpperCamelCase__							) > 0:
															raise ValueError(
															    """Several directories starting with `cookiecutter-template-` in current working directory. """
															    """Please clean your directory by removing all folders starting with `cookiecutter-template-` or """
															    """change your working directory."""							)

										lowercase_					=      (
										    Path(UpperCamelCase__							).parent.parent.parent.parent if self._path is None else Path(self._path							).parent.parent
										)
										lowercase_					=      path_to_transformer_root / """templates""" / """adding_a_new_model"""

										# Execute cookiecutter
										if not self._testing:
															cookiecutter(str(UpperCamelCase__							)							)
										else:
															with open(self._testing_file				,       """r"""							) as configuration_file:
																				lowercase_					=      json.load(UpperCamelCase__							)

															cookiecutter(
															    str(path_to_cookiecutter if self._path is None else self._path							)				,       no_input=UpperCamelCase__				,       extra_context=UpperCamelCase__				,       )

										lowercase_					=      [directory for directory in os.listdir() if """cookiecutter-template-""" in directory[:22]][0]

										# Retrieve configuration
										with open(directory + """/configuration.json"""				,       """r"""							) as configuration_file:
															lowercase_					=      json.load(UpperCamelCase__							)

										lowercase_					=      configuration["""lowercase_modelname"""]
										lowercase_					=      configuration["""generate_tensorflow_pytorch_and_flax"""]
										os.remove(F'''{directory}/configuration.json'''							)

										lowercase_					=      """PyTorch""" in generate_tensorflow_pytorch_and_flax
										lowercase_					=      """TensorFlow""" in generate_tensorflow_pytorch_and_flax
										lowercase_					=      """Flax""" in generate_tensorflow_pytorch_and_flax

										lowercase_					=      F'''{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}'''
										os.makedirs(UpperCamelCase__				,       exist_ok=UpperCamelCase__							)
										os.makedirs(F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}'''				,       exist_ok=UpperCamelCase__							)

										# Tests require submodules as they have parent imports
										with open(F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py'''				,       """w"""							):
															pass

										shutil.move(
										    F'''{directory}/__init__.py'''				,       F'''{model_dir}/__init__.py'''				,       )
										shutil.move(
										    F'''{directory}/configuration_{lowercase_model_name}.py'''				,       F'''{model_dir}/configuration_{lowercase_model_name}.py'''				,       )

										def remove_copy_lines(UpperCamelCase__							: Union[str, Any]							):
															with open(UpperCamelCase__				,       """r"""							) as f:
																				lowercase_					=      f.readlines()
															with open(UpperCamelCase__				,       """w"""							) as f:
																				for line in lines:
																									if "# Copied from transformers." not in line:
																														f.write(UpperCamelCase__							)

										if output_pytorch:
															if not self._testing:
																				remove_copy_lines(F'''{directory}/modeling_{lowercase_model_name}.py'''							)

															shutil.move(
															    F'''{directory}/modeling_{lowercase_model_name}.py'''				,       F'''{model_dir}/modeling_{lowercase_model_name}.py'''				,       )

															shutil.move(
															    F'''{directory}/test_modeling_{lowercase_model_name}.py'''				,       F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py'''				,       )
										else:
															os.remove(F'''{directory}/modeling_{lowercase_model_name}.py'''							)
															os.remove(F'''{directory}/test_modeling_{lowercase_model_name}.py'''							)

										if output_tensorflow:
															if not self._testing:
																				remove_copy_lines(F'''{directory}/modeling_tf_{lowercase_model_name}.py'''							)

															shutil.move(
															    F'''{directory}/modeling_tf_{lowercase_model_name}.py'''				,       F'''{model_dir}/modeling_tf_{lowercase_model_name}.py'''				,       )

															shutil.move(
															    F'''{directory}/test_modeling_tf_{lowercase_model_name}.py'''				,       F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py'''				,       )
										else:
															os.remove(F'''{directory}/modeling_tf_{lowercase_model_name}.py'''							)
															os.remove(F'''{directory}/test_modeling_tf_{lowercase_model_name}.py'''							)

										if output_flax:
															if not self._testing:
																				remove_copy_lines(F'''{directory}/modeling_flax_{lowercase_model_name}.py'''							)

															shutil.move(
															    F'''{directory}/modeling_flax_{lowercase_model_name}.py'''				,       F'''{model_dir}/modeling_flax_{lowercase_model_name}.py'''				,       )

															shutil.move(
															    F'''{directory}/test_modeling_flax_{lowercase_model_name}.py'''				,       F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py'''				,       )
										else:
															os.remove(F'''{directory}/modeling_flax_{lowercase_model_name}.py'''							)
															os.remove(F'''{directory}/test_modeling_flax_{lowercase_model_name}.py'''							)

										shutil.move(
										    F'''{directory}/{lowercase_model_name}.md'''				,       F'''{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md'''				,       )

										shutil.move(
										    F'''{directory}/tokenization_{lowercase_model_name}.py'''				,       F'''{model_dir}/tokenization_{lowercase_model_name}.py'''				,       )

										shutil.move(
										    F'''{directory}/tokenization_fast_{lowercase_model_name}.py'''				,       F'''{model_dir}/tokenization_{lowercase_model_name}_fast.py'''				,       )

										from os import fdopen, remove
										from shutil import copymode, move
										from tempfile import mkstemp

										def replace(UpperCamelCase__							: str				,       UpperCamelCase__							: str				,       UpperCamelCase__							: Optional[int]							):
															# Create temp file
															lowercase_				,       lowercase_					=      mkstemp()
															lowercase_					=      False
															with fdopen(UpperCamelCase__				,       """w"""							) as new_file:
																				with open(UpperCamelCase__							) as old_file:
																									for line in old_file:
																														new_file.write(UpperCamelCase__							)
																														if line_to_copy_below in line:
																																			lowercase_					=      True
																																			for line_to_copy in lines_to_copy:
																																								new_file.write(UpperCamelCase__							)

															if not line_found:
																				raise ValueError(F'''Line {line_to_copy_below} was not found in file.'''							)

															# Copy the file permissions from the old file to the new file
															copymode(UpperCamelCase__				,       UpperCamelCase__							)
															# Remove original file
															remove(UpperCamelCase__							)
															# Move new file
															move(UpperCamelCase__				,       UpperCamelCase__							)

										def skip_units(UpperCamelCase__							: int							):
															return (
															    ("generating PyTorch" in line and not output_pytorch)
															    or ("generating TensorFlow" in line and not output_tensorflow)
															    or ("generating Flax" in line and not output_flax)
															)

										def replace_in_files(UpperCamelCase__							: List[Any]							):
															with open(UpperCamelCase__							) as datafile:
																				lowercase_					=      []
																				lowercase_					=      False
																				lowercase_					=      False
																				for line in datafile:
																									if "# To replace in: " in line and "##" not in line:
																														lowercase_					=      line.split("""\""""							)[1]
																														lowercase_					=      skip_units(UpperCamelCase__							)
																									elif "# Below: " in line and "##" not in line:
																														lowercase_					=      line.split("""\""""							)[1]
																														lowercase_					=      skip_units(UpperCamelCase__							)
																									elif "# End." in line and "##" not in line:
																														if not skip_file and not skip_snippet:
																																			replace(UpperCamelCase__				,       UpperCamelCase__				,       UpperCamelCase__							)

																														lowercase_					=      []
																									elif "# Replace with" in line and "##" not in line:
																														lowercase_					=      []
																									elif "##" not in line:
																														lines_to_copy.append(UpperCamelCase__							)

															remove(UpperCamelCase__							)

										replace_in_files(F'''{directory}/to_replace_{lowercase_model_name}.py'''							)
										os.rmdir(UpperCamelCase__							)





import json
import os
import pickle
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers import is_faiss_available
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bart.tokenization_bart import BartTokenizer
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.dpr.tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer
from transformers.models.rag.configuration_rag import RagConfig
from transformers.models.rag.retrieval_rag import CustomHFIndex, RagRetriever
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_sentencepiece, require_tokenizers, require_torch


if is_faiss_available():
					import faiss







@require_faiss
class 					lowercase							(			lowercase_     ):




				def        a				(   self			):
											snake_case_     							=       tempfile.mkdtemp()
											snake_case_     							=       8

											# DPR tok
											snake_case_     							=       [
											    '[UNK]',
											    '[CLS]',
											    '[SEP]',
											    '[PAD]',
											    '[MASK]',
											    'want',
											    '##want',
											    '##ed',
											    'wa',
											    'un',
											    'runn',
											    '##ing',
											    ',',
											    'low',
											    'lowest',
											]
											snake_case_     							=       os.path.join(self.tmpdirname ,  'dpr_tokenizer'			)
											os.makedirs(snake_case ,  exist_ok=snake_case			)
											snake_case_     							=       os.path.join(snake_case ,  DPR_VOCAB_FILES_NAMES['vocab_file']			)
											with open(self.vocab_file ,  'w' ,  encoding='utf-8'			) as vocab_writer:
																		vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]			)			)

											# BART tok
											snake_case_     							=       [
											    'l',
											    'o',
											    'w',
											    'e',
											    'r',
											    's',
											    't',
											    'i',
											    'd',
											    'n',
											    '\u0120',
											    '\u0120l',
											    '\u0120n',
											    '\u0120lo',
											    '\u0120low',
											    'er',
											    '\u0120lowest',
											    '\u0120newer',
											    '\u0120wider',
											    '<unk>',
											]
											snake_case_     							=       dict(zip(snake_case ,  range(len(snake_case			)			)			)			)
											snake_case_     							=       ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
											snake_case_     							=       {'unk_token': '<unk>'}

											snake_case_     							=       os.path.join(self.tmpdirname ,  'bart_tokenizer'			)
											os.makedirs(snake_case ,  exist_ok=snake_case			)
											snake_case_     							=       os.path.join(snake_case ,  BART_VOCAB_FILES_NAMES['vocab_file']			)
											snake_case_     							=       os.path.join(snake_case ,  BART_VOCAB_FILES_NAMES['merges_file']			)
											with open(self.vocab_file ,  'w' ,  encoding='utf-8'			) as fp:
																		fp.write(json.dumps(snake_case			) + '\n'			)
											with open(self.merges_file ,  'w' ,  encoding='utf-8'			) as fp:
																		fp.write('\n'.join(snake_case			)			)




				def        a				(   self			):
											return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname ,  'dpr_tokenizer'			)			)




				def        a				(   self			):
											return DPRContextEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname ,  'dpr_tokenizer'			)			)




				def        a				(   self			):
											return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname ,  'bart_tokenizer'			)			)




				def        a				(   self			):
											shutil.rmtree(self.tmpdirname			)




				def        a				(   self			):
											snake_case_     							=       Dataset.from_dict(
											    {
											        'id': ['0', '1'],
											        'text': ['foo', 'bar'],
											        'title': ['Foo', 'Bar'],
											        'embeddings': [np.ones(self.retrieval_vector_size			), 2 * np.ones(self.retrieval_vector_size			)],
											    }			)
											dataset.add_faiss_index('embeddings' ,  string_factory='Flat' ,  metric_type=faiss.METRIC_INNER_PRODUCT			)
											return dataset




				def        a				(   self			):
											snake_case_     							=       self.get_dummy_dataset()
											snake_case_     							=       RagConfig(
											    retrieval_vector_size=self.retrieval_vector_size ,  question_encoder=DPRConfig().to_dict() ,  generator=BartConfig().to_dict() ,  )
											with patch('transformers.models.rag.retrieval_rag.load_dataset'			) as mock_load_dataset:
																		snake_case_     							=       dataset
																		snake_case_     							=       RagRetriever(
																		    snake_case ,  question_encoder_tokenizer=self.get_dpr_tokenizer() ,  generator_tokenizer=self.get_bart_tokenizer() ,  )
											return retriever




				def        a				(   self ,  snake_case			):
											snake_case_     							=       self.get_dummy_dataset()
											snake_case_     							=       RagConfig(
											    retrieval_vector_size=self.retrieval_vector_size ,  question_encoder=DPRConfig().to_dict() ,  generator=BartConfig().to_dict() ,  index_name='custom' ,  )
											if from_disk:
																		snake_case_     							=       os.path.join(self.tmpdirname ,  'dataset'			)
																		snake_case_     							=       os.path.join(self.tmpdirname ,  'index.faiss'			)
																		dataset.get_index('embeddings'			).save(os.path.join(self.tmpdirname ,  'index.faiss'			)			)
																		dataset.drop_index('embeddings'			)
																		dataset.save_to_disk(os.path.join(self.tmpdirname ,  'dataset'			)			)
																		del dataset
																		snake_case_     							=       RagRetriever(
																		    snake_case ,  question_encoder_tokenizer=self.get_dpr_tokenizer() ,  generator_tokenizer=self.get_bart_tokenizer() ,  )
											else:
																		snake_case_     							=       RagRetriever(
																		    snake_case ,  question_encoder_tokenizer=self.get_dpr_tokenizer() ,  generator_tokenizer=self.get_bart_tokenizer() ,  index=CustomHFIndex(config.retrieval_vector_size ,  snake_case			) ,  )
											return retriever




				def        a				(   self			):
											snake_case_     							=       Dataset.from_dict(
											    {
											        'id': ['0', '1'],
											        'text': ['foo', 'bar'],
											        'title': ['Foo', 'Bar'],
											        'embeddings': [np.ones(self.retrieval_vector_size + 1			), 2 * np.ones(self.retrieval_vector_size + 1			)],
											    }			)
											dataset.add_faiss_index('embeddings' ,  string_factory='Flat' ,  metric_type=faiss.METRIC_INNER_PRODUCT			)

											snake_case_     							=       os.path.join(self.tmpdirname ,  'hf_bert_base.hnswSQ8_correct_phi_128.c_index'			)
											dataset.save_faiss_index('embeddings' ,  index_file_name + '.index.dpr'			)
											pickle.dump(dataset['id'] ,  open(index_file_name + '.index_meta.dpr' ,  'wb'			)			)

											snake_case_     							=       os.path.join(self.tmpdirname ,  'psgs_w100.tsv.pkl'			)
											snake_case_     							=       {sample['id']: [sample['text'], sample['title']] for sample in dataset}
											pickle.dump(snake_case ,  open(snake_case ,  'wb'			)			)

											snake_case_     							=       RagConfig(
											    retrieval_vector_size=self.retrieval_vector_size ,  question_encoder=DPRConfig().to_dict() ,  generator=BartConfig().to_dict() ,  index_name='legacy' ,  index_path=self.tmpdirname ,  )
											snake_case_     							=       RagRetriever(
											    snake_case ,  question_encoder_tokenizer=self.get_dpr_tokenizer() ,  generator_tokenizer=self.get_bart_tokenizer()			)
											return retriever




				def        a				(   self			):
											snake_case_     							=       1
											snake_case_     							=       self.get_dummy_canonical_hf_index_retriever()
											snake_case_     							=       np.array(
											    [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)] ,  dtype=np.floataa			)
											snake_case_							,       snake_case_							,       snake_case_     							=       retriever.retrieve(snake_case ,  n_docs=snake_case			)
											self.assertEqual(retrieved_doc_embeds.shape ,  (2, n_docs, self.retrieval_vector_size)			)
											self.assertEqual(len(snake_case			) ,  2			)
											self.assertEqual(sorted(doc_dicts[0]			) ,  ['embeddings', 'id', 'text', 'title']			)
											self.assertEqual(len(doc_dicts[0]['id']			) ,  snake_case			)
											self.assertEqual(doc_dicts[0]['id'][0] ,  '1'			)  # max inner product is reached with second doc
											self.assertEqual(doc_dicts[1]['id'][0] ,  '0'			)  # max inner product is reached with first doc
											self.assertListEqual(doc_ids.tolist() ,  [[1], [0]]			)




				def        a				(   self			):
											snake_case_     							=       self.get_dummy_canonical_hf_index_retriever()
											with tempfile.TemporaryDirectory() as tmp_dirname:
																		with patch('transformers.models.rag.retrieval_rag.load_dataset'			) as mock_load_dataset:
																									snake_case_     							=       self.get_dummy_dataset()
																									retriever.save_pretrained(snake_case			)
																									snake_case_     							=       RagRetriever.from_pretrained(snake_case			)
																									self.assertIsInstance(snake_case ,  snake_case			)
																									snake_case_     							=       np.array(
																									    [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)] ,  dtype=np.floataa			)
																									snake_case_     							=       retriever.retrieve(snake_case ,  n_docs=1			)
																									self.assertTrue(out is not None			)




				def        a				(   self			):
											snake_case_     							=       1
											snake_case_     							=       self.get_dummy_custom_hf_index_retriever(from_disk=snake_case			)
											snake_case_     							=       np.array(
											    [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)] ,  dtype=np.floataa			)
											snake_case_							,       snake_case_							,       snake_case_     							=       retriever.retrieve(snake_case ,  n_docs=snake_case			)
											self.assertEqual(retrieved_doc_embeds.shape ,  (2, n_docs, self.retrieval_vector_size)			)
											self.assertEqual(len(snake_case			) ,  2			)
											self.assertEqual(sorted(doc_dicts[0]			) ,  ['embeddings', 'id', 'text', 'title']			)
											self.assertEqual(len(doc_dicts[0]['id']			) ,  snake_case			)
											self.assertEqual(doc_dicts[0]['id'][0] ,  '1'			)  # max inner product is reached with second doc
											self.assertEqual(doc_dicts[1]['id'][0] ,  '0'			)  # max inner product is reached with first doc
											self.assertListEqual(doc_ids.tolist() ,  [[1], [0]]			)




				def        a				(   self			):
											snake_case_     							=       self.get_dummy_custom_hf_index_retriever(from_disk=snake_case			)
											with tempfile.TemporaryDirectory() as tmp_dirname:
																		retriever.save_pretrained(snake_case			)
																		snake_case_     							=       RagRetriever.from_pretrained(snake_case			)
																		self.assertIsInstance(snake_case ,  snake_case			)
																		snake_case_     							=       np.array(
																		    [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)] ,  dtype=np.floataa			)
																		snake_case_     							=       retriever.retrieve(snake_case ,  n_docs=1			)
																		self.assertTrue(out is not None			)




				def        a				(   self			):
											snake_case_     							=       1
											snake_case_     							=       self.get_dummy_custom_hf_index_retriever(from_disk=snake_case			)
											snake_case_     							=       np.array(
											    [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)] ,  dtype=np.floataa			)
											snake_case_							,       snake_case_							,       snake_case_     							=       retriever.retrieve(snake_case ,  n_docs=snake_case			)
											self.assertEqual(retrieved_doc_embeds.shape ,  (2, n_docs, self.retrieval_vector_size)			)
											self.assertEqual(len(snake_case			) ,  2			)
											self.assertEqual(sorted(doc_dicts[0]			) ,  ['embeddings', 'id', 'text', 'title']			)
											self.assertEqual(len(doc_dicts[0]['id']			) ,  snake_case			)
											self.assertEqual(doc_dicts[0]['id'][0] ,  '1'			)  # max inner product is reached with second doc
											self.assertEqual(doc_dicts[1]['id'][0] ,  '0'			)  # max inner product is reached with first doc
											self.assertListEqual(doc_ids.tolist() ,  [[1], [0]]			)




				def        a				(   self			):
											snake_case_     							=       self.get_dummy_custom_hf_index_retriever(from_disk=snake_case			)
											with tempfile.TemporaryDirectory() as tmp_dirname:
																		retriever.save_pretrained(snake_case			)
																		snake_case_     							=       RagRetriever.from_pretrained(snake_case			)
																		self.assertIsInstance(snake_case ,  snake_case			)
																		snake_case_     							=       np.array(
																		    [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)] ,  dtype=np.floataa			)
																		snake_case_     							=       retriever.retrieve(snake_case ,  n_docs=1			)
																		self.assertTrue(out is not None			)




				def        a				(   self			):
											snake_case_     							=       1
											snake_case_     							=       self.get_dummy_legacy_index_retriever()
											snake_case_     							=       np.array(
											    [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)] ,  dtype=np.floataa			)
											snake_case_							,       snake_case_							,       snake_case_     							=       retriever.retrieve(snake_case ,  n_docs=snake_case			)
											self.assertEqual(retrieved_doc_embeds.shape ,  (2, n_docs, self.retrieval_vector_size)			)
											self.assertEqual(len(snake_case			) ,  2			)
											self.assertEqual(sorted(doc_dicts[0]			) ,  ['text', 'title']			)
											self.assertEqual(len(doc_dicts[0]['text']			) ,  snake_case			)
											self.assertEqual(doc_dicts[0]['text'][0] ,  'bar'			)  # max inner product is reached with second doc
											self.assertEqual(doc_dicts[1]['text'][0] ,  'foo'			)  # max inner product is reached with first doc
											self.assertListEqual(doc_ids.tolist() ,  [[1], [0]]			)




				def        a				(   self			):
											snake_case_     							=       self.get_dummy_legacy_index_retriever()
											with tempfile.TemporaryDirectory() as tmp_dirname:
																		retriever.save_pretrained(snake_case			)
																		snake_case_     							=       RagRetriever.from_pretrained(snake_case			)
																		self.assertIsInstance(snake_case ,  snake_case			)
																		snake_case_     							=       np.array(
																		    [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)] ,  dtype=np.floataa			)
																		snake_case_     							=       retriever.retrieve(snake_case ,  n_docs=1			)
																		self.assertTrue(out is not None			)




				@require_torch
				@require_tokenizers
				@require_sentencepiece
				def        a				(   self			):
											import torch

											snake_case_     							=       1
											snake_case_     							=       self.get_dummy_canonical_hf_index_retriever()
											snake_case_     							=       [[5, 7], [10, 11]]
											snake_case_     							=       np.array(
											    [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)] ,  dtype=np.floataa			)
											snake_case_     							=       retriever(snake_case ,  snake_case ,  prefix=retriever.config.generator.prefix ,  n_docs=snake_case			)
											snake_case_							,       snake_case_							,       snake_case_     							=       (
											    out['context_input_ids'],
											    out['context_attention_mask'],
											    out['retrieved_doc_embeds'],
											)
											self.assertEqual(retrieved_doc_embeds.shape ,  (2, n_docs, self.retrieval_vector_size)			)
											self.assertIsInstance(snake_case ,  snake_case			)
											self.assertIsInstance(snake_case ,  snake_case			)
											self.assertIsInstance(snake_case ,  np.ndarray			)

											snake_case_     							=       retriever(
											    snake_case ,  snake_case ,  prefix=retriever.config.generator.prefix ,  n_docs=snake_case ,  return_tensors='pt' ,  )
											snake_case_							,       snake_case_							,       snake_case_							,       snake_case_     							=       (  # noqa: F841
											    out['context_input_ids'],
											    out['context_attention_mask'],
											    out['retrieved_doc_embeds'],
											    out['doc_ids'],
											)
											self.assertEqual(retrieved_doc_embeds.shape ,  (2, n_docs, self.retrieval_vector_size)			)
											self.assertIsInstance(snake_case ,  torch.Tensor			)
											self.assertIsInstance(snake_case ,  torch.Tensor			)
											self.assertIsInstance(snake_case ,  torch.Tensor			)




				@require_torch
				@require_tokenizers
				@require_sentencepiece
				def        a				(   self			):
											snake_case_     							=       self.get_dpr_ctx_encoder_tokenizer()
											snake_case_     							=       1
											snake_case_     							=       self.get_dummy_custom_hf_index_retriever(from_disk=snake_case			)
											retriever.set_ctx_encoder_tokenizer(snake_case			)

											snake_case_     							=       [[5, 7], [10, 11]]
											snake_case_     							=       np.array(
											    [np.ones(self.retrieval_vector_size			), -np.ones(self.retrieval_vector_size			)] ,  dtype=np.floataa			)
											snake_case_     							=       retriever(snake_case ,  snake_case ,  prefix=retriever.config.generator.prefix ,  n_docs=snake_case			)

											self.assertEqual(
											    len(snake_case			) ,  6			)  # check whether the retriever output consist of 6 attributes including tokenized docs
											self.assertEqual(
											    all(k in out for k in ('tokenized_doc_ids', 'tokenized_doc_attention_mask')			) ,  snake_case			)  # check for doc token related keys in dictionary.




