
import json
import os
import pickle
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers import is_faiss_available
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bart.tokenization_bart import BartTokenizer
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.dpr.tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer
from transformers.models.rag.configuration_rag import RagConfig
from transformers.models.rag.retrieval_rag import CustomHFIndex, RagRetriever
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_sentencepiece, require_tokenizers, require_torch


if is_faiss_available():
						import faiss






@require_faiss
class        __lowercase		(       _A							):


					def 							lowerCAmelCase_     (						self    )		->				str:





									'''simple docstring'''
									A_													=	tempfile.mkdtemp()
									A_													=	8

									# DPR tok
									A_													=	[
									    "[UNK]",
									    "[CLS]",
									    "[SEP]",
									    "[PAD]",
									    "[MASK]",
									    "want",
									    "##want",
									    "##ed",
									    "wa",
									    "un",
									    "runn",
									    "##ing",
									    ",",
									    "low",
									    "lowest",
									]
									A_													=	os.path.join(self.tmpdirname							,							'''dpr_tokenizer'''    )
									os.makedirs(a_							,							exist_ok=a_    )
									A_													=	os.path.join(a_							,							DPR_VOCAB_FILES_NAMES['''vocab_file''']    )
									with open(self.vocab_file							,							'''w'''							,							encoding='''utf-8'''    ) as vocab_writer:
													vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]    )    )

									# BART tok
									A_													=	[
									    "l",
									    "o",
									    "w",
									    "e",
									    "r",
									    "s",
									    "t",
									    "i",
									    "d",
									    "n",
									    "\u0120",
									    "\u0120l",
									    "\u0120n",
									    "\u0120lo",
									    "\u0120low",
									    "er",
									    "\u0120lowest",
									    "\u0120newer",
									    "\u0120wider",
									    "<unk>",
									]
									A_													=	dict(zip(a_							,							range(len(a_    )    )    )    )
									A_													=	["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
									A_													=	{"unk_token": "<unk>"}

									A_													=	os.path.join(self.tmpdirname							,							'''bart_tokenizer'''    )
									os.makedirs(a_							,							exist_ok=a_    )
									A_													=	os.path.join(a_							,							BART_VOCAB_FILES_NAMES['''vocab_file''']    )
									A_													=	os.path.join(a_							,							BART_VOCAB_FILES_NAMES['''merges_file''']    )
									with open(self.vocab_file							,							'''w'''							,							encoding='''utf-8'''    ) as fp:
													fp.write(json.dumps(a_    ) + '''\n'''    )
									with open(self.merges_file							,							'''w'''							,							encoding='''utf-8'''    ) as fp:
													fp.write('''\n'''.join(a_    )    )


					def 							lowerCAmelCase_     (						self    )		->				List[Any]:





									'''simple docstring'''
									return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname							,							'''dpr_tokenizer'''    )    )


					def 							lowerCAmelCase_     (						self    )		->				Optional[Any]:





									'''simple docstring'''
									return DPRContextEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname							,							'''dpr_tokenizer'''    )    )


					def 							lowerCAmelCase_     (						self    )		->				List[Any]:





									'''simple docstring'''
									return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname							,							'''bart_tokenizer'''    )    )


					def 							lowerCAmelCase_     (						self    )		->				Optional[Any]:





									'''simple docstring'''
									shutil.rmtree(self.tmpdirname    )


					def 							lowerCAmelCase_     (						self    )		->				Optional[int]:





									'''simple docstring'''
									A_													=	Dataset.from_dict(
									    {
									        '''id''': ['''0''', '''1'''],
									        '''text''': ['''foo''', '''bar'''],
									        '''title''': ['''Foo''', '''Bar'''],
									        '''embeddings''': [np.ones(self.retrieval_vector_size    ), 2 * np.ones(self.retrieval_vector_size    )],
									    }    )
									dataset.add_faiss_index('''embeddings'''							,							string_factory='''Flat'''							,							metric_type=faiss.METRIC_INNER_PRODUCT    )
									return dataset


					def 							lowerCAmelCase_     (						self    )		->				str:





									'''simple docstring'''
									A_													=	self.get_dummy_dataset()
									A_													=	RagConfig(
									    retrieval_vector_size=self.retrieval_vector_size							,							question_encoder=DPRConfig().to_dict()							,							generator=BartConfig().to_dict()							,							)
									with patch('''transformers.models.rag.retrieval_rag.load_dataset'''    ) as mock_load_dataset:
													A_													=	dataset
													A_													=	RagRetriever(
													    a_							,							question_encoder_tokenizer=self.get_dpr_tokenizer()							,							generator_tokenizer=self.get_bart_tokenizer()							,							)
									return retriever


					def 							lowerCAmelCase_     (						self							,							a__    )		->				Any:





									'''simple docstring'''
									A_													=	self.get_dummy_dataset()
									A_													=	RagConfig(
									    retrieval_vector_size=self.retrieval_vector_size							,							question_encoder=DPRConfig().to_dict()							,							generator=BartConfig().to_dict()							,							index_name='''custom'''							,							)
									if from_disk:
													A_													=	os.path.join(self.tmpdirname							,							'''dataset'''    )
													A_													=	os.path.join(self.tmpdirname							,							'''index.faiss'''    )
													dataset.get_index('''embeddings'''    ).save(os.path.join(self.tmpdirname							,							'''index.faiss'''    )    )
													dataset.drop_index('''embeddings'''    )
													dataset.save_to_disk(os.path.join(self.tmpdirname							,							'''dataset'''    )    )
													del dataset
													A_													=	RagRetriever(
													    a_							,							question_encoder_tokenizer=self.get_dpr_tokenizer()							,							generator_tokenizer=self.get_bart_tokenizer()							,							)
									else:
													A_													=	RagRetriever(
													    a_							,							question_encoder_tokenizer=self.get_dpr_tokenizer()							,							generator_tokenizer=self.get_bart_tokenizer()							,							index=CustomHFIndex(config.retrieval_vector_size							,							a_    )							,							)
									return retriever


					def 							lowerCAmelCase_     (						self    )		->				Optional[int]:





									'''simple docstring'''
									A_													=	Dataset.from_dict(
									    {
									        '''id''': ['''0''', '''1'''],
									        '''text''': ['''foo''', '''bar'''],
									        '''title''': ['''Foo''', '''Bar'''],
									        '''embeddings''': [np.ones(self.retrieval_vector_size + 1    ), 2 * np.ones(self.retrieval_vector_size + 1    )],
									    }    )
									dataset.add_faiss_index('''embeddings'''							,							string_factory='''Flat'''							,							metric_type=faiss.METRIC_INNER_PRODUCT    )

									A_													=	os.path.join(self.tmpdirname							,							'''hf_bert_base.hnswSQ8_correct_phi_128.c_index'''    )
									dataset.save_faiss_index('''embeddings'''							,							index_file_name + '''.index.dpr'''    )
									pickle.dump(dataset['''id''']							,							open(index_file_name + '''.index_meta.dpr'''							,							'''wb'''    )    )

									A_													=	os.path.join(self.tmpdirname							,							'''psgs_w100.tsv.pkl'''    )
									A_													=	{sample["id"]: [sample["text"], sample["title"]] for sample in dataset}
									pickle.dump(a_							,							open(a_							,							'''wb'''    )    )

									A_													=	RagConfig(
									    retrieval_vector_size=self.retrieval_vector_size							,							question_encoder=DPRConfig().to_dict()							,							generator=BartConfig().to_dict()							,							index_name='''legacy'''							,							index_path=self.tmpdirname							,							)
									A_													=	RagRetriever(
									    a_							,							question_encoder_tokenizer=self.get_dpr_tokenizer()							,							generator_tokenizer=self.get_bart_tokenizer()    )
									return retriever


					def 							lowerCAmelCase_     (						self    )		->				Tuple:





									'''simple docstring'''
									A_													=	1
									A_													=	self.get_dummy_canonical_hf_index_retriever()
									A_													=	np.array(
									    [np.ones(self.retrieval_vector_size    ), -np.ones(self.retrieval_vector_size    )]							,							dtype=np.floataa    )
									A_													=	retriever.retrieve(a_							,							n_docs=a_    )
									self.assertEqual(retrieved_doc_embeds.shape							,							(2, n_docs, self.retrieval_vector_size)    )
									self.assertEqual(len(a_    )							,							2    )
									self.assertEqual(sorted(doc_dicts[0]    )							,							['''embeddings''', '''id''', '''text''', '''title''']    )
									self.assertEqual(len(doc_dicts[0]['''id''']    )							,							a_    )
									self.assertEqual(doc_dicts[0]['''id'''][0]							,							'''1'''    )  # max inner product is reached with second doc
									self.assertEqual(doc_dicts[1]['''id'''][0]							,							'''0'''    )  # max inner product is reached with first doc
									self.assertListEqual(doc_ids.tolist()							,							[[1], [0]]    )


					def 							lowerCAmelCase_     (						self    )		->				Optional[int]:





									'''simple docstring'''
									A_													=	self.get_dummy_canonical_hf_index_retriever()
									with tempfile.TemporaryDirectory() as tmp_dirname:
													with patch('''transformers.models.rag.retrieval_rag.load_dataset'''    ) as mock_load_dataset:
																	A_													=	self.get_dummy_dataset()
																	retriever.save_pretrained(a_    )
																	A_													=	RagRetriever.from_pretrained(a_    )
																	self.assertIsInstance(a_							,							a_    )
																	A_													=	np.array(
																	    [np.ones(self.retrieval_vector_size    ), -np.ones(self.retrieval_vector_size    )]							,							dtype=np.floataa    )
																	A_													=	retriever.retrieve(a_							,							n_docs=1    )
																	self.assertTrue(out is not None    )


					def 							lowerCAmelCase_     (						self    )		->				Any:





									'''simple docstring'''
									A_													=	1
									A_													=	self.get_dummy_custom_hf_index_retriever(from_disk=a_    )
									A_													=	np.array(
									    [np.ones(self.retrieval_vector_size    ), -np.ones(self.retrieval_vector_size    )]							,							dtype=np.floataa    )
									A_													=	retriever.retrieve(a_							,							n_docs=a_    )
									self.assertEqual(retrieved_doc_embeds.shape							,							(2, n_docs, self.retrieval_vector_size)    )
									self.assertEqual(len(a_    )							,							2    )
									self.assertEqual(sorted(doc_dicts[0]    )							,							['''embeddings''', '''id''', '''text''', '''title''']    )
									self.assertEqual(len(doc_dicts[0]['''id''']    )							,							a_    )
									self.assertEqual(doc_dicts[0]['''id'''][0]							,							'''1'''    )  # max inner product is reached with second doc
									self.assertEqual(doc_dicts[1]['''id'''][0]							,							'''0'''    )  # max inner product is reached with first doc
									self.assertListEqual(doc_ids.tolist()							,							[[1], [0]]    )


					def 							lowerCAmelCase_     (						self    )		->				Tuple:





									'''simple docstring'''
									A_													=	self.get_dummy_custom_hf_index_retriever(from_disk=a_    )
									with tempfile.TemporaryDirectory() as tmp_dirname:
													retriever.save_pretrained(a_    )
													A_													=	RagRetriever.from_pretrained(a_    )
													self.assertIsInstance(a_							,							a_    )
													A_													=	np.array(
													    [np.ones(self.retrieval_vector_size    ), -np.ones(self.retrieval_vector_size    )]							,							dtype=np.floataa    )
													A_													=	retriever.retrieve(a_							,							n_docs=1    )
													self.assertTrue(out is not None    )


					def 							lowerCAmelCase_     (						self    )		->				List[Any]:





									'''simple docstring'''
									A_													=	1
									A_													=	self.get_dummy_custom_hf_index_retriever(from_disk=a_    )
									A_													=	np.array(
									    [np.ones(self.retrieval_vector_size    ), -np.ones(self.retrieval_vector_size    )]							,							dtype=np.floataa    )
									A_													=	retriever.retrieve(a_							,							n_docs=a_    )
									self.assertEqual(retrieved_doc_embeds.shape							,							(2, n_docs, self.retrieval_vector_size)    )
									self.assertEqual(len(a_    )							,							2    )
									self.assertEqual(sorted(doc_dicts[0]    )							,							['''embeddings''', '''id''', '''text''', '''title''']    )
									self.assertEqual(len(doc_dicts[0]['''id''']    )							,							a_    )
									self.assertEqual(doc_dicts[0]['''id'''][0]							,							'''1'''    )  # max inner product is reached with second doc
									self.assertEqual(doc_dicts[1]['''id'''][0]							,							'''0'''    )  # max inner product is reached with first doc
									self.assertListEqual(doc_ids.tolist()							,							[[1], [0]]    )


					def 							lowerCAmelCase_     (						self    )		->				str:





									'''simple docstring'''
									A_													=	self.get_dummy_custom_hf_index_retriever(from_disk=a_    )
									with tempfile.TemporaryDirectory() as tmp_dirname:
													retriever.save_pretrained(a_    )
													A_													=	RagRetriever.from_pretrained(a_    )
													self.assertIsInstance(a_							,							a_    )
													A_													=	np.array(
													    [np.ones(self.retrieval_vector_size    ), -np.ones(self.retrieval_vector_size    )]							,							dtype=np.floataa    )
													A_													=	retriever.retrieve(a_							,							n_docs=1    )
													self.assertTrue(out is not None    )


					def 							lowerCAmelCase_     (						self    )		->				Tuple:





									'''simple docstring'''
									A_													=	1
									A_													=	self.get_dummy_legacy_index_retriever()
									A_													=	np.array(
									    [np.ones(self.retrieval_vector_size    ), -np.ones(self.retrieval_vector_size    )]							,							dtype=np.floataa    )
									A_													=	retriever.retrieve(a_							,							n_docs=a_    )
									self.assertEqual(retrieved_doc_embeds.shape							,							(2, n_docs, self.retrieval_vector_size)    )
									self.assertEqual(len(a_    )							,							2    )
									self.assertEqual(sorted(doc_dicts[0]    )							,							['''text''', '''title''']    )
									self.assertEqual(len(doc_dicts[0]['''text''']    )							,							a_    )
									self.assertEqual(doc_dicts[0]['''text'''][0]							,							'''bar'''    )  # max inner product is reached with second doc
									self.assertEqual(doc_dicts[1]['''text'''][0]							,							'''foo'''    )  # max inner product is reached with first doc
									self.assertListEqual(doc_ids.tolist()							,							[[1], [0]]    )


					def 							lowerCAmelCase_     (						self    )		->				List[Any]:





									'''simple docstring'''
									A_													=	self.get_dummy_legacy_index_retriever()
									with tempfile.TemporaryDirectory() as tmp_dirname:
													retriever.save_pretrained(a_    )
													A_													=	RagRetriever.from_pretrained(a_    )
													self.assertIsInstance(a_							,							a_    )
													A_													=	np.array(
													    [np.ones(self.retrieval_vector_size    ), -np.ones(self.retrieval_vector_size    )]							,							dtype=np.floataa    )
													A_													=	retriever.retrieve(a_							,							n_docs=1    )
													self.assertTrue(out is not None    )


					@require_torch
					@require_tokenizers
					@require_sentencepiece
					def 							lowerCAmelCase_     (						self    )		->				int:





									'''simple docstring'''
									import torch

									A_													=	1
									A_													=	self.get_dummy_canonical_hf_index_retriever()
									A_													=	[[5, 7], [1_0, 1_1]]
									A_													=	np.array(
									    [np.ones(self.retrieval_vector_size    ), -np.ones(self.retrieval_vector_size    )]							,							dtype=np.floataa    )
									A_													=	retriever(a_							,							a_							,							prefix=retriever.config.generator.prefix							,							n_docs=a_    )
									A_													=	(
									    out["context_input_ids"],
									    out["context_attention_mask"],
									    out["retrieved_doc_embeds"],
									)
									self.assertEqual(retrieved_doc_embeds.shape							,							(2, n_docs, self.retrieval_vector_size)    )
									self.assertIsInstance(a_							,							a_    )
									self.assertIsInstance(a_							,							a_    )
									self.assertIsInstance(a_							,							np.ndarray    )

									A_													=	retriever(
									    a_							,							a_							,							prefix=retriever.config.generator.prefix							,							n_docs=a_							,							return_tensors='''pt'''							,							)
									A_													=	(  # noqa: F841
									    out["context_input_ids"],
									    out["context_attention_mask"],
									    out["retrieved_doc_embeds"],
									    out["doc_ids"],
									)
									self.assertEqual(retrieved_doc_embeds.shape							,							(2, n_docs, self.retrieval_vector_size)    )
									self.assertIsInstance(a_							,							torch.Tensor    )
									self.assertIsInstance(a_							,							torch.Tensor    )
									self.assertIsInstance(a_							,							torch.Tensor    )


					@require_torch
					@require_tokenizers
					@require_sentencepiece
					def 							lowerCAmelCase_     (						self    )		->				int:





									'''simple docstring'''
									A_													=	self.get_dpr_ctx_encoder_tokenizer()
									A_													=	1
									A_													=	self.get_dummy_custom_hf_index_retriever(from_disk=a_    )
									retriever.set_ctx_encoder_tokenizer(a_    )

									A_													=	[[5, 7], [1_0, 1_1]]
									A_													=	np.array(
									    [np.ones(self.retrieval_vector_size    ), -np.ones(self.retrieval_vector_size    )]							,							dtype=np.floataa    )
									A_													=	retriever(a_							,							a_							,							prefix=retriever.config.generator.prefix							,							n_docs=a_    )

									self.assertEqual(
									    len(a_    )							,							6    )  # check whether the retriever output consist of 6 attributes including tokenized docs
									self.assertEqual(
									    all(k in out for k in ('''tokenized_doc_ids''', '''tokenized_doc_attention_mask''')    )							,							a_    )  # check for doc token related keys in dictionary.






'''simple docstring'''






from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase									=   logging.get_logger(__name__)

lowerCAmelCase									=   {
    """EleutherAI/gpt-neox-20b""": """https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/config.json""",
    # See all GPTNeoX models at https://huggingface.co/models?filter=gpt_neox
}
class 		lowerCamelCase	(   _A							):
	snake_case_   = "gpt_neox"



	def __init__( self		,   a_=50_432		,   a_=6_144		,   a_=44		,   a_=64		,   a_=24_576		,   a_="gelu"		,   a_=0.25		,   a_=10_000		,   a_=0.0		,   a_=0.0		,   a_=0.1		,   a_=2_048		,   a_=0.02		,   a_=1e-5		,   a_=True		,   a_=0		,   a_=2		,   a_=False		,   a_=True		,   a_=None		,   **a_		,   ):
			super().__init__(bos_token_id=a_		,   eos_token_id=a_		,   **a_						)
			lowerCAmelCase  :     Dict       						= vocab_size
			lowerCAmelCase  :     str       						= max_position_embeddings
			lowerCAmelCase  :     int       						= hidden_size
			lowerCAmelCase  :     Optional[Any]       						= num_hidden_layers
			lowerCAmelCase  :     int       						= num_attention_heads
			lowerCAmelCase  :     Any       						= intermediate_size
			lowerCAmelCase  :     Tuple       						= hidden_act
			lowerCAmelCase  :     Any       						= rotary_pct
			lowerCAmelCase  :     str       						= rotary_emb_base
			lowerCAmelCase  :     Tuple       						= attention_dropout
			lowerCAmelCase  :     Optional[int]       						= hidden_dropout
			lowerCAmelCase  :     int       						= classifier_dropout
			lowerCAmelCase  :     str       						= initializer_range
			lowerCAmelCase  :     str       						= layer_norm_eps
			lowerCAmelCase  :     List[Any]       						= use_cache
			lowerCAmelCase  :     Dict       						= tie_word_embeddings
			lowerCAmelCase  :     Any       						= use_parallel_residual
			lowerCAmelCase  :     Union[str, Any]       						= rope_scaling
			self._rope_scaling_validation()

			if self.hidden_size % self.num_attention_heads != 0:
					raise ValueError(
					    "The hidden size is not divisble by the number of attention heads! Make sure to update them!"						)





	def 	_lowerCamelCase     ( self						):
			if self.rope_scaling is None:
					return

			if not isinstance(self.rope_scaling		,   a_						) or len(self.rope_scaling						) != 2:
					raise ValueError(
					    "`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, "
					    F'''got {self.rope_scaling}'''						)
			lowerCAmelCase  :     int       						= self.rope_scaling.get("type"		,   a_						)
			lowerCAmelCase  :     int       						= self.rope_scaling.get("factor"		,   a_						)
			if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
					raise ValueError(
					    F'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''						)
			if rope_scaling_factor is None or not isinstance(a_		,   a_						) or rope_scaling_factor <= 1.0:
					raise ValueError(F'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''						)



