


'''simple docstring'''





import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


UpperCAmelCase__  			=				logging.getLogger(__name__)
class 						a__					:





			'''simple docstring'''

			def __init__(  self							:			Union[str, Any]	)      ->		List[str]:
						__A=	False
			def 					lowerCAmelCase     (  self							:			Any			,    lowerCAmelCase_							:			Optional[int]			,    lowerCAmelCase_							:			Union[str, Any]			,    lowerCAmelCase_							:			str			,    lowerCAmelCase_							:			Dict	)      ->		List[str]:
						if not self.initialized:
									__A=	RagRetriever(
									    lowerCAmelCase_			,    question_encoder_tokenizer=lowerCAmelCase_			,    generator_tokenizer=lowerCAmelCase_			,    index=lowerCAmelCase_			,    init_retrieval=lowerCAmelCase_			,    )
									__A=	True
			def 					lowerCAmelCase     (  self							:			Optional[Any]	)      ->		Any:
						self.retriever.index.init_index()

			def 					lowerCAmelCase     (  self							:			int			,    lowerCAmelCase_							:			Optional[Any]			,    lowerCAmelCase_							:			Any	)      ->		List[Any]:
						__A,	__A=	self.retriever._main_retrieve(lowerCAmelCase_			,    lowerCAmelCase_	)
						return doc_ids, retrieved_doc_embeds


class 						a__					(					a_   ):





			'''simple docstring'''

			def __init__(  self							:			List[Any]			,    lowerCAmelCase_							:			int			,    lowerCAmelCase_							:			List[Any]			,    lowerCAmelCase_							:			Tuple			,    lowerCAmelCase_							:			Union[str, Any]			,    lowerCAmelCase_							:			Tuple=None	)      ->		Optional[int]:
						if index is not None and index.is_initialized() and len(lowerCAmelCase_	) > 0:
									raise ValueError(
									    'When using Ray for distributed fine-tuning, '
									    'you\'ll need to provide the paths instead, '
									    'as the dataset and the index are loaded '
									    'separately. More info in examples/rag/use_own_knowledge_dataset.py '	)
						super().__init__(
						    lowerCAmelCase_			,    question_encoder_tokenizer=lowerCAmelCase_			,    generator_tokenizer=lowerCAmelCase_			,    index=lowerCAmelCase_			,    init_retrieval=lowerCAmelCase_			,    )
						__A=	retrieval_workers
						if len(self.retrieval_workers	) > 0:
									ray.get(
									    [
									        worker.create_rag_retriever.remote(lowerCAmelCase_			,    lowerCAmelCase_			,    lowerCAmelCase_			,    lowerCAmelCase_	)
									        for worker in self.retrieval_workers
									    ]	)
			def 					lowerCAmelCase     (  self							:			Optional[int]	)      ->		Dict:
						logger.info('initializing retrieval'	)

						if len(self.retrieval_workers	) > 0:
									ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]	)
						else:
									# Non-distributed training. Load index into this same process.
									self.index.init_index()
			def 					lowerCAmelCase     (  self							:			List[Any]			,    lowerCAmelCase_							:			Any			,    lowerCAmelCase_							:			Any	)      ->		Union[str, Any]:
						if len(self.retrieval_workers	) > 0:
									# Select a random retrieval actor.
									__A=	self.retrieval_workers[random.randint(0			,    len(self.retrieval_workers	) - 1	)]
									__A,	__A=	ray.get(random_worker.retrieve.remote(lowerCAmelCase_			,    lowerCAmelCase_	)	)
						else:
									__A,	__A=	self._main_retrieve(lowerCAmelCase_			,    lowerCAmelCase_	)
						return retrieved_doc_embeds, doc_ids, self.index.get_doc_dicts(lowerCAmelCase_	)
			@classmethod
			def 					lowerCAmelCase     (  cls							:			List[Any]			,    lowerCAmelCase_							:			Optional[Any]			,    lowerCAmelCase_							:			Dict=None			,    **lowerCAmelCase_							:			str	)      ->		Tuple:
						return super(lowerCAmelCase_			,    cls	).get_tokenizers(lowerCAmelCase_			,    lowerCAmelCase_			,    **lowerCAmelCase_	)

			@classmethod
			def 					lowerCAmelCase     (  cls							:			Union[str, Any]			,    lowerCAmelCase_							:			Optional[Any]			,    lowerCAmelCase_							:			str			,    lowerCAmelCase_							:			List[str]=None			,    **lowerCAmelCase_							:			Tuple	)      ->		Any:
						__A=	kwargs.pop('config'			,    lowerCAmelCase_	) or RagConfig.from_pretrained(lowerCAmelCase_			,    **lowerCAmelCase_	)
						__A=	RagTokenizer.from_pretrained(lowerCAmelCase_			,    config=lowerCAmelCase_	)
						__A=	rag_tokenizer.question_encoder
						__A=	rag_tokenizer.generator
						if indexed_dataset is not None:
									__A=	'custom'
									__A=	CustomHFIndex(config.retrieval_vector_size			,    lowerCAmelCase_	)
						else:
									__A=	cls._build_index(lowerCAmelCase_	)
						return cls(
						    lowerCAmelCase_			,    question_encoder_tokenizer=lowerCAmelCase_			,    generator_tokenizer=lowerCAmelCase_			,    retrieval_workers=lowerCAmelCase_			,    index=lowerCAmelCase_			,    )







'''simple docstring'''





import numpy as np

from transformers import Pipeline






def 						UpperCAmelCase__(				_SCREAMING_SNAKE_CASE					: Any ):






			"""simple docstring"""




			__A=	np.max(_SCREAMING_SNAKE_CASE,axis=-1,keepdims=_SCREAMING_SNAKE_CASE )
			__A=	np.exp(outputs - maxes )
			return shifted_exp / shifted_exp.sum(axis=-1,keepdims=_SCREAMING_SNAKE_CASE )






class 						a__					(					a_   ):





			'''simple docstring'''

			def 					lowerCAmelCase     (  self							:			Optional[Any]			,    **lowerCAmelCase_							:			List[str]	)      ->		List[str]:
						__A=	{}
						if "second_text" in kwargs:
									__A=	kwargs['second_text']
						return preprocess_kwargs, {}, {}
			def 					lowerCAmelCase     (  self							:			str			,    lowerCAmelCase_							:			Union[str, Any]			,    lowerCAmelCase_							:			Optional[int]=None	)      ->		int:
						return self.tokenizer(lowerCAmelCase_			,    text_pair=lowerCAmelCase_			,    return_tensors=self.framework	)
			def 					lowerCAmelCase     (  self							:			Dict			,    lowerCAmelCase_							:			List[str]	)      ->		Tuple:
						return self.model(**lowerCAmelCase_	)

			def 					lowerCAmelCase     (  self							:			str			,    lowerCAmelCase_							:			Dict	)      ->		Optional[int]:
						__A=	model_outputs.logits[0].numpy()
						__A=	softmax(lowerCAmelCase_	)

						__A=	np.argmax(lowerCAmelCase_	)
						__A=	self.model.config.idalabel[best_class]
						__A=	probabilities[best_class].item()
						__A=	logits.tolist()
						return {"label": label, "score": score, "logits": logits}



