





'''simple docstring'''






import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


A   : int             =			logging.getLogger(__name__)







class 			lowerCamelCase					:


					def __init__(					self	:  List[str]       ):





											'''simple docstring'''





											_snake_case:   Union[str, Any]  					=		False


					def    SCREAMING_SNAKE_CASE_		(					self	:  Dict       ,     __snake_case	:  Tuple       ,     __snake_case	:  Union[str, Any]       ,     __snake_case	:  Any       ,     __snake_case	:  Any       ):





											'''simple docstring'''





											if not self.initialized:
																	_snake_case:   Tuple  					=		RagRetriever(
																	    UpperCAmelCase__       ,     question_encoder_tokenizer=UpperCAmelCase__       ,     generator_tokenizer=UpperCAmelCase__       ,     index=UpperCAmelCase__       ,     init_retrieval=UpperCAmelCase__       ,     )
																	_snake_case:   str  					=		True


					def    SCREAMING_SNAKE_CASE_		(					self	:  Dict       ):





											'''simple docstring'''





											self.retriever.index.init_index()




					def    SCREAMING_SNAKE_CASE_		(					self	:  List[str]       ,     __snake_case	:  Dict       ,     __snake_case	:  int       ):





											'''simple docstring'''





											_snake_case:   str  					=		self.retriever._main_retrieve(UpperCAmelCase__       ,     UpperCAmelCase__       )
											return doc_ids, retrieved_doc_embeds







class 			lowerCamelCase					(						__UpperCAmelCase  ):


					def __init__(					self	:  Optional[int]       ,     __snake_case	:  Tuple       ,     __snake_case	:  Union[str, Any]       ,     __snake_case	:  List[Any]       ,     __snake_case	:  Union[str, Any]       ,     __snake_case	:  Optional[Any]=None       ):





											'''simple docstring'''





											if index is not None and index.is_initialized() and len(UpperCAmelCase__       ) > 0:
																	raise ValueError(
																	    'When using Ray for distributed fine-tuning, '
																	    'you\'ll need to provide the paths instead, '
																	    'as the dataset and the index are loaded '
																	    'separately. More info in examples/rag/use_own_knowledge_dataset.py '       )
											super().__init__(
											    UpperCAmelCase__       ,     question_encoder_tokenizer=UpperCAmelCase__       ,     generator_tokenizer=UpperCAmelCase__       ,     index=UpperCAmelCase__       ,     init_retrieval=UpperCAmelCase__       ,     )
											_snake_case:   Union[str, Any]  					=		retrieval_workers
											if len(self.retrieval_workers       ) > 0:
																	ray.get(
																	    [
																	        worker.create_rag_retriever.remote(UpperCAmelCase__       ,     UpperCAmelCase__       ,     UpperCAmelCase__       ,     UpperCAmelCase__       )
																	        for worker in self.retrieval_workers
																	    ]       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  List[Any]       ):





											'''simple docstring'''





											logger.info('initializing retrieval'       )

											if len(self.retrieval_workers       ) > 0:
																	ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]       )
											else:
																	# Non-distributed training. Load index into this same process.
																	self.index.init_index()


					def    SCREAMING_SNAKE_CASE_		(					self	:  Any       ,     __snake_case	:  Any       ,     __snake_case	:  List[str]       ):





											'''simple docstring'''





											if len(self.retrieval_workers       ) > 0:
																	# Select a random retrieval actor.
																	_snake_case:   Tuple  					=		self.retrieval_workers[random.randint(0       ,     len(self.retrieval_workers       ) - 1       )]
																	_snake_case:   Dict  					=		ray.get(random_worker.retrieve.remote(UpperCAmelCase__       ,     UpperCAmelCase__       )       )
											else:
																	_snake_case:   List[Any]  					=		self._main_retrieve(UpperCAmelCase__       ,     UpperCAmelCase__       )
											return retrieved_doc_embeds, doc_ids, self.index.get_doc_dicts(UpperCAmelCase__       )


					@classmethod
					def    SCREAMING_SNAKE_CASE_		(					cls	:  Dict       ,     __snake_case	:  Any       ,     __snake_case	:  Optional[Any]=None       ,     **__snake_case	:  str       ):





											'''simple docstring'''





											return super(UpperCAmelCase__       ,     cls       ).get_tokenizers(UpperCAmelCase__       ,     UpperCAmelCase__       ,     **UpperCAmelCase__       )




					@classmethod
					def    SCREAMING_SNAKE_CASE_		(					cls	:  Dict       ,     __snake_case	:  Union[str, Any]       ,     __snake_case	:  str       ,     __snake_case	:  Any=None       ,     **__snake_case	:  Optional[Any]       ):





											'''simple docstring'''





											_snake_case:   Union[str, Any]  					=		kwargs.pop('config'       ,     UpperCAmelCase__       ) or RagConfig.from_pretrained(UpperCAmelCase__       ,     **UpperCAmelCase__       )
											_snake_case:   Any  					=		RagTokenizer.from_pretrained(UpperCAmelCase__       ,     config=UpperCAmelCase__       )
											_snake_case:   Tuple  					=		rag_tokenizer.question_encoder
											_snake_case:   int  					=		rag_tokenizer.generator
											if indexed_dataset is not None:
																	_snake_case:   Tuple  					=		'''custom'''
																	_snake_case:   Union[str, Any]  					=		CustomHFIndex(config.retrieval_vector_size       ,     UpperCAmelCase__       )
											else:
																	_snake_case:   List[Any]  					=		cls._build_index(UpperCAmelCase__       )
											return cls(
											    UpperCAmelCase__       ,     question_encoder_tokenizer=UpperCAmelCase__       ,     generator_tokenizer=UpperCAmelCase__       ,     retrieval_workers=UpperCAmelCase__       ,     index=UpperCAmelCase__       ,     )






'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_flax_available, is_torch_available


A   : Dict             =			{'configuration_speech_encoder_decoder': ['SpeechEncoderDecoderConfig']}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		A   : int             =			['SpeechEncoderDecoderModel']

try:
		if not is_flax_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		A   : Union[str, Any]             =			['FlaxSpeechEncoderDecoderModel']

if TYPE_CHECKING:
		from .configuration_speech_encoder_decoder import SpeechEncoderDecoderConfig

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_speech_encoder_decoder import SpeechEncoderDecoderModel

		try:
				if not is_flax_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_flax_speech_encoder_decoder import FlaxSpeechEncoderDecoderModel

else:
		import sys

		A   : int             =			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





