






import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


A_						:						str						      =					logging.getLogger(__name__)




class 							_lowerCAmelCase:





       """simple docstring"""







       def __init__(   self				):
         UpperCamelCase_:   Optional[int]        =							False





       def      _a       (   self    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase				):
         if not self.initialized:
           UpperCamelCase_:   Optional[Any]        =							RagRetriever(
               _lowerCamelCase    ,     question_encoder_tokenizer=_lowerCamelCase    ,     generator_tokenizer=_lowerCamelCase    ,     index=_lowerCamelCase    ,     init_retrieval=_lowerCamelCase    ,     )
           UpperCamelCase_:   str        =							True





       def      _a       (   self				):
         self.retriever.index.init_index()





       def      _a       (   self    ,     _lowerCamelCase    ,     _lowerCamelCase				):
         UpperCamelCase_ ,UpperCamelCase_:   Any        =							self.retriever._main_retrieve(_lowerCamelCase    ,     _lowerCamelCase				)
         return doc_ids, retrieved_doc_embeds







class 							_lowerCAmelCase(			UpperCAmelCase_   ):





       """simple docstring"""







       def __init__(   self    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase=None				):
         if index is not None and index.is_initialized() and len(_lowerCamelCase				) > 0:
           raise ValueError(
               'When using Ray for distributed fine-tuning, '
               'you\'ll need to provide the paths instead, '
               'as the dataset and the index are loaded '
               'separately. More info in examples/rag/use_own_knowledge_dataset.py '				)
         super().__init__(
             _lowerCamelCase    ,     question_encoder_tokenizer=_lowerCamelCase    ,     generator_tokenizer=_lowerCamelCase    ,     index=_lowerCamelCase    ,     init_retrieval=_lowerCamelCase    ,     )
         UpperCamelCase_:   List[str]        =							retrieval_workers
         if len(self.retrieval_workers				) > 0:
           ray.get(
               [
                   worker.create_rag_retriever.remote(_lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase				)
                   for worker in self.retrieval_workers
               ]				)





       def      _a       (   self				):
         logger.info('initializing retrieval'				)

         if len(self.retrieval_workers				) > 0:
           ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]				)
         else:
           # Non-distributed training. Load index into this same process.
           self.index.init_index()





       def      _a       (   self    ,     _lowerCamelCase    ,     _lowerCamelCase				):
         if len(self.retrieval_workers				) > 0:
           # Select a random retrieval actor.
           UpperCamelCase_:   Union[str, Any]        =							self.retrieval_workers[random.randint(0    ,     len(self.retrieval_workers				) - 1				)]
           UpperCamelCase_ ,UpperCamelCase_:   str        =							ray.get(random_worker.retrieve.remote(_lowerCamelCase    ,     _lowerCamelCase				)				)
         else:
           UpperCamelCase_ ,UpperCamelCase_:   Dict        =							self._main_retrieve(_lowerCamelCase    ,     _lowerCamelCase				)
         return retrieved_doc_embeds, doc_ids, self.index.get_doc_dicts(_lowerCamelCase				)





       @classmethod
       def      _a       (   cls    ,     _lowerCamelCase    ,     _lowerCamelCase=None    ,     **_lowerCamelCase				):
         return super(_lowerCamelCase    ,     cls				).get_tokenizers(_lowerCamelCase    ,     _lowerCamelCase    ,     **_lowerCamelCase				)





       @classmethod
       def      _a       (   cls    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase=None    ,     **_lowerCamelCase				):
         UpperCamelCase_:   List[str]        =							kwargs.pop('config'    ,     _lowerCamelCase				) or RagConfig.from_pretrained(_lowerCamelCase    ,     **_lowerCamelCase				)
         UpperCamelCase_:   Optional[int]        =							RagTokenizer.from_pretrained(_lowerCamelCase    ,     config=_lowerCamelCase				)
         UpperCamelCase_:   List[str]        =							rag_tokenizer.question_encoder
         UpperCamelCase_:   List[Any]        =							rag_tokenizer.generator
         if indexed_dataset is not None:
           UpperCamelCase_:   Union[str, Any]        =							'custom'
           UpperCamelCase_:   int        =							CustomHFIndex(config.retrieval_vector_size    ,     _lowerCamelCase				)
         else:
           UpperCamelCase_:   str        =							cls._build_index(_lowerCamelCase				)
         return cls(
             _lowerCamelCase    ,     question_encoder_tokenizer=_lowerCamelCase    ,     generator_tokenizer=_lowerCamelCase    ,     retrieval_workers=_lowerCamelCase    ,     index=_lowerCamelCase    ,     )




'''simple docstring'''
import unittest

from transformers import AutoConfig, AutoTokenizer, BertConfig, TensorType, is_flax_available
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, slow


if is_flax_available():
					import jax

					from transformers.models.auto.modeling_flax_auto import FlaxAutoModel
					from transformers.models.bert.modeling_flax_bert import FlaxBertModel
					from transformers.models.roberta.modeling_flax_roberta import FlaxRobertaModel






@require_flax
class 				lowercase__      (				unittest.TestCase  ):





	@slow
	def       UpperCamelCase_    (					self   :							Union[str, Any]					):







							'''simple docstring'''
							for model_name in ["bert-base-cased", "bert-large-uncased"]:
													with self.subTest(lowerCamelCase__					):
																			_UpperCamelCase   :						Optional[int]												=  AutoConfig.from_pretrained(lowerCamelCase__					)
																			self.assertIsNotNone(lowerCamelCase__					)
																			self.assertIsInstance(lowerCamelCase__					,lowerCamelCase__					)

																			_UpperCamelCase   :						Dict												=  FlaxAutoModel.from_pretrained(lowerCamelCase__					)
																			self.assertIsNotNone(lowerCamelCase__					)
																			self.assertIsInstance(lowerCamelCase__					,lowerCamelCase__					)





	@slow
	def       UpperCamelCase_    (					self   :							List[Any]					):







							'''simple docstring'''
							for model_name in ["roberta-base", "roberta-large"]:
													with self.subTest(lowerCamelCase__					):
																			_UpperCamelCase   :						Any												=  AutoConfig.from_pretrained(lowerCamelCase__					)
																			self.assertIsNotNone(lowerCamelCase__					)
																			self.assertIsInstance(lowerCamelCase__					,lowerCamelCase__					)

																			_UpperCamelCase   :						Any												=  FlaxAutoModel.from_pretrained(lowerCamelCase__					)
																			self.assertIsNotNone(lowerCamelCase__					)
																			self.assertIsInstance(lowerCamelCase__					,lowerCamelCase__					)





	@slow
	def       UpperCamelCase_    (					self   :							Union[str, Any]					):







							'''simple docstring'''
							for model_name in ["bert-base-cased", "bert-large-uncased"]:
													_UpperCamelCase   :						Optional[int]												=  AutoTokenizer.from_pretrained(lowerCamelCase__					)
													_UpperCamelCase   :						List[Any]												=  FlaxBertModel.from_pretrained(lowerCamelCase__					)
													_UpperCamelCase   :						Optional[int]												=  tokenizer('Do you support jax jitted function?'					,return_tensors=TensorType.JAX					)

													@jax.jit
													def eval(**lowerCamelCase__   :							Union[str, Any]					):
																			return model(**lowerCamelCase__					)

													eval(**lowerCamelCase__					).block_until_ready()





	@slow
	def       UpperCamelCase_    (					self   :							Dict					):







							'''simple docstring'''
							for model_name in ["roberta-base", "roberta-large"]:
													_UpperCamelCase   :						Optional[int]												=  AutoTokenizer.from_pretrained(lowerCamelCase__					)
													_UpperCamelCase   :						Tuple												=  FlaxRobertaModel.from_pretrained(lowerCamelCase__					)
													_UpperCamelCase   :						Optional[Any]												=  tokenizer('Do you support jax jitted function?'					,return_tensors=TensorType.JAX					)

													@jax.jit
													def eval(**lowerCamelCase__   :							Union[str, Any]					):
																			return model(**lowerCamelCase__					)

													eval(**lowerCamelCase__					).block_until_ready()





	def       UpperCamelCase_    (					self   :							Union[str, Any]					):







							'''simple docstring'''
							with self.assertRaisesRegex(
							    lowerCamelCase__					,'bert-base is not a local folder and is not a valid model identifier'					):
													_UpperCamelCase   :						int												=  FlaxAutoModel.from_pretrained('bert-base'					)





	def       UpperCamelCase_    (					self   :							str					):







							'''simple docstring'''
							with self.assertRaisesRegex(
							    lowerCamelCase__					,R'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'					):
													_UpperCamelCase   :						Tuple												=  FlaxAutoModel.from_pretrained(lowerCamelCase__					,revision='aaaaaa'					)





	def       UpperCamelCase_    (					self   :							List[str]					):







							'''simple docstring'''
							with self.assertRaisesRegex(
							    lowerCamelCase__					,'hf-internal-testing/config-no-model does not appear to have a file named flax_model.msgpack'					,):
													_UpperCamelCase   :						List[Any]												=  FlaxAutoModel.from_pretrained('hf-internal-testing/config-no-model'					)





	def       UpperCamelCase_    (					self   :							List[str]					):







							'''simple docstring'''
							with self.assertRaisesRegex(lowerCamelCase__					,'Use `from_pt=True` to load this model'					):
													_UpperCamelCase   :						Tuple												=  FlaxAutoModel.from_pretrained('hf-internal-testing/tiny-bert-pt-only'					)


