





"""simple docstring"""





import tempfile
import unittest

import numpy as np
from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import BertConfig, is_flax_available
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_flax


if is_flax_available():
							import os

							from flax.core.frozen_dict import unfreeze
							from flax.traverse_util import flatten_dict

							from transformers import FlaxBertModel

							_UpperCamelCase					      =   '0.12'  # assumed parallelism: 8
@require_flax
@is_staging_test
class 		SCREAMING_SNAKE_CASE_				(     unittest.TestCase ):
				"""simple docstring"""




				@classmethod
				def        __lowercase	(  cls	:List[str]      ):
											__lowerCamelCase   :   Optional[Any]       =TOKEN
											HfFolder.save_token(UpperCamelCase_      )




				@classmethod
				def        __lowercase	(  cls	:Tuple      ):
											try:
																		delete_repo(token=cls._token    ,     repo_id='''test-model-flax'''      )
											except HTTPError:
																		pass

											try:
																		delete_repo(token=cls._token    ,     repo_id='''valid_org/test-model-flax-org'''      )
											except HTTPError:
																		pass




				def        __lowercase	(  self	:Tuple      ):
											__lowerCamelCase   :   Optional[int]       =BertConfig(
											    vocab_size=99    ,     hidden_size=32    ,     num_hidden_layers=5    ,     num_attention_heads=4    ,     intermediate_size=37      )
											__lowerCamelCase   :   List[str]       =FlaxBertModel(UpperCamelCase_      )
											model.push_to_hub('''test-model-flax'''    ,     use_auth_token=self._token      )

											__lowerCamelCase   :   Any       =FlaxBertModel.from_pretrained(f'{USER}/test-model-flax'      )

											__lowerCamelCase   :   List[str]       =flatten_dict(unfreeze(model.params      )      )
											__lowerCamelCase   :   List[Any]       =flatten_dict(unfreeze(new_model.params      )      )

											for key in base_params.keys():
																		__lowerCamelCase   :   Dict       =(base_params[key] - new_params[key]).sum().item()
																		self.assertLessEqual(UpperCamelCase_    ,     1e-3    ,     msg=f'{key} not identical'      )

											# Reset repo
											delete_repo(token=self._token    ,     repo_id='''test-model-flax'''      )

											# Push to hub via save_pretrained
											with tempfile.TemporaryDirectory() as tmp_dir:
																		model.save_pretrained(UpperCamelCase_    ,     repo_id='''test-model-flax'''    ,     push_to_hub=UpperCamelCase_    ,     use_auth_token=self._token      )

											__lowerCamelCase   :   Dict       =FlaxBertModel.from_pretrained(f'{USER}/test-model-flax'      )

											__lowerCamelCase   :   Any       =flatten_dict(unfreeze(model.params      )      )
											__lowerCamelCase   :   Optional[int]       =flatten_dict(unfreeze(new_model.params      )      )

											for key in base_params.keys():
																		__lowerCamelCase   :   Tuple       =(base_params[key] - new_params[key]).sum().item()
																		self.assertLessEqual(UpperCamelCase_    ,     1e-3    ,     msg=f'{key} not identical'      )




				def        __lowercase	(  self	:Optional[int]      ):
											__lowerCamelCase   :   Dict       =BertConfig(
											    vocab_size=99    ,     hidden_size=32    ,     num_hidden_layers=5    ,     num_attention_heads=4    ,     intermediate_size=37      )
											__lowerCamelCase   :   Dict       =FlaxBertModel(UpperCamelCase_      )
											model.push_to_hub('''valid_org/test-model-flax-org'''    ,     use_auth_token=self._token      )

											__lowerCamelCase   :   Optional[int]       =FlaxBertModel.from_pretrained('''valid_org/test-model-flax-org'''      )

											__lowerCamelCase   :   Tuple       =flatten_dict(unfreeze(model.params      )      )
											__lowerCamelCase   :   Union[str, Any]       =flatten_dict(unfreeze(new_model.params      )      )

											for key in base_params.keys():
																		__lowerCamelCase   :   Any       =(base_params[key] - new_params[key]).sum().item()
																		self.assertLessEqual(UpperCamelCase_    ,     1e-3    ,     msg=f'{key} not identical'      )

											# Reset repo
											delete_repo(token=self._token    ,     repo_id='''valid_org/test-model-flax-org'''      )

											# Push to hub via save_pretrained
											with tempfile.TemporaryDirectory() as tmp_dir:
																		model.save_pretrained(
																		    UpperCamelCase_    ,     repo_id='''valid_org/test-model-flax-org'''    ,     push_to_hub=UpperCamelCase_    ,     use_auth_token=self._token      )

											__lowerCamelCase   :   Any       =FlaxBertModel.from_pretrained('''valid_org/test-model-flax-org'''      )

											__lowerCamelCase   :   Optional[int]       =flatten_dict(unfreeze(model.params      )      )
											__lowerCamelCase   :   int       =flatten_dict(unfreeze(new_model.params      )      )

											for key in base_params.keys():
																		__lowerCamelCase   :   List[str]       =(base_params[key] - new_params[key]).sum().item()
																		self.assertLessEqual(UpperCamelCase_    ,     1e-3    ,     msg=f'{key} not identical'      )



def 					lowerCAmelCase_      (   SCREAMING_SNAKE_CASE							:				Any     ,    SCREAMING_SNAKE_CASE							:				Optional[Any] ):


							'''simple docstring'''







							__lowerCamelCase   :   List[str]       =True
							__lowerCamelCase   :   Optional[int]       =flatten_dict(modela.params )
							__lowerCamelCase   :   int       =flatten_dict(modela.params )
							for key in flat_params_a.keys():
														if np.sum(np.abs(flat_params_a[key] - flat_params_a[key] ) ) > 1E-4:
																					__lowerCamelCase   :   str       =False

							return models_are_equal
@require_flax
class 		SCREAMING_SNAKE_CASE_				(     unittest.TestCase ):
				"""simple docstring"""




				def        __lowercase	(  self	:Any      ):
											__lowerCamelCase   :   Union[str, Any]       =BertConfig.from_pretrained('''hf-internal-testing/tiny-bert-flax-only'''      )
											__lowerCamelCase   :   Optional[int]       =FlaxBertModel(UpperCamelCase_      )

											__lowerCamelCase   :   Dict       ='bert'
											with tempfile.TemporaryDirectory() as tmp_dir:
																		model.save_pretrained(os.path.join(UpperCamelCase_    ,     UpperCamelCase_      )      )

																		with self.assertRaises(UpperCamelCase_      ):
																									__lowerCamelCase   :   Optional[Any]       =FlaxBertModel.from_pretrained(UpperCamelCase_      )

																		__lowerCamelCase   :   Optional[Any]       =FlaxBertModel.from_pretrained(UpperCamelCase_    ,     subfolder=UpperCamelCase_      )

											self.assertTrue(check_models_equal(UpperCamelCase_    ,     UpperCamelCase_      )      )




				def        __lowercase	(  self	:Union[str, Any]      ):
											__lowerCamelCase   :   Any       =BertConfig.from_pretrained('''hf-internal-testing/tiny-bert-flax-only'''      )
											__lowerCamelCase   :   Optional[Any]       =FlaxBertModel(UpperCamelCase_      )

											__lowerCamelCase   :   Union[str, Any]       ='bert'
											with tempfile.TemporaryDirectory() as tmp_dir:
																		model.save_pretrained(os.path.join(UpperCamelCase_    ,     UpperCamelCase_      )    ,     max_shard_size='''10KB'''      )

																		with self.assertRaises(UpperCamelCase_      ):
																									__lowerCamelCase   :   Optional[Any]       =FlaxBertModel.from_pretrained(UpperCamelCase_      )

																		__lowerCamelCase   :   List[str]       =FlaxBertModel.from_pretrained(UpperCamelCase_    ,     subfolder=UpperCamelCase_      )

											self.assertTrue(check_models_equal(UpperCamelCase_    ,     UpperCamelCase_      )      )




				def        __lowercase	(  self	:int      ):
											__lowerCamelCase   :   int       ='bert'
											__lowerCamelCase   :   Dict       ='hf-internal-testing/tiny-random-bert-subfolder'

											with self.assertRaises(UpperCamelCase_      ):
																		__lowerCamelCase   :   Any       =FlaxBertModel.from_pretrained(UpperCamelCase_      )

											__lowerCamelCase   :   Dict       =FlaxBertModel.from_pretrained(UpperCamelCase_    ,     subfolder=UpperCamelCase_      )

											self.assertIsNotNone(UpperCamelCase_      )




				def        __lowercase	(  self	:Union[str, Any]      ):
											__lowerCamelCase   :   str       ='bert'
											__lowerCamelCase   :   Optional[int]       ='hf-internal-testing/tiny-random-bert-sharded-subfolder'
											with self.assertRaises(UpperCamelCase_      ):
																		__lowerCamelCase   :   Tuple       =FlaxBertModel.from_pretrained(UpperCamelCase_      )

											__lowerCamelCase   :   List[Any]       =FlaxBertModel.from_pretrained(UpperCamelCase_    ,     subfolder=UpperCamelCase_      )

											self.assertIsNotNone(UpperCamelCase_      )






"""simple docstring"""



import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


UpperCamelCase__          =	logging.getLogger(__name__)





class 				a			:



      def __init__(   self      ):
          UpperCAmelCase__	:      Union[str, Any]     							=					False



      def  __snake_case							(   self     ,	UpperCamelCase_     ,	UpperCamelCase_     ,	UpperCamelCase_     ,	UpperCamelCase_      ):
          if not self.initialized:
              UpperCAmelCase__	:      Optional[Any]     							=					RagRetriever(
                  UpperCamelCase_     ,	question_encoder_tokenizer=UpperCamelCase_     ,	generator_tokenizer=UpperCamelCase_     ,	index=UpperCamelCase_     ,	init_retrieval=UpperCamelCase_     ,	)
              UpperCAmelCase__	:      Union[str, Any]     							=					True



      def  __snake_case							(   self      ):
          self.retriever.index.init_index()



      def  __snake_case							(   self     ,	UpperCamelCase_     ,	UpperCamelCase_      ):
          UpperCAmelCase__							,							UpperCAmelCase__	:      Union[str, Any]     							=					self.retriever._main_retrieve(UpperCamelCase_     ,	UpperCamelCase_      )
          return doc_ids, retrieved_doc_embeds





class 				a			(					lowercase      ):



      def __init__(   self     ,	UpperCamelCase_     ,	UpperCamelCase_     ,	UpperCamelCase_     ,	UpperCamelCase_     ,	UpperCamelCase_=None      ):
          if index is not None and index.is_initialized() and len(UpperCamelCase_      ) > 0:
              raise ValueError(
                  'When using Ray for distributed fine-tuning, '
                  'you\'ll need to provide the paths instead, '
                  'as the dataset and the index are loaded '
                  'separately. More info in examples/rag/use_own_knowledge_dataset.py '      )
          super().__init__(
              UpperCamelCase_     ,	question_encoder_tokenizer=UpperCamelCase_     ,	generator_tokenizer=UpperCamelCase_     ,	index=UpperCamelCase_     ,	init_retrieval=UpperCamelCase_     ,	)
          UpperCAmelCase__	:      Union[str, Any]     							=					retrieval_workers
          if len(self.retrieval_workers      ) > 0:
              ray.get(
                  [
                      worker.create_rag_retriever.remote(UpperCamelCase_     ,	UpperCamelCase_     ,	UpperCamelCase_     ,	UpperCamelCase_      )
                      for worker in self.retrieval_workers
                  ]      )



      def  __snake_case							(   self      ):
          logger.info('initializing retrieval'      )

          if len(self.retrieval_workers      ) > 0:
              ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]      )
          else:
              # Non-distributed training. Load index into this same process.
              self.index.init_index()



      def  __snake_case							(   self     ,	UpperCamelCase_     ,	UpperCamelCase_      ):
          if len(self.retrieval_workers      ) > 0:
              # Select a random retrieval actor.
              UpperCAmelCase__	:      Optional[Any]     							=					self.retrieval_workers[random.randint(0     ,	len(self.retrieval_workers      ) - 1      )]
              UpperCAmelCase__							,							UpperCAmelCase__	:      List[Any]     							=					ray.get(random_worker.retrieve.remote(UpperCamelCase_     ,	UpperCamelCase_      )      )
          else:
              UpperCAmelCase__							,							UpperCAmelCase__	:      Dict     							=					self._main_retrieve(UpperCamelCase_     ,	UpperCamelCase_      )
          return retrieved_doc_embeds, doc_ids, self.index.get_doc_dicts(UpperCamelCase_      )



      @classmethod
      def  __snake_case							(   cls     ,	UpperCamelCase_     ,	UpperCamelCase_=None     ,	**UpperCamelCase_      ):
          return super(UpperCamelCase_     ,	cls      ).get_tokenizers(UpperCamelCase_     ,	UpperCamelCase_     ,	**UpperCamelCase_      )



      @classmethod
      def  __snake_case							(   cls     ,	UpperCamelCase_     ,	UpperCamelCase_     ,	UpperCamelCase_=None     ,	**UpperCamelCase_      ):
          UpperCAmelCase__	:      Union[str, Any]     							=					kwargs.pop('config'     ,	UpperCamelCase_      ) or RagConfig.from_pretrained(UpperCamelCase_     ,	**UpperCamelCase_      )
          UpperCAmelCase__	:      Union[str, Any]     							=					RagTokenizer.from_pretrained(UpperCamelCase_     ,	config=UpperCamelCase_      )
          UpperCAmelCase__	:      List[Any]     							=					rag_tokenizer.question_encoder
          UpperCAmelCase__	:      Optional[Any]     							=					rag_tokenizer.generator
          if indexed_dataset is not None:
              UpperCAmelCase__	:      List[str]     							=					'custom'
              UpperCAmelCase__	:      Tuple     							=					CustomHFIndex(config.retrieval_vector_size     ,	UpperCamelCase_      )
          else:
              UpperCAmelCase__	:      Optional[Any]     							=					cls._build_index(UpperCamelCase_      )
          return cls(
              UpperCamelCase_     ,	question_encoder_tokenizer=UpperCamelCase_     ,	generator_tokenizer=UpperCamelCase_     ,	retrieval_workers=UpperCamelCase_     ,	index=UpperCamelCase_     ,	)

