


import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


UpperCAmelCase			=    logging.getLogger(__name__)







class lowerCAmelCase_						:

 '''simple docstring'''

 def __init__(	self					):
       snake_case_       =      False



 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase					):
       if not self.initialized:
             snake_case_       =      RagRetriever(
                 _UpperCAmelCase		,      question_encoder_tokenizer=_UpperCAmelCase		,      generator_tokenizer=_UpperCAmelCase		,      index=_UpperCAmelCase		,      init_retrieval=_UpperCAmelCase		,      )
             snake_case_       =      True



 def 						UpperCamelCase__     (	self					):
       self.retriever.index.init_index()



 def 						UpperCamelCase__     (	self					):
       # delete the old self.retriever object before assigning the new index
       del self.retriever
       snake_case_       =      False





 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase		,      _UpperCAmelCase					):
       snake_case_       ,    snake_case_       =      self.retriever._main_retrieve(_UpperCAmelCase		,      _UpperCAmelCase					)
       snake_case_       =      self.retriever.index.get_doc_dicts(_UpperCAmelCase					)
       return doc_ids, retrieved_doc_embeds, doc_dicts







class lowerCAmelCase_						( lowerCamelCase__       ):

 '''simple docstring'''

 def __init__(	self		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase=None					):
       if index is not None and index.is_initialized() and len(_UpperCAmelCase					) > 0:
             raise ValueError(
                 '''When using Ray for distributed fine-tuning, '''
                 '''you\'ll need to provide the paths instead, '''
                 '''as the dataset and the index are loaded '''
                 '''separately. More info in examples/rag/use_own_knowledge_dataset.py '''					)

       super().__init__(
           _UpperCAmelCase		,      question_encoder_tokenizer=_UpperCAmelCase		,      generator_tokenizer=_UpperCAmelCase		,      index=_UpperCAmelCase		,      init_retrieval=_UpperCAmelCase		,      )

       snake_case_       =      retrieval_workers
       snake_case_       =      question_encoder_tokenizer
       snake_case_       =      generator_tokenizer
       if len(self.retrieval_workers					) > 0:
             ray.get(
                 [
                     worker.create_rag_retriever.remote(_UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase					)
                     for worker in self.retrieval_workers
                 ]					)



 def 						UpperCamelCase__     (	self					):
       logger.info('''initializing retrieval'''					)

       if len(self.retrieval_workers					) > 0:
             ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]					)
       else:
             # Non-distributed training. Load index into this same process.
             self.index.init_index()



 def 						UpperCamelCase__     (	self		,      _UpperCAmelCase		,      _UpperCAmelCase					):
       if len(self.retrieval_workers					) > 0:
             # Select a random retrieval actor.
             snake_case_       =      self.retrieval_workers[random.randint(0		,      len(self.retrieval_workers					) - 1					)]
             snake_case_       ,    snake_case_       ,    snake_case_       =      ray.get(
                 random_worker.retrieve.remote(_UpperCAmelCase		,      _UpperCAmelCase					)					)
       else:
             snake_case_       ,    snake_case_       =      self._main_retrieve(_UpperCAmelCase		,      _UpperCAmelCase					)
             snake_case_       =      self.index.get_doc_dicts(_UpperCAmelCase					)
       return retrieved_doc_embeds, doc_ids, doc_dicts



 @classmethod
 def 						UpperCamelCase__     (	cls		,      _UpperCAmelCase		,      _UpperCAmelCase=None		,      **_UpperCAmelCase					):
       return super(_UpperCAmelCase		,      cls					).get_tokenizers(_UpperCAmelCase		,      _UpperCAmelCase		,      **_UpperCAmelCase					)



 @classmethod
 def 						UpperCamelCase__     (	cls		,      _UpperCAmelCase		,      _UpperCAmelCase		,      _UpperCAmelCase=None		,      **_UpperCAmelCase					):
       snake_case_       =      kwargs.pop('''config'''		,      _UpperCAmelCase					) or RagConfig.from_pretrained(_UpperCAmelCase		,      **_UpperCAmelCase					)
       snake_case_       =      RagTokenizer.from_pretrained(_UpperCAmelCase		,      config=_UpperCAmelCase					)
       snake_case_       =      rag_tokenizer.question_encoder
       snake_case_       =      rag_tokenizer.generator

       if indexed_dataset is not None:
             snake_case_       =      '''custom'''
             snake_case_       =      CustomHFIndex(config.retrieval_vector_size		,      _UpperCAmelCase					)
       else:
             snake_case_       =      cls._build_index(_UpperCAmelCase					)

       return cls(
           _UpperCAmelCase		,      question_encoder_tokenizer=_UpperCAmelCase		,      generator_tokenizer=_UpperCAmelCase		,      retrieval_workers=_UpperCAmelCase		,      index=_UpperCAmelCase		,      )





 def 						UpperCamelCase__     (	self					):
       logger.info('''re-loading the new dataset with embeddings'''					)
       # access from the training loop

       ray.get([worker.clear_object.remote() for worker in self.retrieval_workers]					)

       # build the index object again
       snake_case_       =      self._build_index(self.config					)

       ray.get(
           [
               worker.create_rag_retriever.remote(
                   self.config		,      self.question_encoder_tokenizer		,      self.generator_tokenizer		,      _UpperCAmelCase					)
               for worker in self.retrieval_workers
           ]					)



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCAmelCase			=    {
    """configuration_big_bird""": ["""BIG_BIRD_PRETRAINED_CONFIG_ARCHIVE_MAP""", """BigBirdConfig""", """BigBirdOnnxConfig"""],
}

try:
   if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase			=    ["""BigBirdTokenizer"""]

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase			=    ["""BigBirdTokenizerFast"""]

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase			=    [
       """BIG_BIRD_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """BigBirdForCausalLM""",
       """BigBirdForMaskedLM""",
       """BigBirdForMultipleChoice""",
       """BigBirdForPreTraining""",
       """BigBirdForQuestionAnswering""",
       """BigBirdForSequenceClassification""",
       """BigBirdForTokenClassification""",
       """BigBirdLayer""",
       """BigBirdModel""",
       """BigBirdPreTrainedModel""",
       """load_tf_weights_in_big_bird""",
   ]

try:
   if not is_flax_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase			=    [
       """FlaxBigBirdForCausalLM""",
       """FlaxBigBirdForMaskedLM""",
       """FlaxBigBirdForMultipleChoice""",
       """FlaxBigBirdForPreTraining""",
       """FlaxBigBirdForQuestionAnswering""",
       """FlaxBigBirdForSequenceClassification""",
       """FlaxBigBirdForTokenClassification""",
       """FlaxBigBirdModel""",
       """FlaxBigBirdPreTrainedModel""",
   ]

if TYPE_CHECKING:
   from .configuration_big_bird import BIG_BIRD_PRETRAINED_CONFIG_ARCHIVE_MAP, BigBirdConfig, BigBirdOnnxConfig

   try:
      if not is_sentencepiece_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_big_bird import BigBirdTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_big_bird_fast import BigBirdTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_big_bird import (
          BIG_BIRD_PRETRAINED_MODEL_ARCHIVE_LIST,
          BigBirdForCausalLM,
          BigBirdForMaskedLM,
          BigBirdForMultipleChoice,
          BigBirdForPreTraining,
          BigBirdForQuestionAnswering,
          BigBirdForSequenceClassification,
          BigBirdForTokenClassification,
          BigBirdLayer,
          BigBirdModel,
          BigBirdPreTrainedModel,
          load_tf_weights_in_big_bird,
      )

   try:
      if not is_flax_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_flax_big_bird import (
          FlaxBigBirdForCausalLM,
          FlaxBigBirdForMaskedLM,
          FlaxBigBirdForMultipleChoice,
          FlaxBigBirdForPreTraining,
          FlaxBigBirdForQuestionAnswering,
          FlaxBigBirdForSequenceClassification,
          FlaxBigBirdForTokenClassification,
          FlaxBigBirdModel,
          FlaxBigBirdPreTrainedModel,
      )

else:
   import sys

   UpperCAmelCase			=    _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)