



"""simple docstring"""
import copy

from ...configuration_utils import PretrainedConfig
from ...utils import add_start_docstrings


A:     Optional[Any]										=	R"\n    [`RagConfig`] stores the configuration of a *RagModel*. Configuration objects inherit from [`PretrainedConfig`] and\n    can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information.\n\n    Args:\n        title_sep (`str`, *optional*, defaults to  `\" / \"`):\n            Separator inserted between the title and the text of the retrieved document when calling [`RagRetriever`].\n        doc_sep (`str`, *optional*, defaults to  `\" // \"`):\n            Separator inserted between the text of the retrieved document and the original input when calling\n            [`RagRetriever`].\n        n_docs (`int`, *optional*, defaults to 5):\n            Number of documents to retrieve.\n        max_combined_length (`int`, *optional*, defaults to 300):\n            Max length of contextualized input returned by [`~RagRetriever.__call__`].\n        retrieval_vector_size (`int`, *optional*, defaults to 768):\n            Dimensionality of the document embeddings indexed by [`RagRetriever`].\n        retrieval_batch_size (`int`, *optional*, defaults to 8):\n            Retrieval batch size, defined as the number of queries issues concurrently to the faiss index encapsulated\n            [`RagRetriever`].\n        dataset (`str`, *optional*, defaults to `\"wiki_dpr\"`):\n            A dataset identifier of the indexed dataset in HuggingFace Datasets (list all available datasets and ids\n            using `datasets.list_datasets()`).\n        dataset_split (`str`, *optional*, defaults to `\"train\"`)\n            Which split of the `dataset` to load.\n        index_name (`str`, *optional*, defaults to `\"compressed\"`)\n            The index name of the index associated with the `dataset`. One can choose between `\"legacy\"`, `\"exact\"` and\n            `\"compressed\"`.\n        index_path (`str`, *optional*)\n            The path to the serialized faiss index on disk.\n        passages_path (`str`, *optional*):\n            A path to text passages compatible with the faiss index. Required if using\n            [`~models.rag.retrieval_rag.LegacyIndex`]\n        use_dummy_dataset (`bool`, *optional*, defaults to `False`)\n            Whether to load a \"dummy\" variant of the dataset specified by `dataset`.\n        label_smoothing (`float`, *optional*, defaults to 0.0):\n            Only relevant if `return_loss` is set to `True`. Controls the `epsilon` parameter value for label smoothing\n            in the loss calculation. If set to 0, no label smoothing is performed.\n        do_marginalize (`bool`, *optional*, defaults to `False`):\n            If `True`, the logits are marginalized over all documents by making use of\n            `torch.nn.functional.log_softmax`.\n        reduce_loss (`bool`, *optional*, defaults to `False`):\n            Whether or not to reduce the NLL loss using the `torch.Tensor.sum` operation.\n        do_deduplication (`bool`, *optional*, defaults to `True`):\n            Whether or not to deduplicate the generations from different context documents for a given input. Has to be\n            set to `False` if used while training with distributed backend.\n        exclude_bos_score (`bool`, *optional*, defaults to `False`):\n            Whether or not to disregard the BOS token when computing the loss.\n        output_retrieved(`bool`, *optional*, defaults to `False`):\n            If set to `True`, `retrieved_doc_embeds`, `retrieved_doc_ids`, `context_input_ids` and\n            `context_attention_mask` are returned. See returned tensors for more detail.\n        use_cache (`bool`, *optional*, defaults to `True`):\n            Whether or not the model should return the last key/values attentions (not used by all models).\n        forced_eos_token_id (`int`, *optional*):\n            The id of the token to force as the last generated token when `max_length` is reached. Usually set to\n            `eos_token_id`.\n"



@add_start_docstrings(UpperCamelCase__				)
class    SCREAMING_SNAKE_CASE__						(      UpperCamelCase__				):
   __lowerCAmelCase						:     int		=		'rag'
   __lowerCAmelCase						:     str		=		True

   def __init__(     self   ,  _SCREAMING_SNAKE_CASE=None   ,  _SCREAMING_SNAKE_CASE=True   ,  _SCREAMING_SNAKE_CASE=None   ,  _SCREAMING_SNAKE_CASE=None   ,  _SCREAMING_SNAKE_CASE=None   ,  _SCREAMING_SNAKE_CASE=None   ,  _SCREAMING_SNAKE_CASE=None   ,  _SCREAMING_SNAKE_CASE=" / "   ,  _SCREAMING_SNAKE_CASE=" // "   ,  _SCREAMING_SNAKE_CASE=5   ,  _SCREAMING_SNAKE_CASE=300   ,  _SCREAMING_SNAKE_CASE=768   ,  _SCREAMING_SNAKE_CASE=8   ,  _SCREAMING_SNAKE_CASE="wiki_dpr"   ,  _SCREAMING_SNAKE_CASE="train"   ,  _SCREAMING_SNAKE_CASE="compressed"   ,  _SCREAMING_SNAKE_CASE=None   ,  _SCREAMING_SNAKE_CASE=None   ,  _SCREAMING_SNAKE_CASE=False   ,  _SCREAMING_SNAKE_CASE=False   ,  _SCREAMING_SNAKE_CASE=0.0   ,  _SCREAMING_SNAKE_CASE=True   ,  _SCREAMING_SNAKE_CASE=False   ,  _SCREAMING_SNAKE_CASE=False   ,  _SCREAMING_SNAKE_CASE=False   ,  _SCREAMING_SNAKE_CASE=True   ,  _SCREAMING_SNAKE_CASE=None   ,  **_SCREAMING_SNAKE_CASE   ,  )						->   Dict:





    '''simple docstring'''




    super().__init__(
        bos_token_id=__A   ,  pad_token_id=__A   ,  eos_token_id=__A   ,  decoder_start_token_id=__A   ,  forced_eos_token_id=__A   ,  is_encoder_decoder=__A   ,  prefix=__A   ,  vocab_size=__A   ,  **__A   ,  )
    assert (
        "question_encoder" in kwargs and "generator" in kwargs
    ), "Config has to be initialized with question_encoder and generator config"
    UpperCAmelCase			:      Union[str, Any]			=			kwargs.pop("""question_encoder"""		)
    UpperCAmelCase			:      List[Any]			=			question_encoder_config.pop("""model_type"""		)
    UpperCAmelCase			:      Union[str, Any]			=			kwargs.pop("""generator"""		)
    UpperCAmelCase			:      Union[str, Any]			=			decoder_config.pop("""model_type"""		)

    from ..auto.configuration_auto import AutoConfig

    UpperCAmelCase			:      Any			=			AutoConfig.for_model(__A   ,  **__A		)
    UpperCAmelCase			:      Tuple			=			AutoConfig.for_model(__A   ,  **__A		)

    UpperCAmelCase			:      List[Any]			=			reduce_loss
    UpperCAmelCase			:      List[str]			=			label_smoothing
    UpperCAmelCase			:      List[Any]			=			exclude_bos_score
    UpperCAmelCase			:      List[str]			=			do_marginalize

    UpperCAmelCase			:      List[Any]			=			title_sep
    UpperCAmelCase			:      List[str]			=			doc_sep
    UpperCAmelCase			:      List[str]			=			n_docs
    UpperCAmelCase			:      int			=			max_combined_length

    UpperCAmelCase			:      Tuple			=			dataset
    UpperCAmelCase			:      str			=			dataset_split
    UpperCAmelCase			:      Optional[int]			=			index_name

    UpperCAmelCase			:      List[str]			=			retrieval_vector_size
    UpperCAmelCase			:      List[Any]			=			retrieval_batch_size
    UpperCAmelCase			:      Optional[int]			=			passages_path
    UpperCAmelCase			:      Union[str, Any]			=			index_path
    UpperCAmelCase			:      Any			=			use_dummy_dataset

    UpperCAmelCase			:      int			=			output_retrieved

    UpperCAmelCase			:      Any			=			do_deduplication

    UpperCAmelCase			:      Dict			=			use_cache

    if self.forced_eos_token_id is None:
     UpperCAmelCase			:      int			=			getattr(self.generator   ,  """forced_eos_token_id"""   ,  __A		)

   @classmethod
   def 			SCREAMING_SNAKE_CASE  (     cls   ,  _SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE   ,  **_SCREAMING_SNAKE_CASE		)						->   PretrainedConfig:





    '''simple docstring'''




    return cls(question_encoder=question_encoder_config.to_dict()   ,  generator=generator_config.to_dict()   ,  **__A		)




   def 			SCREAMING_SNAKE_CASE  (     self		)						->   int:





    '''simple docstring'''




    UpperCAmelCase			:      Union[str, Any]			=			copy.deepcopy(self.__dict__		)
    UpperCAmelCase			:      Optional[Any]			=			self.question_encoder.to_dict()
    UpperCAmelCase			:      List[Any]			=			self.generator.to_dict()
    UpperCAmelCase			:      Optional[int]			=			self.__class__.model_type
    return output







"""simple docstring"""
import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin



class    SCREAMING_SNAKE_CASE__						(      UpperCAmelCase__   ,    unittest.TestCase				):
   __lowerCAmelCase						:     List[Any]		=		TransfoXLTokenizer
   __lowerCAmelCase						:     Optional[int]		=		False
   __lowerCAmelCase						:     Optional[Any]		=		False

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   List[str]:





    '''simple docstring'''




    super().setUp()

    UpperCAmelCase			:      Optional[Any]			=			[
        """<unk>""",
        """[CLS]""",
        """[SEP]""",
        """want""",
        """unwanted""",
        """wa""",
        """un""",
        """running""",
        """,""",
        """low""",
        """l""",
    ]
    UpperCAmelCase			:      int			=			os.path.join(self.tmpdirname   ,  VOCAB_FILES_NAMES["""vocab_file"""]		)
    with open(self.vocab_file   ,  """w"""   ,  encoding="""utf-8"""		) as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]		)		)

   def 			SCREAMING_SNAKE_CASE  (     self   ,  **_SCREAMING_SNAKE_CASE		)						->   List[Any]:





    '''simple docstring'''




    UpperCAmelCase			:      Optional[Any]			=			True
    return TransfoXLTokenizer.from_pretrained(self.tmpdirname   ,  **_SCREAMING_SNAKE_CASE		)

   def 			SCREAMING_SNAKE_CASE  (     self   ,  _SCREAMING_SNAKE_CASE		)						->   List[Any]:





    '''simple docstring'''




    UpperCAmelCase			:      int			=			"""<unk> UNwanted , running"""
    UpperCAmelCase			:      Dict			=			"""<unk> unwanted, running"""
    return input_text, output_text

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Any:





    '''simple docstring'''




    UpperCAmelCase			:      List[str]			=			TransfoXLTokenizer(vocab_file=self.vocab_file   ,  lower_case=_SCREAMING_SNAKE_CASE		)

    UpperCAmelCase			:      Dict			=			tokenizer.tokenize("""<unk> UNwanted , running"""		)
    self.assertListEqual(_SCREAMING_SNAKE_CASE   ,  ["""<unk>""", """unwanted""", """,""", """running"""]		)

    self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE		)   ,  [0, 4, 8, 7]		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Tuple:





    '''simple docstring'''




    UpperCAmelCase			:      Tuple			=			TransfoXLTokenizer(lower_case=_SCREAMING_SNAKE_CASE		)

    self.assertListEqual(
        tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """		)   ,  ["""hello""", """!""", """how""", """are""", """you""", """?"""]		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Optional[Any]:





    '''simple docstring'''




    UpperCAmelCase			:      Optional[Any]			=			TransfoXLTokenizer(lower_case=_SCREAMING_SNAKE_CASE		)

    self.assertListEqual(
        tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """		)   ,  ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   int:





    '''simple docstring'''




    UpperCAmelCase			:      List[Any]			=			TransfoXLTokenizer(lower_case=_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      Dict			=			"""Hello (bracket) and side-scrolled [and] Henry's $5,000 with 3.34 m. What's up!?"""
    UpperCAmelCase			:      Optional[Any]			=			[
        """Hello""",
        """(""",
        """bracket""",
        """)""",
        """and""",
        """side""",
        """@-@""",
        """scrolled""",
        """[""",
        """and""",
        """]""",
        """Henry""",
        """'s""",
        """$""",
        """5""",
        """@,@""",
        """000""",
        """with""",
        """3""",
        """@.@""",
        """34""",
        """m""",
        """.""",
        """What""",
        """'s""",
        """up""",
        """!""",
        """?""",
    ]

    self.assertListEqual(tokenizer.tokenize(_SCREAMING_SNAKE_CASE		)   ,  _SCREAMING_SNAKE_CASE		)

    self.assertEqual(tokenizer.convert_tokens_to_string(_SCREAMING_SNAKE_CASE		)   ,  _SCREAMING_SNAKE_CASE		)




   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Dict:





    '''simple docstring'''




    UpperCAmelCase			:      List[str]			=			self.get_tokenizer()
    UpperCAmelCase			:      Tuple			=			len(_SCREAMING_SNAKE_CASE		)

    tokenizer.add_tokens(["""new1""", """new2"""]		)
    tokenizer.move_added_token("""new1"""   ,  1		)

    # Check that moved token is not copied (duplicate)
    self.assertEqual(len(_SCREAMING_SNAKE_CASE		)   ,  original_len + 2		)
    # Check that token is moved to specified id
    self.assertEqual(tokenizer.encode("""new1"""		)   ,  [1]		)
    self.assertEqual(tokenizer.decode([1]		)   ,  """new1"""		)
