





'''simple docstring'''


import copy

from ...configuration_utils import PretrainedConfig
from ...utils import add_start_docstrings


lowerCAmelCase_   :   List[Any]       =					R"\n    [`RagConfig`] stores the configuration of a *RagModel*. Configuration objects inherit from [`PretrainedConfig`] and\n    can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information.\n\n    Args:\n        title_sep (`str`, *optional*, defaults to  `\" / \"`):\n            Separator inserted between the title and the text of the retrieved document when calling [`RagRetriever`].\n        doc_sep (`str`, *optional*, defaults to  `\" // \"`):\n            Separator inserted between the text of the retrieved document and the original input when calling\n            [`RagRetriever`].\n        n_docs (`int`, *optional*, defaults to 5):\n            Number of documents to retrieve.\n        max_combined_length (`int`, *optional*, defaults to 300):\n            Max length of contextualized input returned by [`~RagRetriever.__call__`].\n        retrieval_vector_size (`int`, *optional*, defaults to 768):\n            Dimensionality of the document embeddings indexed by [`RagRetriever`].\n        retrieval_batch_size (`int`, *optional*, defaults to 8):\n            Retrieval batch size, defined as the number of queries issues concurrently to the faiss index encapsulated\n            [`RagRetriever`].\n        dataset (`str`, *optional*, defaults to `\"wiki_dpr\"`):\n            A dataset identifier of the indexed dataset in HuggingFace Datasets (list all available datasets and ids\n            using `datasets.list_datasets()`).\n        dataset_split (`str`, *optional*, defaults to `\"train\"`)\n            Which split of the `dataset` to load.\n        index_name (`str`, *optional*, defaults to `\"compressed\"`)\n            The index name of the index associated with the `dataset`. One can choose between `\"legacy\"`, `\"exact\"` and\n            `\"compressed\"`.\n        index_path (`str`, *optional*)\n            The path to the serialized faiss index on disk.\n        passages_path (`str`, *optional*):\n            A path to text passages compatible with the faiss index. Required if using\n            [`~models.rag.retrieval_rag.LegacyIndex`]\n        use_dummy_dataset (`bool`, *optional*, defaults to `False`)\n            Whether to load a \"dummy\" variant of the dataset specified by `dataset`.\n        label_smoothing (`float`, *optional*, defaults to 0.0):\n            Only relevant if `return_loss` is set to `True`. Controls the `epsilon` parameter value for label smoothing\n            in the loss calculation. If set to 0, no label smoothing is performed.\n        do_marginalize (`bool`, *optional*, defaults to `False`):\n            If `True`, the logits are marginalized over all documents by making use of\n            `torch.nn.functional.log_softmax`.\n        reduce_loss (`bool`, *optional*, defaults to `False`):\n            Whether or not to reduce the NLL loss using the `torch.Tensor.sum` operation.\n        do_deduplication (`bool`, *optional*, defaults to `True`):\n            Whether or not to deduplicate the generations from different context documents for a given input. Has to be\n            set to `False` if used while training with distributed backend.\n        exclude_bos_score (`bool`, *optional*, defaults to `False`):\n            Whether or not to disregard the BOS token when computing the loss.\n        output_retrieved(`bool`, *optional*, defaults to `False`):\n            If set to `True`, `retrieved_doc_embeds`, `retrieved_doc_ids`, `context_input_ids` and\n            `context_attention_mask` are returned. See returned tensors for more detail.\n        use_cache (`bool`, *optional*, defaults to `True`):\n            Whether or not the model should return the last key/values attentions (not used by all models).\n        forced_eos_token_id (`int`, *optional*):\n            The id of the token to force as the last generated token when `max_length` is reached. Usually set to\n            `eos_token_id`.\n"






@add_start_docstrings(__lowerCAmelCase     )
class       UpperCamelCase__  (		__lowerCAmelCase     ):
	lowerCAmelCase__					:			Optional[Any]								=						"rag"
	lowerCAmelCase__					:			Optional[Any]								=						True







	def __init__(				self			:   List[Any] , lowerCamelCase			:   int=None , lowerCamelCase			:   List[str]=True , lowerCamelCase			:   Optional[Any]=None , lowerCamelCase			:   Union[str, Any]=None , lowerCamelCase			:   Dict=None , lowerCamelCase			:   Tuple=None , lowerCamelCase			:   List[Any]=None , lowerCamelCase			:   Tuple=" / " , lowerCamelCase			:   Union[str, Any]=" // " , lowerCamelCase			:   int=5 , lowerCamelCase			:   Optional[Any]=3_0_0 , lowerCamelCase			:   Tuple=7_6_8 , lowerCamelCase			:   Union[str, Any]=8 , lowerCamelCase			:   Any="wiki_dpr" , lowerCamelCase			:   Any="train" , lowerCamelCase			:   Union[str, Any]="compressed" , lowerCamelCase			:   Optional[Any]=None , lowerCamelCase			:   str=None , lowerCamelCase			:   Any=False , lowerCamelCase			:   Optional[int]=False , lowerCamelCase			:   str=0.0 , lowerCamelCase			:   Optional[int]=True , lowerCamelCase			:   Any=False , lowerCamelCase			:   Any=False , lowerCamelCase			:   int=False , lowerCamelCase			:   Any=True , lowerCamelCase			:   Optional[Any]=None , **lowerCamelCase			:   Optional[Any] , ):




				'''simple docstring'''
				super().__init__(
				    bos_token_id=lowerCamelCase , pad_token_id=lowerCamelCase , eos_token_id=lowerCamelCase , decoder_start_token_id=lowerCamelCase , forced_eos_token_id=lowerCamelCase , is_encoder_decoder=lowerCamelCase , prefix=lowerCamelCase , vocab_size=lowerCamelCase , **lowerCamelCase , )
				assert (
				    "question_encoder" in kwargs and "generator" in kwargs
				), "Config has to be initialized with question_encoder and generator config"
				a__     = kwargs.pop("question_encoder"		)
				a__     = question_encoder_config.pop("model_type"		)
				a__     = kwargs.pop("generator"		)
				a__     = decoder_config.pop("model_type"		)

				from ..auto.configuration_auto import AutoConfig

				a__     = AutoConfig.for_model(lowerCamelCase , **lowerCamelCase		)
				a__     = AutoConfig.for_model(lowerCamelCase , **lowerCamelCase		)

				a__     = reduce_loss
				a__     = label_smoothing
				a__     = exclude_bos_score
				a__     = do_marginalize

				a__     = title_sep
				a__     = doc_sep
				a__     = n_docs
				a__     = max_combined_length

				a__     = dataset
				a__     = dataset_split
				a__     = index_name

				a__     = retrieval_vector_size
				a__     = retrieval_batch_size
				a__     = passages_path
				a__     = index_path
				a__     = use_dummy_dataset

				a__     = output_retrieved

				a__     = do_deduplication

				a__     = use_cache

				if self.forced_eos_token_id is None:
							a__     = getattr(self.generator , "forced_eos_token_id" , lowerCamelCase		)







	@classmethod
	def 						__a   (				cls			:   Union[str, Any] , lowerCamelCase			:   PretrainedConfig , lowerCamelCase			:   PretrainedConfig , **lowerCamelCase			:   Optional[Any]		):




				'''simple docstring'''
				return cls(question_encoder=question_encoder_config.to_dict() , generator=generator_config.to_dict() , **lowerCamelCase		)







	def 						__a   (				self			:   List[Any]		):




				'''simple docstring'''
				a__     = copy.deepcopy(self.__dict__		)
				a__     = self.question_encoder.to_dict()
				a__     = self.generator.to_dict()
				a__     = self.__class__.model_type
				return output










'''simple docstring'''


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_   :   str       =					logging.get_logger(__name__)

lowerCAmelCase_   :   Optional[Any]       =					"▁"

lowerCAmelCase_   :   Union[str, Any]       =					{"vocab_file": "sentencepiece.bpe.model"}

lowerCAmelCase_   :   str       =					{
    "vocab_file": {
        "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model",
    }
}

lowerCAmelCase_   :   Optional[int]       =					{
    "facebook/xglm-564M": 2048,
}






class       UpperCamelCase__  (		__lowerCAmelCase     ):
	lowerCAmelCase__					:			int								=						VOCAB_FILES_NAMES
	lowerCAmelCase__					:			Optional[int]								=						PRETRAINED_VOCAB_FILES_MAP
	lowerCAmelCase__					:			Tuple								=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCAmelCase__					:			Optional[int]								=						["input_ids", "attention_mask"]







	def __init__(				self			:   Optional[Any] , lowerCamelCase			:   Union[str, Any] , lowerCamelCase			:   int="<s>" , lowerCamelCase			:   List[Any]="</s>" , lowerCamelCase			:   Any="</s>" , lowerCamelCase			:   Tuple="<s>" , lowerCamelCase			:   Optional[Any]="<unk>" , lowerCamelCase			:   Tuple="<pad>" , lowerCamelCase			:   Optional[Dict[str, Any]] = None , **lowerCamelCase			:   Union[str, Any] , ):




				'''simple docstring'''
				a__     = {} if sp_model_kwargs is None else sp_model_kwargs

				# Compatibility with the original tokenizer
				a__     = 7
				a__     = [F'''<madeupword{i}>''' for i in range(self.num_madeup_words		)]

				a__     = kwargs.get("additional_special_tokens" , []		)
				kwargs["additional_special_tokens"] += [
				    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
				]

				super().__init__(
				    bos_token=lowerCamelCase , eos_token=lowerCamelCase , unk_token=lowerCamelCase , sep_token=lowerCamelCase , cls_token=lowerCamelCase , pad_token=lowerCamelCase , sp_model_kwargs=self.sp_model_kwargs , **lowerCamelCase , )

				a__     = spm.SentencePieceProcessor(**self.sp_model_kwargs		)
				self.sp_model.Load(str(lowerCamelCase		)		)
				a__     = vocab_file

				# Original fairseq vocab and spm vocab must be "aligned":
				# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
				# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
				# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
				# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

				# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
				a__     = 1

				# Mimic fairseq token-to-id alignment for the first 4 token
				a__     = {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

				a__     = len(self.sp_model		)
				a__     = {F'''<madeupword{i}>''': sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words		)}
				self.fairseq_tokens_to_ids.update(lowerCamelCase		)

				a__     = {v: k for k, v in self.fairseq_tokens_to_ids.items()}







	def __getstate__(				self			:   Optional[Any]		):




				'''simple docstring'''
				a__     = self.__dict__.copy()
				a__     = None
				a__     = self.sp_model.serialized_model_proto()
				return state







	def __setstate__(				self			:   Dict , lowerCamelCase			:   List[Any]		):




				'''simple docstring'''
				a__     = d

				# for backward compatibility
				if not hasattr(self , "sp_model_kwargs"		):
							a__     = {}

				a__     = spm.SentencePieceProcessor(**self.sp_model_kwargs		)
				self.sp_model.LoadFromSerializedProto(self.sp_model_proto		)







	def 						__a   (				self			:   Union[str, Any] , lowerCamelCase			:   List[int] , lowerCamelCase			:   Optional[List[int]] = None		):




				'''simple docstring'''

				if token_ids_a is None:
							return [self.sep_token_id] + token_ids_a
				a__     = [self.sep_token_id]
				return sep + token_ids_a + sep + sep + token_ids_a







	def 						__a   (				self			:   int , lowerCamelCase			:   List[int] , lowerCamelCase			:   Optional[List[int]] = None , lowerCamelCase			:   bool = False		):




				'''simple docstring'''

				if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=lowerCamelCase , token_ids_a=lowerCamelCase , already_has_special_tokens=lowerCamelCase		)

				if token_ids_a is None:
							return [1] + ([0] * len(lowerCamelCase		))
				return [1] + ([0] * len(lowerCamelCase		)) + [1, 1] + ([0] * len(lowerCamelCase		))







	def 						__a   (				self			:   Dict , lowerCamelCase			:   List[int] , lowerCamelCase			:   Optional[List[int]] = None		):




				'''simple docstring'''
				a__     = [self.sep_token_id]

				if token_ids_a is None:
							return len(sep + token_ids_a		) * [0]
				return len(sep + token_ids_a + sep + sep + token_ids_a		) * [0]







	@property
	def 						__a   (				self			:   List[str]		):




				'''simple docstring'''
				return len(self.sp_model		) + self.fairseq_offset + self.num_madeup_words







	def 						__a   (				self			:   Dict		):




				'''simple docstring'''
				a__     = {self.convert_ids_to_tokens(lowerCamelCase		): i for i in range(self.vocab_size		)}
				vocab.update(self.added_tokens_encoder		)
				return vocab







	def 						__a   (				self			:   int , lowerCamelCase			:   str		):




				'''simple docstring'''
				return self.sp_model.encode(lowerCamelCase , out_type=lowerCamelCase		)







	def 						__a   (				self			:   List[str] , lowerCamelCase			:   Union[str, Any]		):




				'''simple docstring'''
				if token in self.fairseq_tokens_to_ids:
							return self.fairseq_tokens_to_ids[token]
				a__     = self.sp_model.PieceToId(lowerCamelCase		)

				# Need to return unknown token if the SP model returned 0
				return spm_id + self.fairseq_offset if spm_id else self.unk_token_id







	def 						__a   (				self			:   Any , lowerCamelCase			:   List[str]		):




				'''simple docstring'''
				if index in self.fairseq_ids_to_tokens:
							return self.fairseq_ids_to_tokens[index]
				return self.sp_model.IdToPiece(index - self.fairseq_offset		)







	def 						__a   (				self			:   int , lowerCamelCase			:   Optional[Any]		):




				'''simple docstring'''
				a__     = "".join(lowerCamelCase		).replace(lowerCamelCase , " "		).strip()
				return out_string







	def 						__a   (				self			:   int , lowerCamelCase			:   str , lowerCamelCase			:   Optional[str] = None		):




				'''simple docstring'''
				if not os.path.isdir(lowerCamelCase		):
							logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''		)
							return
				a__     = os.path.join(
				    lowerCamelCase , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)

				if os.path.abspath(self.vocab_file		) != os.path.abspath(lowerCamelCase		) and os.path.isfile(self.vocab_file		):
							copyfile(self.vocab_file , lowerCamelCase		)
				elif not os.path.isfile(self.vocab_file		):
							with open(lowerCamelCase , "wb"		) as fi:
										a__     = self.sp_model.serialized_model_proto()
										fi.write(lowerCamelCase		)

				return (out_vocab_file,)



