



import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase__       =	get_tests_dir("fixtures/test_sentencepiece.model")







@require_sentencepiece
class       __lowerCAmelCase   (      lowercase__    ,						unittest.TestCase						):
			UpperCamelCase        =					XLMProphetNetTokenizer
			UpperCamelCase        =					False
			UpperCamelCase        =					True



			def    _lowerCamelCase	(						self       :  Dict)       ->   int:



						"""simple docstring"""






						super().setUp()

						# We have a SentencePiece fixture for testing
						_UpperCAmelCase			=       XLMProphetNetTokenizer(lowercase_			,		keep_accents=lowercase_)
						tokenizer.save_pretrained(self.tmpdirname)



			def    _lowerCamelCase	(						self       :  Union[str, Any])       ->   Optional[Any]:



						"""simple docstring"""






						_UpperCAmelCase			=       "[PAD]"
						_UpperCAmelCase			=       0

						self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase_)			,		lowercase_)
						self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase_)			,		lowercase_)



			def    _lowerCamelCase	(						self       :  Optional[Any])       ->   List[str]:



						"""simple docstring"""






						_UpperCAmelCase			=       list(self.get_tokenizer().get_vocab().keys())

						self.assertEqual(vocab_keys[0]			,		'[PAD]')
						self.assertEqual(vocab_keys[1]			,		'[CLS]')
						self.assertEqual(vocab_keys[-1]			,		'j')
						self.assertEqual(len(lowercase_)			,		10_12)



			def    _lowerCamelCase	(						self       :  List[str])       ->   int:



						"""simple docstring"""






						self.assertEqual(self.get_tokenizer().vocab_size			,		10_12)



			def    _lowerCamelCase	(						self       :  Tuple)       ->   Any:



						"""simple docstring"""






						_UpperCAmelCase			=       XLMProphetNetTokenizer(lowercase_			,		keep_accents=lowercase_)

						_UpperCAmelCase			=       tokenizer.tokenize('This is a test')
						self.assertListEqual(lowercase_			,		['▁This', '▁is', '▁a', '▁t', 'est'])

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(lowercase_)			,		[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]			,		)

						_UpperCAmelCase			=       tokenizer.tokenize('I was born in 92000, and this is falsé.')
						self.assertListEqual(
						    lowercase_			,		[
						        SPIECE_UNDERLINE + 'I',
						        SPIECE_UNDERLINE + 'was',
						        SPIECE_UNDERLINE + 'b',
						        'or',
						        'n',
						        SPIECE_UNDERLINE + 'in',
						        SPIECE_UNDERLINE + '',
						        '9',
						        '2',
						        '0',
						        '0',
						        '0',
						        ',',
						        SPIECE_UNDERLINE + 'and',
						        SPIECE_UNDERLINE + 'this',
						        SPIECE_UNDERLINE + 'is',
						        SPIECE_UNDERLINE + 'f',
						        'al',
						        's',
						        'é',
						        '.',
						    ]			,		)
						_UpperCAmelCase			=       tokenizer.convert_tokens_to_ids(lowercase_)
						self.assertListEqual(
						    lowercase_			,		[
						        value + tokenizer.fairseq_offset
						        for value in [8, 21, 84, 55, 24, 19, 7, -9, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, -9, 4]
						    ]			,		)

						_UpperCAmelCase			=       tokenizer.convert_ids_to_tokens(lowercase_)
						self.assertListEqual(
						    lowercase_			,		[
						        SPIECE_UNDERLINE + 'I',
						        SPIECE_UNDERLINE + 'was',
						        SPIECE_UNDERLINE + 'b',
						        'or',
						        'n',
						        SPIECE_UNDERLINE + 'in',
						        SPIECE_UNDERLINE + '',
						        '[UNK]',
						        '2',
						        '0',
						        '0',
						        '0',
						        ',',
						        SPIECE_UNDERLINE + 'and',
						        SPIECE_UNDERLINE + 'this',
						        SPIECE_UNDERLINE + 'is',
						        SPIECE_UNDERLINE + 'f',
						        'al',
						        's',
						        '[UNK]',
						        '.',
						    ]			,		)



			@cached_property
			def    _lowerCamelCase	(						self       :  Any)       ->   List[str]:



						"""simple docstring"""






						return XLMProphetNetTokenizer.from_pretrained('microsoft/xprophetnet-large-wiki100-cased')



			@slow
			def    _lowerCamelCase	(						self       :  Union[str, Any])       ->   Optional[Any]:



						"""simple docstring"""






						_UpperCAmelCase			=       "Hello World!"
						_UpperCAmelCase			=       [3_53_89, 66_72, 49, 2]
						self.assertListEqual(lowercase_			,		self.big_tokenizer.encode(lowercase_))



			@slow
			def    _lowerCamelCase	(						self       :  Optional[Any])       ->   Dict:



						"""simple docstring"""






						_UpperCAmelCase			=       {"input_ids": [[1_10_73, 8_27_83, 18, 26, 8_27_83, 5_49, 5_15_40, 2_48, 1_72_09, 13_01, 2_17, 20, 21_51_86, 13_25, 1_47, 1_72_09, 13_01, 2_17, 20, 5_63_70, 53, 12_20_20, 20, 1_64_77, 27, 8_73_55, 45_48, 20, 47_28, 7_83_92, 17, 15_99_69, 18, 26, 2_44_91, 6_29, 15, 5_38, 2_27_04, 54_39, 15, 27_88, 2_44_91, 98_85, 15, 4_35_34, 6_05, 15, 8_14, 1_84_03, 3_32_00, 29, 15, 4_35_34, 2_44_58, 1_24_10, 1_11, 2_49_66, 8_36_69, 96_37, 14_40_68, 26, 8_50, 2_23_46, 27, 1_47, 2_49_66, 8_36_69, 8_34_90, 26, 3_91_13, 7_35, 27, 6_89, 6_56, 28_00, 13_39, 46_00, 53, 12_20_20, 11_57_85, 34, 8_16, 13_39, 4_68_87, 18, 1_47, 5_39_05, 19_51, 4_22_38, 4_11_70, 1_77_32, 8_34, 4_36, 15, 2_75_23, 9_87_33, 2_17, 1_47, 55_42, 49_81, 9_30, 1_73_47, 16, 2], [2_00_91, 6_29, 94, 8_27_86, 58, 4_90, 20, 15_28, 84, 5_39_05, 3_44, 8_05_92, 11_01_28, 1_88_22, 52_67, 13_06, 62, 15_25_37, 3_08, 79_97, 4_01, 12_44_27, 5_49, 3_54_42, 2_25, 1_09, 1_50_55, 2_57_48, 1_47, 71_19, 4_37_12, 34, 7_67, 13_53_66, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_92, 6_37_84, 11_94_66, 17, 14_78_08, 8_82_14, 18, 6_56, 81, 32, 32_96, 1_02_80, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
						# fmt: on

						self.tokenizer_integration_test_util(
						    expected_encoding=lowercase_			,		model_name='microsoft/xprophetnet-large-wiki100-cased'			,		revision='1acad1643ddd54a44df6a1b797ada8373685d90e'			,		)





import copy

from ...configuration_utils import PretrainedConfig
from ...utils import add_start_docstrings


UpperCAmelCase__       =	r"\n    [`RagConfig`] stores the configuration of a *RagModel*. Configuration objects inherit from [`PretrainedConfig`] and\n    can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information.\n\n    Args:\n        title_sep (`str`, *optional*, defaults to  `\" / \"`):\n            Separator inserted between the title and the text of the retrieved document when calling [`RagRetriever`].\n        doc_sep (`str`, *optional*, defaults to  `\" // \"`):\n            Separator inserted between the text of the retrieved document and the original input when calling\n            [`RagRetriever`].\n        n_docs (`int`, *optional*, defaults to 5):\n            Number of documents to retrieve.\n        max_combined_length (`int`, *optional*, defaults to 300):\n            Max length of contextualized input returned by [`~RagRetriever.__call__`].\n        retrieval_vector_size (`int`, *optional*, defaults to 768):\n            Dimensionality of the document embeddings indexed by [`RagRetriever`].\n        retrieval_batch_size (`int`, *optional*, defaults to 8):\n            Retrieval batch size, defined as the number of queries issues concurrently to the faiss index encapsulated\n            [`RagRetriever`].\n        dataset (`str`, *optional*, defaults to `\"wiki_dpr\"`):\n            A dataset identifier of the indexed dataset in HuggingFace Datasets (list all available datasets and ids\n            using `datasets.list_datasets()`).\n        dataset_split (`str`, *optional*, defaults to `\"train\"`)\n            Which split of the `dataset` to load.\n        index_name (`str`, *optional*, defaults to `\"compressed\"`)\n            The index name of the index associated with the `dataset`. One can choose between `\"legacy\"`, `\"exact\"` and\n            `\"compressed\"`.\n        index_path (`str`, *optional*)\n            The path to the serialized faiss index on disk.\n        passages_path (`str`, *optional*):\n            A path to text passages compatible with the faiss index. Required if using\n            [`~models.rag.retrieval_rag.LegacyIndex`]\n        use_dummy_dataset (`bool`, *optional*, defaults to `False`)\n            Whether to load a \"dummy\" variant of the dataset specified by `dataset`.\n        label_smoothing (`float`, *optional*, defaults to 0.0):\n            Only relevant if `return_loss` is set to `True`. Controls the `epsilon` parameter value for label smoothing\n            in the loss calculation. If set to 0, no label smoothing is performed.\n        do_marginalize (`bool`, *optional*, defaults to `False`):\n            If `True`, the logits are marginalized over all documents by making use of\n            `torch.nn.functional.log_softmax`.\n        reduce_loss (`bool`, *optional*, defaults to `False`):\n            Whether or not to reduce the NLL loss using the `torch.Tensor.sum` operation.\n        do_deduplication (`bool`, *optional*, defaults to `True`):\n            Whether or not to deduplicate the generations from different context documents for a given input. Has to be\n            set to `False` if used while training with distributed backend.\n        exclude_bos_score (`bool`, *optional*, defaults to `False`):\n            Whether or not to disregard the BOS token when computing the loss.\n        output_retrieved(`bool`, *optional*, defaults to `False`):\n            If set to `True`, `retrieved_doc_embeds`, `retrieved_doc_ids`, `context_input_ids` and\n            `context_attention_mask` are returned. See returned tensors for more detail.\n        use_cache (`bool`, *optional*, defaults to `True`):\n            Whether or not the model should return the last key/values attentions (not used by all models).\n        forced_eos_token_id (`int`, *optional*):\n            The id of the token to force as the last generated token when `max_length` is reached. Usually set to\n            `eos_token_id`.\n"







@add_start_docstrings(A						)
class       __lowerCAmelCase   (      A						):
			UpperCamelCase        =					'''rag'''
			UpperCamelCase        =					True



			def __init__(						self       :  str			,		A       :  int=None			,		A       :  Dict=True			,		A       :  int=None			,		A       :  Any=None			,		A       :  Any=None			,		A       :  Any=None			,		A       :  int=None			,		A       :  str=" / "			,		A       :  int=" // "			,		A       :  Any=5			,		A       :  Optional[Any]=3_00			,		A       :  Optional[Any]=7_68			,		A       :  Optional[int]=8			,		A       :  Union[str, Any]="wiki_dpr"			,		A       :  int="train"			,		A       :  Tuple="compressed"			,		A       :  str=None			,		A       :  Any=None			,		A       :  Tuple=False			,		A       :  List[str]=False			,		A       :  Tuple=0.0			,		A       :  Any=True			,		A       :  int=False			,		A       :  Dict=False			,		A       :  List[str]=False			,		A       :  Optional[Any]=True			,		A       :  Tuple=None			,		**A       :  Optional[Any]			,		)       ->   Optional[Any]:



						"""simple docstring"""






						super().__init__(
						    bos_token_id=A			,		pad_token_id=A			,		eos_token_id=A			,		decoder_start_token_id=A			,		forced_eos_token_id=A			,		is_encoder_decoder=A			,		prefix=A			,		vocab_size=A			,		**A			,		)
						assert (
						    "question_encoder" in kwargs and "generator" in kwargs
						), "Config has to be initialized with question_encoder and generator config"
						_UpperCAmelCase			=       kwargs.pop('question_encoder')
						_UpperCAmelCase			=       question_encoder_config.pop('model_type')
						_UpperCAmelCase			=       kwargs.pop('generator')
						_UpperCAmelCase			=       decoder_config.pop('model_type')

						from ..auto.configuration_auto import AutoConfig

						_UpperCAmelCase			=       AutoConfig.for_model(A			,		**A)
						_UpperCAmelCase			=       AutoConfig.for_model(A			,		**A)

						_UpperCAmelCase			=       reduce_loss
						_UpperCAmelCase			=       label_smoothing
						_UpperCAmelCase			=       exclude_bos_score
						_UpperCAmelCase			=       do_marginalize

						_UpperCAmelCase			=       title_sep
						_UpperCAmelCase			=       doc_sep
						_UpperCAmelCase			=       n_docs
						_UpperCAmelCase			=       max_combined_length

						_UpperCAmelCase			=       dataset
						_UpperCAmelCase			=       dataset_split
						_UpperCAmelCase			=       index_name

						_UpperCAmelCase			=       retrieval_vector_size
						_UpperCAmelCase			=       retrieval_batch_size
						_UpperCAmelCase			=       passages_path
						_UpperCAmelCase			=       index_path
						_UpperCAmelCase			=       use_dummy_dataset

						_UpperCAmelCase			=       output_retrieved

						_UpperCAmelCase			=       do_deduplication

						_UpperCAmelCase			=       use_cache

						if self.forced_eos_token_id is None:
									_UpperCAmelCase			=       getattr(self.generator			,		'forced_eos_token_id'			,		A)



			@classmethod
			def    _lowerCamelCase	(						cls       :  Union[str, Any]			,		A       :  PretrainedConfig			,		A       :  PretrainedConfig			,		**A       :  Optional[Any])       ->   PretrainedConfig:



						"""simple docstring"""






						return cls(question_encoder=question_encoder_config.to_dict()			,		generator=generator_config.to_dict()			,		**A)



			def    _lowerCamelCase	(						self       :  Dict)       ->   Optional[Any]:



						"""simple docstring"""






						_UpperCAmelCase			=       copy.deepcopy(self.__dict__)
						_UpperCAmelCase			=       self.question_encoder.to_dict()
						_UpperCAmelCase			=       self.generator.to_dict()
						_UpperCAmelCase			=       self.__class__.model_type
						return output
