


import json
import os
import shutil
import tempfile
from unittest import TestCase

from transformers import BartTokenizer, BartTokenizerFast, DPRQuestionEncoderTokenizer, DPRQuestionEncoderTokenizerFast
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_tokenizers, require_torch, slow
from transformers.utils import is_datasets_available, is_faiss_available, is_torch_available


if is_torch_available() and is_datasets_available() and is_faiss_available():
	from transformers.models.rag.configuration_rag import RagConfig
	from transformers.models.rag.tokenization_rag import RagTokenizer






@require_faiss
@require_torch
class      UpperCamelCase(	_a ):


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Any    )      ->						List[str]:




											'''simple docstring'''

											__snake_case				=		tempfile.mkdtemp()
											__snake_case				=		8

											# DPR tok
											__snake_case				=		[
											    "[UNK]",
											    "[CLS]",
											    "[SEP]",
											    "[PAD]",
											    "[MASK]",
											    "want",
											    "##want",
											    "##ed",
											    "wa",
											    "un",
											    "runn",
											    "##ing",
											    ",",
											    "low",
											    "lowest",
											]
											__snake_case				=		os.path.join(self.tmpdirname  ,     "dpr_tokenizer"    )
											os.makedirs(SCREAMING_SNAKE_CASE  ,     exist_ok=SCREAMING_SNAKE_CASE    )
											__snake_case				=		os.path.join(SCREAMING_SNAKE_CASE  ,     DPR_VOCAB_FILES_NAMES["vocab_file"]    )
											with open(self.vocab_file  ,     "w"  ,     encoding="utf-8"    ) as vocab_writer:
																	vocab_writer.write("".join([x + "\n" for x in vocab_tokens]    )    )

											# BART tok
											__snake_case				=		[
											    "l",
											    "o",
											    "w",
											    "e",
											    "r",
											    "s",
											    "t",
											    "i",
											    "d",
											    "n",
											    "\u0120",
											    "\u0120l",
											    "\u0120n",
											    "\u0120lo",
											    "\u0120low",
											    "er",
											    "\u0120lowest",
											    "\u0120newer",
											    "\u0120wider",
											    "<unk>",
											]
											__snake_case				=		dict(zip(SCREAMING_SNAKE_CASE  ,     range(len(SCREAMING_SNAKE_CASE    )    )    )    )
											__snake_case				=		["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
											__snake_case				=		{"unk_token": "<unk>"}

											__snake_case				=		os.path.join(self.tmpdirname  ,     "bart_tokenizer"    )
											os.makedirs(SCREAMING_SNAKE_CASE  ,     exist_ok=SCREAMING_SNAKE_CASE    )
											__snake_case				=		os.path.join(SCREAMING_SNAKE_CASE  ,     BART_VOCAB_FILES_NAMES["vocab_file"]    )
											__snake_case				=		os.path.join(SCREAMING_SNAKE_CASE  ,     BART_VOCAB_FILES_NAMES["merges_file"]    )
											with open(self.vocab_file  ,     "w"  ,     encoding="utf-8"    ) as fp:
																	fp.write(json.dumps(SCREAMING_SNAKE_CASE    ) + "\n"    )
											with open(self.merges_file  ,     "w"  ,     encoding="utf-8"    ) as fp:
																	fp.write("\n".join(SCREAMING_SNAKE_CASE    )    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Dict    )      ->						DPRQuestionEncoderTokenizer:




											'''simple docstring'''

											return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname  ,     "dpr_tokenizer"    )    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Dict    )      ->						BartTokenizer:




											'''simple docstring'''

											return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname  ,     "bart_tokenizer"    )    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		int    )      ->						Optional[int]:




											'''simple docstring'''

											shutil.rmtree(self.tmpdirname    )


					@require_tokenizers
					def 				SCREAMING_SNAKE_CASE_	(    self						:		int    )      ->						str:




											'''simple docstring'''

											__snake_case				=		os.path.join(self.tmpdirname  ,     "rag_tokenizer"    )
											__snake_case				=		RagConfig(question_encoder=DPRConfig().to_dict()  ,     generator=BartConfig().to_dict()    )
											__snake_case				=		RagTokenizer(question_encoder=self.get_dpr_tokenizer()  ,     generator=self.get_bart_tokenizer()    )
											rag_config.save_pretrained(SCREAMING_SNAKE_CASE    )
											rag_tokenizer.save_pretrained(SCREAMING_SNAKE_CASE    )
											__snake_case				=		RagTokenizer.from_pretrained(SCREAMING_SNAKE_CASE  ,     config=SCREAMING_SNAKE_CASE    )
											self.assertIsInstance(new_rag_tokenizer.question_encoder  ,     SCREAMING_SNAKE_CASE    )
											self.assertEqual(new_rag_tokenizer.question_encoder.get_vocab()  ,     rag_tokenizer.question_encoder.get_vocab()    )
											self.assertIsInstance(new_rag_tokenizer.generator  ,     SCREAMING_SNAKE_CASE    )
											self.assertEqual(new_rag_tokenizer.generator.get_vocab()  ,     rag_tokenizer.generator.get_vocab()    )


					@slow
					def 				SCREAMING_SNAKE_CASE_	(    self						:		Tuple    )      ->						int:




											'''simple docstring'''

											__snake_case				=		RagTokenizer.from_pretrained("facebook/rag-token-nq"    )
											__snake_case				=		[
											    "who got the first nobel prize in physics",
											    "when is the next deadpool movie being released",
											    "which mode is used for short wave broadcast service",
											    "who is the owner of reading football club",
											    "when is the next scandal episode coming out",
											    "when is the last time the philadelphia won the superbowl",
											    "what is the most current adobe flash player version",
											    "how many episodes are there in dragon ball z",
											    "what is the first step in the evolution of the eye",
											    "where is gall bladder situated in human body",
											    "what is the main mineral in lithium batteries",
											    "who is the president of usa right now",
											    "where do the greasers live in the outsiders",
											    "panda is a national animal of which country",
											    "what is the name of manchester united stadium",
											]
											__snake_case				=		tokenizer(SCREAMING_SNAKE_CASE    )
											self.assertIsNotNone(SCREAMING_SNAKE_CASE    )






					@slow
					def 				SCREAMING_SNAKE_CASE_	(    self						:		str    )      ->						List[Any]:




											'''simple docstring'''

											__snake_case				=		RagTokenizer.from_pretrained("facebook/rag-sequence-nq"    )
											__snake_case				=		[
											    "who got the first nobel prize in physics",
											    "when is the next deadpool movie being released",
											    "which mode is used for short wave broadcast service",
											    "who is the owner of reading football club",
											    "when is the next scandal episode coming out",
											    "when is the last time the philadelphia won the superbowl",
											    "what is the most current adobe flash player version",
											    "how many episodes are there in dragon ball z",
											    "what is the first step in the evolution of the eye",
											    "where is gall bladder situated in human body",
											    "what is the main mineral in lithium batteries",
											    "who is the president of usa right now",
											    "where do the greasers live in the outsiders",
											    "panda is a national animal of which country",
											    "what is the name of manchester united stadium",
											]
											__snake_case				=		tokenizer(SCREAMING_SNAKE_CASE    )
											self.assertIsNotNone(SCREAMING_SNAKE_CASE    )









from datasets.utils.patching import _PatchedModuleObj, patch_submodule

from . import _test_patching





def 						_lowerCAmelCase							(     )    ->						Tuple:


						'''simple docstring'''






						import os as original_os
						from os import path as original_path
						from os import rename as original_rename
						from os.path import dirname as original_dirname
						from os.path import join as original_join

						assert _test_patching.os is original_os
						assert _test_patching.path is original_path
						assert _test_patching.join is original_join

						assert _test_patching.renamed_os is original_os
						assert _test_patching.renamed_path is original_path
						assert _test_patching.renamed_join is original_join

						__snake_case				=		"__test_patch_submodule_mock__"
						with patch_submodule(_test_patching   ,						"os.path.join"   ,						_lowerCAmelCase			):
												# Every way to access os.path.join must be patched, and the rest must stay untouched

												# check os.path.join
												assert isinstance(_test_patching.os   ,						_PatchedModuleObj			)
												assert isinstance(_test_patching.os.path   ,						_PatchedModuleObj			)
												assert _test_patching.os.path.join is mock

												# check path.join
												assert isinstance(_test_patching.path   ,						_PatchedModuleObj			)
												assert _test_patching.path.join is mock

												# check join
												assert _test_patching.join is mock

												# check that the other attributes are untouched
												assert _test_patching.os.rename is original_rename
												assert _test_patching.path.dirname is original_dirname
												assert _test_patching.os.path.dirname is original_dirname

												# Even renamed modules or objects must be patched

												# check renamed_os.path.join
												assert isinstance(_test_patching.renamed_os   ,						_PatchedModuleObj			)
												assert isinstance(_test_patching.renamed_os.path   ,						_PatchedModuleObj			)
												assert _test_patching.renamed_os.path.join is mock

												# check renamed_path.join
												assert isinstance(_test_patching.renamed_path   ,						_PatchedModuleObj			)
												assert _test_patching.renamed_path.join is mock

												# check renamed_join
												assert _test_patching.renamed_join is mock

												# check that the other attributes are untouched
												assert _test_patching.renamed_os.rename is original_rename
												assert _test_patching.renamed_path.dirname is original_dirname
												assert _test_patching.renamed_os.path.dirname is original_dirname

						# check that everthing is back to normal when the patch is over

						assert _test_patching.os is original_os
						assert _test_patching.path is original_path
						assert _test_patching.join is original_join

						assert _test_patching.renamed_os is original_os
						assert _test_patching.renamed_path is original_path
						assert _test_patching.renamed_join is original_join





def 						_lowerCAmelCase							(     )    ->						int:


						'''simple docstring'''






						assert _test_patching.open is open

						__snake_case				=		"__test_patch_submodule_builtin_mock__"
						# _test_patching has "open" in its globals
						assert _test_patching.open is open
						with patch_submodule(_test_patching   ,						"open"   ,						_lowerCAmelCase			):
												assert _test_patching.open is mock

						# check that everthing is back to normal when the patch is over

						assert _test_patching.open is open





def 						_lowerCAmelCase							(     )    ->						Optional[Any]:


						'''simple docstring'''






						__snake_case				=		"__test_patch_submodule_missing_mock__"
						with patch_submodule(_test_patching   ,						"pandas.read_csv"   ,						_lowerCAmelCase			):
												pass





def 						_lowerCAmelCase							(     )    ->						Optional[Any]:


						'''simple docstring'''






						__snake_case				=		"__test_patch_submodule_missing_builtin_mock__"
						# _test_patching doesn't have "len" in its globals
						assert getattr(_test_patching   ,						"len"   ,						_lowerCAmelCase			) is None
						with patch_submodule(_test_patching   ,						"len"   ,						_lowerCAmelCase			):
												assert _test_patching.len is mock
						assert _test_patching.len is len





def 						_lowerCAmelCase							(     )    ->						Optional[Any]:


						'''simple docstring'''






						__snake_case				=		"__test_patch_submodule_start_and_stop_mock__"
						__snake_case				=		patch_submodule(_test_patching   ,						"open"   ,						_lowerCAmelCase			)
						assert _test_patching.open is open
						patch.start()
						assert _test_patching.open is mock
						patch.stop()
						assert _test_patching.open is open





def 						_lowerCAmelCase							(     )    ->						Any:


						'''simple docstring'''






						from os import rename as original_rename
						from os.path import dirname as original_dirname
						from os.path import join as original_join

						__snake_case				=		"__test_patch_submodule_successive_join__"
						__snake_case				=		"__test_patch_submodule_successive_dirname__"
						__snake_case				=		"__test_patch_submodule_successive_rename__"
						assert _test_patching.os.path.join is original_join
						assert _test_patching.os.path.dirname is original_dirname
						assert _test_patching.os.rename is original_rename

						with patch_submodule(_test_patching   ,						"os.path.join"   ,						_lowerCAmelCase			):
												with patch_submodule(_test_patching   ,						"os.rename"   ,						_lowerCAmelCase			):
																		with patch_submodule(_test_patching   ,						"os.path.dirname"   ,						_lowerCAmelCase			):
																								assert _test_patching.os.path.join is mock_join
																								assert _test_patching.os.path.dirname is mock_dirname
																								assert _test_patching.os.rename is mock_rename

    # try another order
						with patch_submodule(_test_patching   ,						"os.rename"   ,						_lowerCAmelCase			):
												with patch_submodule(_test_patching   ,						"os.path.join"   ,						_lowerCAmelCase			):
																		with patch_submodule(_test_patching   ,						"os.path.dirname"   ,						_lowerCAmelCase			):
																								assert _test_patching.os.path.join is mock_join
																								assert _test_patching.os.path.dirname is mock_dirname
																								assert _test_patching.os.rename is mock_rename

						assert _test_patching.os.path.join is original_join
						assert _test_patching.os.path.dirname is original_dirname
						assert _test_patching.os.rename is original_rename





def 						_lowerCAmelCase							(     )    ->						str:


						'''simple docstring'''






						__snake_case				=		"__test_patch_submodule_doesnt_exist_mock__"
						with patch_submodule(_test_patching   ,						"__module_that_doesn_exist__.__attribute_that_doesn_exist__"   ,						_lowerCAmelCase			):
												pass
						with patch_submodule(_test_patching   ,						"os.__attribute_that_doesn_exist__"   ,						_lowerCAmelCase			):
												pass





