





'''simple docstring'''




import json
import os
import shutil
import tempfile
from unittest import TestCase

from transformers import BartTokenizer, BartTokenizerFast, DPRQuestionEncoderTokenizer, DPRQuestionEncoderTokenizerFast
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_tokenizers, require_torch, slow
from transformers.utils import is_datasets_available, is_faiss_available, is_torch_available


if is_torch_available() and is_datasets_available() and is_faiss_available():
							from transformers.models.rag.configuration_rag import RagConfig
							from transformers.models.rag.tokenization_rag import RagTokenizer




@require_faiss
@require_torch
class      UpperCamelCase__					(					lowercase_			):




					"""simple docstring"""


					def lowerCamelCase_			(						self				:			Union[str, Any]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				List[str]		=			tempfile.mkdtemp()
							SCREAMING_SNAKE_CASE :				Optional[int]		=			8

							# DPR tok
							SCREAMING_SNAKE_CASE :				int		=			[
							    """[UNK]""",
							    """[CLS]""",
							    """[SEP]""",
							    """[PAD]""",
							    """[MASK]""",
							    """want""",
							    """##want""",
							    """##ed""",
							    """wa""",
							    """un""",
							    """runn""",
							    """##ing""",
							    """,""",
							    """low""",
							    """lowest""",
							]
							SCREAMING_SNAKE_CASE :				List[Any]		=			os.path.join(self.tmpdirname      ,						"""dpr_tokenizer"""						)
							os.makedirs(lowerCamelCase_      ,						exist_ok=lowerCamelCase_						)
							SCREAMING_SNAKE_CASE :				str		=			os.path.join(lowerCamelCase_      ,						DPR_VOCAB_FILES_NAMES["""vocab_file"""]						)
							with open(self.vocab_file      ,						"""w"""      ,						encoding="""utf-8"""						) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]						)						)

							# BART tok
							SCREAMING_SNAKE_CASE :				List[str]		=			[
							    """l""",
							    """o""",
							    """w""",
							    """e""",
							    """r""",
							    """s""",
							    """t""",
							    """i""",
							    """d""",
							    """n""",
							    """\u0120""",
							    """\u0120l""",
							    """\u0120n""",
							    """\u0120lo""",
							    """\u0120low""",
							    """er""",
							    """\u0120lowest""",
							    """\u0120newer""",
							    """\u0120wider""",
							    """<unk>""",
							]
							SCREAMING_SNAKE_CASE :				Tuple		=			dict(zip(lowerCamelCase_      ,						range(len(lowerCamelCase_						)						)						)						)
							SCREAMING_SNAKE_CASE :				Optional[int]		=			["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
							SCREAMING_SNAKE_CASE :				Optional[Any]		=			{"""unk_token""": """<unk>"""}

							SCREAMING_SNAKE_CASE :				List[str]		=			os.path.join(self.tmpdirname      ,						"""bart_tokenizer"""						)
							os.makedirs(lowerCamelCase_      ,						exist_ok=lowerCamelCase_						)
							SCREAMING_SNAKE_CASE :				Dict		=			os.path.join(lowerCamelCase_      ,						BART_VOCAB_FILES_NAMES["""vocab_file"""]						)
							SCREAMING_SNAKE_CASE :				Any		=			os.path.join(lowerCamelCase_      ,						BART_VOCAB_FILES_NAMES["""merges_file"""]						)
							with open(self.vocab_file      ,						"""w"""      ,						encoding="""utf-8"""						) as fp:
									fp.write(json.dumps(lowerCamelCase_						) + """\n"""						)
							with open(self.merges_file      ,						"""w"""      ,						encoding="""utf-8"""						) as fp:
									fp.write("""\n""".join(lowerCamelCase_						)						)


					def lowerCamelCase_			(						self				:			Dict						):

							'''simple docstring'''



							return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname      ,						"""dpr_tokenizer"""						)						)


					def lowerCamelCase_			(						self				:			Dict						):

							'''simple docstring'''



							return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname      ,						"""bart_tokenizer"""						)						)


					def lowerCamelCase_			(						self				:			Any						):

							'''simple docstring'''



							shutil.rmtree(self.tmpdirname						)


					@require_tokenizers
					def lowerCamelCase_			(						self				:			Dict						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Tuple		=			os.path.join(self.tmpdirname      ,						"""rag_tokenizer"""						)
							SCREAMING_SNAKE_CASE :				int		=			RagConfig(question_encoder=DPRConfig().to_dict()      ,						generator=BartConfig().to_dict()						)
							SCREAMING_SNAKE_CASE :				Optional[Any]		=			RagTokenizer(question_encoder=self.get_dpr_tokenizer()      ,						generator=self.get_bart_tokenizer()						)
							rag_config.save_pretrained(lowerCamelCase_						)
							rag_tokenizer.save_pretrained(lowerCamelCase_						)
							SCREAMING_SNAKE_CASE :				Optional[Any]		=			RagTokenizer.from_pretrained(lowerCamelCase_      ,						config=lowerCamelCase_						)
							self.assertIsInstance(new_rag_tokenizer.question_encoder      ,						lowerCamelCase_						)
							self.assertEqual(new_rag_tokenizer.question_encoder.get_vocab()      ,						rag_tokenizer.question_encoder.get_vocab()						)
							self.assertIsInstance(new_rag_tokenizer.generator      ,						lowerCamelCase_						)
							self.assertEqual(new_rag_tokenizer.generator.get_vocab()      ,						rag_tokenizer.generator.get_vocab()						)


					@slow
					def lowerCamelCase_			(						self				:			List[str]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				List[Any]		=			RagTokenizer.from_pretrained("""facebook/rag-token-nq"""						)
							SCREAMING_SNAKE_CASE :				List[str]		=			[
							    """who got the first nobel prize in physics""",
							    """when is the next deadpool movie being released""",
							    """which mode is used for short wave broadcast service""",
							    """who is the owner of reading football club""",
							    """when is the next scandal episode coming out""",
							    """when is the last time the philadelphia won the superbowl""",
							    """what is the most current adobe flash player version""",
							    """how many episodes are there in dragon ball z""",
							    """what is the first step in the evolution of the eye""",
							    """where is gall bladder situated in human body""",
							    """what is the main mineral in lithium batteries""",
							    """who is the president of usa right now""",
							    """where do the greasers live in the outsiders""",
							    """panda is a national animal of which country""",
							    """what is the name of manchester united stadium""",
							]
							SCREAMING_SNAKE_CASE :				Tuple		=			tokenizer(lowerCamelCase_						)
							self.assertIsNotNone(lowerCamelCase_						)



					@slow
					def lowerCamelCase_			(						self				:			Union[str, Any]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				int		=			RagTokenizer.from_pretrained("""facebook/rag-sequence-nq"""						)
							SCREAMING_SNAKE_CASE :				List[str]		=			[
							    """who got the first nobel prize in physics""",
							    """when is the next deadpool movie being released""",
							    """which mode is used for short wave broadcast service""",
							    """who is the owner of reading football club""",
							    """when is the next scandal episode coming out""",
							    """when is the last time the philadelphia won the superbowl""",
							    """what is the most current adobe flash player version""",
							    """how many episodes are there in dragon ball z""",
							    """what is the first step in the evolution of the eye""",
							    """where is gall bladder situated in human body""",
							    """what is the main mineral in lithium batteries""",
							    """who is the president of usa right now""",
							    """where do the greasers live in the outsiders""",
							    """panda is a national animal of which country""",
							    """what is the name of manchester united stadium""",
							]
							SCREAMING_SNAKE_CASE :				str		=			tokenizer(lowerCamelCase_						)
							self.assertIsNotNone(lowerCamelCase_						)










'''simple docstring'''




from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCAmelCase            =       logging.get_logger(__name__)

__UpperCAmelCase            =       {
    """naver-clova-ix/donut-base""": """https://huggingface.co/naver-clova-ix/donut-base/resolve/main/config.json""",
    # See all Donut models at https://huggingface.co/models?filter=donut-swin
}




class      UpperCamelCase__					(					lowercase_			):




					"""simple docstring"""
					SCREAMING_SNAKE_CASE__				=	'''donut-swin'''

					SCREAMING_SNAKE_CASE__				=	{
					    '''num_attention_heads''': '''num_heads''',
					    '''num_hidden_layers''': '''num_layers''',
					}


					def __init__(						self				:			str      ,						lowerCamelCase_				:			Union[str, Any]=2_24      ,						lowerCamelCase_				:			Union[str, Any]=4      ,						lowerCamelCase_				:			Optional[Any]=3      ,						lowerCamelCase_				:			Dict=96      ,						lowerCamelCase_				:			int=[2, 2, 6, 2]      ,						lowerCamelCase_				:			Optional[Any]=[3, 6, 12, 24]      ,						lowerCamelCase_				:			List[str]=7      ,						lowerCamelCase_				:			List[str]=4.0      ,						lowerCamelCase_				:			Optional[Any]=True      ,						lowerCamelCase_				:			List[str]=0.0      ,						lowerCamelCase_				:			Dict=0.0      ,						lowerCamelCase_				:			Dict=0.1      ,						lowerCamelCase_				:			Union[str, Any]="gelu"      ,						lowerCamelCase_				:			List[str]=False      ,						lowerCamelCase_				:			Optional[int]=0.02      ,						lowerCamelCase_				:			Any=1e-5      ,						**lowerCamelCase_				:			Union[str, Any]      ,						):

							'''simple docstring'''



							super().__init__(**lowerCamelCase_						)

							SCREAMING_SNAKE_CASE :				int		=			image_size
							SCREAMING_SNAKE_CASE :				Dict		=			patch_size
							SCREAMING_SNAKE_CASE :				str		=			num_channels
							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			embed_dim
							SCREAMING_SNAKE_CASE :				Optional[int]		=			depths
							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			len(lowerCamelCase_						)
							SCREAMING_SNAKE_CASE :				Optional[int]		=			num_heads
							SCREAMING_SNAKE_CASE :				Optional[int]		=			window_size
							SCREAMING_SNAKE_CASE :				Optional[Any]		=			mlp_ratio
							SCREAMING_SNAKE_CASE :				Dict		=			qkv_bias
							SCREAMING_SNAKE_CASE :				Optional[int]		=			hidden_dropout_prob
							SCREAMING_SNAKE_CASE :				Tuple		=			attention_probs_dropout_prob
							SCREAMING_SNAKE_CASE :				Dict		=			drop_path_rate
							SCREAMING_SNAKE_CASE :				List[str]		=			hidden_act
							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			use_absolute_embeddings
							SCREAMING_SNAKE_CASE :				Tuple		=			layer_norm_eps
							SCREAMING_SNAKE_CASE :				Optional[Any]		=			initializer_range
							# we set the hidden_size attribute in order to make Swin work with VisionEncoderDecoderModel
							# this indicates the channel dimension after the last stage of the model
							SCREAMING_SNAKE_CASE :				str		=			int(embed_dim * 2 ** (len(lowerCamelCase_						) - 1)						)



