
'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


_lowercase			:  int   							=	logging.get_logger(__name__)

_lowercase			:  Dict   							=	{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_lowercase			:  str   							=	{
    "vocab_file": {
        "junnyu/roformer_chinese_small": "https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_base": "https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_char_small": (
            "https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_chinese_char_base": (
            "https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_discriminator": (
            "https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_generator": (
            "https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"
        ),
    }
}

_lowercase			:  List[str]   							=	{
    "junnyu/roformer_chinese_small": 1_5_3_6,
    "junnyu/roformer_chinese_base": 1_5_3_6,
    "junnyu/roformer_chinese_char_small": 5_1_2,
    "junnyu/roformer_chinese_char_base": 5_1_2,
    "junnyu/roformer_small_discriminator": 1_2_8,
    "junnyu/roformer_small_generator": 1_2_8,
}


_lowercase			:  Dict   							=	{
    "junnyu/roformer_chinese_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_base": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_base": {"do_lower_case": True},
    "junnyu/roformer_small_discriminator": {"do_lower_case": True},
    "junnyu/roformer_small_generator": {"do_lower_case": True},
}






class 			lowerCAmelCase__       (  lowerCamelCase_					):
					lowerCAmelCase_					       =					VOCAB_FILES_NAMES
					lowerCAmelCase_					       =					PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_					       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_					       =					PRETRAINED_INIT_CONFIGURATION
					lowerCAmelCase_					       =					RoFormerTokenizer






					def __init__(       self    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE=True    ,							__SCREAMING_SNAKE_CASE="[UNK]"    ,							__SCREAMING_SNAKE_CASE="[SEP]"    ,							__SCREAMING_SNAKE_CASE="[PAD]"    ,							__SCREAMING_SNAKE_CASE="[CLS]"    ,							__SCREAMING_SNAKE_CASE="[MASK]"    ,							__SCREAMING_SNAKE_CASE=True    ,							__SCREAMING_SNAKE_CASE=None    ,							**__SCREAMING_SNAKE_CASE    ,							):


											"""simple docstring"""





											super().__init__(
											    __SCREAMING_SNAKE_CASE    ,							tokenizer_file=__SCREAMING_SNAKE_CASE    ,							do_lower_case=__SCREAMING_SNAKE_CASE    ,							unk_token=__SCREAMING_SNAKE_CASE    ,							sep_token=__SCREAMING_SNAKE_CASE    ,							pad_token=__SCREAMING_SNAKE_CASE    ,							cls_token=__SCREAMING_SNAKE_CASE    ,							mask_token=__SCREAMING_SNAKE_CASE    ,							tokenize_chinese_chars=__SCREAMING_SNAKE_CASE    ,							strip_accents=__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE    ,							)

											lowercase_     :					Any													=	json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
											if (
											    pre_tok_state.get('''lowercase'''    ,							__SCREAMING_SNAKE_CASE		) != do_lower_case
											    or pre_tok_state.get('''strip_accents'''    ,							__SCREAMING_SNAKE_CASE		) != strip_accents
											):
																	lowercase_     :					Dict													=	getattr(__SCREAMING_SNAKE_CASE    ,							pre_tok_state.pop('''type'''		)		)
																	lowercase_     :					int													=	do_lower_case
																	lowercase_     :					Tuple													=	strip_accents
																	lowercase_     :					Optional[int]													=	pre_tok_class(**__SCREAMING_SNAKE_CASE		)

											lowercase_     :					List[Any]													=	do_lower_case






					def __getstate__(       self		):


											"""simple docstring"""





											lowercase_     :					Optional[Any]													=	self.__dict__.copy()
											lowercase_     :					str													=	BertPreTokenizer()
											return state






					def __setstate__(       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Optional[Any]													=	d
											lowercase_     :					int													=	self.__dict__['''_tokenizer'''].get_vocab()
											lowercase_     :					Optional[int]													=	PreTokenizer.custom(JiebaPreTokenizer(__SCREAMING_SNAKE_CASE		)		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE=None		):


											"""simple docstring"""





											lowercase_     :					List[Any]													=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

											if token_ids_a:
																	output += token_ids_a + [self.sep_token_id]

											return output






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											lowercase_     :					Tuple													=	[self.sep_token_id]
											lowercase_     :					int													=	[self.cls_token_id]
											if token_ids_a is None:
																	return len(cls + token_ids_a + sep		) * [0]
											return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											lowercase_     :					Optional[int]													=	self._tokenizer.model.save(__SCREAMING_SNAKE_CASE    ,							name=__SCREAMING_SNAKE_CASE		)
											return tuple(__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE=False    ,							**__SCREAMING_SNAKE_CASE    ,							):


											"""simple docstring"""





											lowercase_     :					Tuple													=	BertPreTokenizer()
											return super().save_pretrained(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		)





'''simple docstring'''







import warnings

from ...utils import logging
from .image_processing_donut import DonutImageProcessor


_lowercase			:  Any   							=	logging.get_logger(__name__)






class 			lowerCAmelCase__       (  lowerCamelCase_					):






					def __init__(       self    ,							*__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											warnings.warn(
											    '''The class DonutFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please'''
											    ''' use DonutImageProcessor instead.'''    ,							__SCREAMING_SNAKE_CASE    ,							)
											super().__init__(*__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		)



