





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


snake_case 				=       logging.get_logger(__name__)

snake_case 				=       {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

snake_case 				=       {
    """vocab_file""": {
        """junnyu/roformer_chinese_small""": """https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_base""": """https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_char_small""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_chinese_char_base""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_discriminator""": (
            """https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_generator""": (
            """https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"""
        ),
    }
}

snake_case 				=       {
    """junnyu/roformer_chinese_small""": 1_5_3_6,
    """junnyu/roformer_chinese_base""": 1_5_3_6,
    """junnyu/roformer_chinese_char_small""": 5_1_2,
    """junnyu/roformer_chinese_char_base""": 5_1_2,
    """junnyu/roformer_small_discriminator""": 1_2_8,
    """junnyu/roformer_small_generator""": 1_2_8,
}


snake_case 				=       {
    """junnyu/roformer_chinese_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_base""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_base""": {"""do_lower_case""": True},
    """junnyu/roformer_small_discriminator""": {"""do_lower_case""": True},
    """junnyu/roformer_small_generator""": {"""do_lower_case""": True},
}





class 				A_				(      UpperCAmelCase	):







       """simple docstring"""





       SCREAMING_SNAKE_CASE_      : List[Any]      =  VOCAB_FILES_NAMES
       SCREAMING_SNAKE_CASE_      : Optional[int]      =  PRETRAINED_VOCAB_FILES_MAP
       SCREAMING_SNAKE_CASE_      : str      =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       SCREAMING_SNAKE_CASE_      : Optional[int]      =  PRETRAINED_INIT_CONFIGURATION
       SCREAMING_SNAKE_CASE_      : Tuple      =  RoFormerTokenizer







       def __init__(    self	:	Union[str, Any]						,__A	:	Optional[int]=None						,__A	:	str=None						,__A	:	Optional[Any]=True						,__A	:	Dict="[UNK]"						,__A	:	int="[SEP]"						,__A	:	Dict="[PAD]"						,__A	:	List[str]="[CLS]"						,__A	:	Optional[Any]="[MASK]"						,__A	:	Tuple=True						,__A	:	Optional[int]=None						,**__A	:	Optional[int]						,)						->					Any:
         super().__init__(
             __A						,tokenizer_file=__A						,do_lower_case=__A						,unk_token=__A						,sep_token=__A						,pad_token=__A						,cls_token=__A						,mask_token=__A						,tokenize_chinese_chars=__A						,strip_accents=__A						,**__A						,)

         _lowercase							= json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
         if (
             pre_tok_state.get('lowercase'						,__A  ) != do_lower_case
             or pre_tok_state.get('strip_accents'						,__A  ) != strip_accents
         ):
           _lowercase							= getattr(__A						,pre_tok_state.pop('type'  )  )
           _lowercase							= do_lower_case
           _lowercase							= strip_accents
           _lowercase							= pre_tok_class(**__A  )

         _lowercase							= do_lower_case







       def __getstate__(    self	:	Optional[int]  )						->					str:
         _lowercase							= self.__dict__.copy()
         _lowercase							= BertPreTokenizer()
         return state







       def __setstate__(    self	:	List[str]						,__A	:	str  )						->					Tuple:
         _lowercase							= d
         _lowercase							= self.__dict__['_tokenizer'].get_vocab()
         _lowercase							= PreTokenizer.custom(JiebaPreTokenizer(__A  )  )







       def    __UpperCAmelCase (    self	:	Optional[Any]						,__A	:	Optional[Any]						,__A	:	str=None  )						->					Dict:
         _lowercase							= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
           output += token_ids_a + [self.sep_token_id]

         return output







       def    __UpperCAmelCase (    self	:	Any						,__A	:	List[int]						,__A	:	Optional[List[int]] = None  )						->					List[int]:
         _lowercase							= [self.sep_token_id]
         _lowercase							= [self.cls_token_id]
         if token_ids_a is None:
           return len(cls + token_ids_a + sep  ) * [0]
         return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]







       def    __UpperCAmelCase (    self	:	Optional[Any]						,__A	:	str						,__A	:	Optional[str] = None  )						->					Tuple[str]:
         _lowercase							= self._tokenizer.model.save(__A						,name=__A  )
         return tuple(__A  )







       def    __UpperCAmelCase (    self	:	List[Any]						,__A	:	Dict						,__A	:	Dict=None						,__A	:	Optional[Any]=None						,__A	:	Any=False						,**__A	:	Any						,)						->					List[Any]:
         _lowercase							= BertPreTokenizer()
         return super().save_pretrained(__A						,__A						,__A						,__A						,**__A  )






import warnings

from ...utils import logging
from .image_processing_donut import DonutImageProcessor


snake_case 				=       logging.get_logger(__name__)





class 				A_				(      UpperCAmelCase	):







       """simple docstring"""





       def __init__(    self	:	Dict						,*__A	:	Optional[int]						,**__A	:	List[Any]  )						->					None:
         warnings.warn(
             'The class DonutFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please'
             ' use DonutImageProcessor instead.'						,__A						,)
         super().__init__(*__A						,**__A  )