






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


_SCREAMING_SNAKE_CASE   =  logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE   =  {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_SCREAMING_SNAKE_CASE   =  {
    "vocab_file": {
        "junnyu/roformer_chinese_small": "https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_base": "https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_char_small": (
            "https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_chinese_char_base": (
            "https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_discriminator": (
            "https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_generator": (
            "https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"
        ),
    }
}

_SCREAMING_SNAKE_CASE   =  {
    "junnyu/roformer_chinese_small": 15_36,
    "junnyu/roformer_chinese_base": 15_36,
    "junnyu/roformer_chinese_char_small": 5_12,
    "junnyu/roformer_chinese_char_base": 5_12,
    "junnyu/roformer_small_discriminator": 1_28,
    "junnyu/roformer_small_generator": 1_28,
}


_SCREAMING_SNAKE_CASE   =  {
    "junnyu/roformer_chinese_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_base": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_base": {"do_lower_case": True},
    "junnyu/roformer_small_discriminator": {"do_lower_case": True},
    "junnyu/roformer_small_generator": {"do_lower_case": True},
}






class 					SCREAMING_SNAKE_CASE_  (       _a ):
       """simple docstring"""


       __lowerCAmelCase    :		Dict					=VOCAB_FILES_NAMES
       __lowerCAmelCase    :		str					=PRETRAINED_VOCAB_FILES_MAP
       __lowerCAmelCase    :		List[str]					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __lowerCAmelCase    :		Tuple					=PRETRAINED_INIT_CONFIGURATION
       __lowerCAmelCase    :		Optional[Any]					=RoFormerTokenizer





       def __init__(				self	:Any,      snake_case	:Union[str, Any]=None,      snake_case	:int=None,      snake_case	:Union[str, Any]=True,      snake_case	:List[Any]="[UNK]",      snake_case	:Union[str, Any]="[SEP]",      snake_case	:int="[PAD]",      snake_case	:Union[str, Any]="[CLS]",      snake_case	:Optional[int]="[MASK]",      snake_case	:Dict=True,      snake_case	:Optional[Any]=None,      **snake_case	:Union[str, Any],      ):



          """simple docstring"""

          super().__init__(
              snake_case,      tokenizer_file=snake_case,      do_lower_case=snake_case,      unk_token=snake_case,      sep_token=snake_case,      pad_token=snake_case,      cls_token=snake_case,      mask_token=snake_case,      tokenize_chinese_chars=snake_case,      strip_accents=snake_case,      **snake_case,      )

          _lowercase              =json.loads(self.backend_tokenizer.normalizer.__getstate__())
          if (
              pre_tok_state.get('lowercase',      snake_case) != do_lower_case
              or pre_tok_state.get('strip_accents',      snake_case) != strip_accents
          ):
             _lowercase              =getattr(snake_case,      pre_tok_state.pop('type'))
             _lowercase              =do_lower_case
             _lowercase              =strip_accents
             _lowercase              =pre_tok_class(**snake_case)

          _lowercase              =do_lower_case





       def __getstate__(				self	:Any):



          """simple docstring"""

          _lowercase              =self.__dict__.copy()
          _lowercase              =BertPreTokenizer()
          return state





       def __setstate__(				self	:List[Any],      snake_case	:Optional[int]):



          """simple docstring"""

          _lowercase              =d
          _lowercase              =self.__dict__['_tokenizer'].get_vocab()
          _lowercase              =PreTokenizer.custom(JiebaPreTokenizer(snake_case))





       def 				UpperCamelCase__						(				self	:Optional[Any],      snake_case	:Dict,      snake_case	:List[Any]=None):



          """simple docstring"""

          _lowercase              =[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
             output += token_ids_a + [self.sep_token_id]

          return output





       def 				UpperCamelCase__						(				self	:Optional[int],      snake_case	:List[int],      snake_case	:Optional[List[int]] = None):



          """simple docstring"""

          _lowercase              =[self.sep_token_id]
          _lowercase              =[self.cls_token_id]
          if token_ids_a is None:
             return len(cls + token_ids_a + sep) * [0]
          return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]





       def 				UpperCamelCase__						(				self	:Union[str, Any],      snake_case	:str,      snake_case	:Optional[str] = None):



          """simple docstring"""

          _lowercase              =self._tokenizer.model.save(snake_case,      name=snake_case)
          return tuple(snake_case)





       def 				UpperCamelCase__						(				self	:List[str],      snake_case	:List[Any],      snake_case	:List[str]=None,      snake_case	:List[Any]=None,      snake_case	:str=False,      **snake_case	:Union[str, Any],      ):



          """simple docstring"""

          _lowercase              =BertPreTokenizer()
          return super().save_pretrained(snake_case,      snake_case,      snake_case,      snake_case,      **snake_case)










from ...configuration_utils import PretrainedConfig
from ...utils import logging


_SCREAMING_SNAKE_CASE   =  logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE   =  {
    "uw-madison/mra-base-512-4": "https://huggingface.co/uw-madison/mra-base-512-4/resolve/main/config.json",
}






class 					SCREAMING_SNAKE_CASE_  (       _a ):
       """simple docstring"""


       __lowerCAmelCase    :		str					='''mra'''





       def __init__(				self	:Any,      snake_case	:List[str]=5_0265,      snake_case	:List[Any]=768,      snake_case	:Optional[Any]=12,      snake_case	:Optional[Any]=12,      snake_case	:str=3072,      snake_case	:Tuple="gelu",      snake_case	:Optional[int]=0.1,      snake_case	:int=0.1,      snake_case	:Any=512,      snake_case	:Union[str, Any]=1,      snake_case	:Union[str, Any]=0.0_2,      snake_case	:List[Any]=1e-5,      snake_case	:Optional[int]="absolute",      snake_case	:Optional[int]=4,      snake_case	:str="full",      snake_case	:Optional[int]=0,      snake_case	:List[Any]=0,      snake_case	:int=1,      snake_case	:List[Any]=0,      snake_case	:Dict=2,      **snake_case	:Dict,      ):



          """simple docstring"""

          super().__init__(pad_token_id=snake_case,      bos_token_id=snake_case,      eos_token_id=snake_case,      **snake_case)

          _lowercase              =vocab_size
          _lowercase              =max_position_embeddings
          _lowercase              =hidden_size
          _lowercase              =num_hidden_layers
          _lowercase              =num_attention_heads
          _lowercase              =intermediate_size
          _lowercase              =hidden_act
          _lowercase              =hidden_dropout_prob
          _lowercase              =attention_probs_dropout_prob
          _lowercase              =initializer_range
          _lowercase              =type_vocab_size
          _lowercase              =layer_norm_eps
          _lowercase              =position_embedding_type
          _lowercase              =block_per_row
          _lowercase              =approx_mode
          _lowercase              =initial_prior_first_n_blocks
          _lowercase              =initial_prior_diagonal_n_blocks


