

"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


__lowerCamelCase										=		logging.get_logger(__name__)

__lowerCamelCase										=		{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__lowerCamelCase										=		{
    "vocab_file": {
        "junnyu/roformer_chinese_small": "https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_base": "https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_char_small": (
            "https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_chinese_char_base": (
            "https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_discriminator": (
            "https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_generator": (
            "https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"
        ),
    }
}

__lowerCamelCase										=		{
    "junnyu/roformer_chinese_small": 15_36,
    "junnyu/roformer_chinese_base": 15_36,
    "junnyu/roformer_chinese_char_small": 5_12,
    "junnyu/roformer_chinese_char_base": 5_12,
    "junnyu/roformer_small_discriminator": 1_28,
    "junnyu/roformer_small_generator": 1_28,
}


__lowerCamelCase										=		{
    "junnyu/roformer_chinese_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_base": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_base": {"do_lower_case": True},
    "junnyu/roformer_small_discriminator": {"do_lower_case": True},
    "junnyu/roformer_small_generator": {"do_lower_case": True},
}
class        _lowercase					(     __UpperCAmelCase     ):
     _lowerCamelCase     =       VOCAB_FILES_NAMES
     _lowerCamelCase     =       PRETRAINED_VOCAB_FILES_MAP
     _lowerCamelCase     =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _lowerCamelCase     =       PRETRAINED_INIT_CONFIGURATION
     _lowerCamelCase     =       RoFormerTokenizer







     def __init__(      self ,						UpperCamelCase_=None ,						UpperCamelCase_=None ,						UpperCamelCase_=True ,						UpperCamelCase_="[UNK]" ,						UpperCamelCase_="[SEP]" ,						UpperCamelCase_="[PAD]" ,						UpperCamelCase_="[CLS]" ,						UpperCamelCase_="[MASK]" ,						UpperCamelCase_=True ,						UpperCamelCase_=None ,						**UpperCamelCase_ ,						):
      super().__init__(
          UpperCamelCase_ ,						tokenizer_file=UpperCamelCase_ ,						do_lower_case=UpperCamelCase_ ,						unk_token=UpperCamelCase_ ,						sep_token=UpperCamelCase_ ,						pad_token=UpperCamelCase_ ,						cls_token=UpperCamelCase_ ,						mask_token=UpperCamelCase_ ,						tokenize_chinese_chars=UpperCamelCase_ ,						strip_accents=UpperCamelCase_ ,						**UpperCamelCase_ ,						)

      __magic_name__     =      json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
      if (
          pre_tok_state.get('''lowercase''' ,						UpperCamelCase_       ) != do_lower_case
          or pre_tok_state.get('''strip_accents''' ,						UpperCamelCase_       ) != strip_accents
      ):
       __magic_name__     =      getattr(UpperCamelCase_ ,						pre_tok_state.pop('''type'''       )       )
       __magic_name__     =      do_lower_case
       __magic_name__     =      strip_accents
       __magic_name__     =      pre_tok_class(**UpperCamelCase_       )

      __magic_name__     =      do_lower_case







     def __getstate__(      self       ):
      __magic_name__     =      self.__dict__.copy()
      __magic_name__     =      BertPreTokenizer()
      return state







     def __setstate__(      self ,						UpperCamelCase_       ):
      __magic_name__     =      d
      __magic_name__     =      self.__dict__['''_tokenizer'''].get_vocab()
      __magic_name__     =      PreTokenizer.custom(JiebaPreTokenizer(UpperCamelCase_       )       )







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_=None       ):
      __magic_name__     =      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

      return output







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_ = None       ):
      __magic_name__     =      [self.sep_token_id]
      __magic_name__     =      [self.cls_token_id]
      if token_ids_a is None:
       return len(cls + token_ids_a + sep       ) * [0]
      return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_ = None       ):
      __magic_name__     =      self._tokenizer.model.save(UpperCamelCase_ ,						name=UpperCamelCase_       )
      return tuple(UpperCamelCase_       )







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_=None ,						UpperCamelCase_=None ,						UpperCamelCase_=False ,						**UpperCamelCase_ ,						):
      __magic_name__     =      BertPreTokenizer()
      return super().save_pretrained(UpperCamelCase_ ,						UpperCamelCase_ ,						UpperCamelCase_ ,						UpperCamelCase_ ,						**UpperCamelCase_       )





"""simple docstring"""






class        _lowercase					:







     def __init__(      self ,						UpperCamelCase_       ):
      __magic_name__     =      size
      __magic_name__     =      [0] * size
      __magic_name__     =      [0] * size







     @staticmethod
     def 	lowerCAmelCase__    (      UpperCamelCase_       ):
      return index | (index + 1)







     @staticmethod
     def 	lowerCAmelCase__    (      UpperCamelCase_       ):
      return (index & (index + 1)) - 1







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_       ):
      __magic_name__     =      value
      while index < self.size:
       __magic_name__     =      self.get_prev(UpperCamelCase_       ) + 1
       if current_left_border == index:
        __magic_name__     =      value
       else:
        __magic_name__     =      max(UpperCamelCase_ ,						UpperCamelCase_ ,						UpperCamelCase_       )
       __magic_name__     =      self.get_next(UpperCamelCase_       )







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_       ):
      right -= 1  # Because of right is exclusive
      __magic_name__     =      0
      while left <= right:
       __magic_name__     =      self.get_prev(UpperCamelCase_       )
       if left <= current_left:
        __magic_name__     =      max(UpperCamelCase_ ,						self.tree[right]       )
        __magic_name__     =      current_left
       else:
        __magic_name__     =      max(UpperCamelCase_ ,						self.arr[right]       )
        right -= 1
      return result


if __name__ == "__main__":
 import doctest

 doctest.testmod()

