



'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


UpperCamelCase_       :  Any					=  logging.get_logger(__name__)

UpperCamelCase_       :  Any					=  {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCamelCase_       :  str					=  {
    """vocab_file""": {
        """junnyu/roformer_chinese_small""": """https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_base""": """https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_char_small""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_chinese_char_base""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_discriminator""": (
            """https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_generator""": (
            """https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"""
        ),
    }
}

UpperCamelCase_       :  Optional[int]					=  {
    """junnyu/roformer_chinese_small""": 1_536,
    """junnyu/roformer_chinese_base""": 1_536,
    """junnyu/roformer_chinese_char_small""": 512,
    """junnyu/roformer_chinese_char_base""": 512,
    """junnyu/roformer_small_discriminator""": 128,
    """junnyu/roformer_small_generator""": 128,
}


UpperCamelCase_       :  List[Any]					=  {
    """junnyu/roformer_chinese_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_base""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_base""": {"""do_lower_case""": True},
    """junnyu/roformer_small_discriminator""": {"""do_lower_case""": True},
    """junnyu/roformer_small_generator""": {"""do_lower_case""": True},
}



class    lowerCamelCase__				(			__SCREAMING_SNAKE_CASE						):






      """simple docstring"""
      UpperCamelCase__			 =     VOCAB_FILES_NAMES
      UpperCamelCase__			 =     PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase__			 =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase__			 =     PRETRAINED_INIT_CONFIGURATION
      UpperCamelCase__			 =     RoFormerTokenizer




      def __init__(       self							:     Optional[int]			,a__							:     Any=None			,a__							:     Optional[Any]=None			,a__							:     int=True			,a__							:     Any="[UNK]"			,a__							:     Tuple="[SEP]"			,a__							:     Optional[Any]="[PAD]"			,a__							:     Union[str, Any]="[CLS]"			,a__							:     int="[MASK]"			,a__							:     str=True			,a__							:     List[Any]=None			,**a__							:     Any			,):
             super().__init__(
                 __lowerCamelCase			,tokenizer_file=__lowerCamelCase			,do_lower_case=__lowerCamelCase			,unk_token=__lowerCamelCase			,sep_token=__lowerCamelCase			,pad_token=__lowerCamelCase			,cls_token=__lowerCamelCase			,mask_token=__lowerCamelCase			,tokenize_chinese_chars=__lowerCamelCase			,strip_accents=__lowerCamelCase			,**__lowerCamelCase			,)

             a__								= json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
             if (
                 pre_tok_state.get("lowercase"			,__lowerCamelCase   ) != do_lower_case
                 or pre_tok_state.get("strip_accents"			,__lowerCamelCase   ) != strip_accents
             ):
                    a__								= getattr(__lowerCamelCase			,pre_tok_state.pop("type"   )   )
                    a__								= do_lower_case
                    a__								= strip_accents
                    a__								= pre_tok_class(**__lowerCamelCase   )

             a__								= do_lower_case




      def __getstate__(       self							:     int   ):
             a__								= self.__dict__.copy()
             a__								= BertPreTokenizer()
             return state




      def __setstate__(       self							:     Dict			,a__							:     Optional[Any]   ):
             a__								= d
             a__								= self.__dict__["_tokenizer"].get_vocab()
             a__								= PreTokenizer.custom(JiebaPreTokenizer(__lowerCamelCase   )   )




      def lowerCAmelCase_					(       self							:     Optional[Any]			,a__							:     Any			,a__							:     List[Any]=None   ):
             a__								= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

             if token_ids_a:
                    output += token_ids_a + [self.sep_token_id]

             return output




      def lowerCAmelCase_					(       self							:     Dict			,a__							:     List[int]			,a__							:     Optional[List[int]] = None   ):
             a__								= [self.sep_token_id]
             a__								= [self.cls_token_id]
             if token_ids_a is None:
                    return len(cls + token_ids_a + sep   ) * [0]
             return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]




      def lowerCAmelCase_					(       self							:     Union[str, Any]			,a__							:     str			,a__							:     Optional[str] = None   ):
             a__								= self._tokenizer.model.save(__lowerCamelCase			,name=__lowerCamelCase   )
             return tuple(__lowerCamelCase   )




      def lowerCAmelCase_					(       self							:     Optional[int]			,a__							:     List[str]			,a__							:     List[str]=None			,a__							:     Union[str, Any]=None			,a__							:     List[str]=False			,**a__							:     List[Any]			,):
             a__								= BertPreTokenizer()
             return super().save_pretrained(__lowerCamelCase			,__lowerCamelCase			,__lowerCamelCase			,__lowerCamelCase			,**__lowerCamelCase   )





"""simple docstring"""






from copy import deepcopy



class     UpperCAmelCase    :



   def __init__(       self    :	Optional[Any]							,       __lowerCamelCase    :	list[int] | None = None							,       __lowerCamelCase    :	int | None = None				):



     """simple docstring"""

     if arr is None and size is not None:
       _snake_case   	=      size
       _snake_case   	=      [0] * size
     elif arr is not None:
       self.init(__lowerCamelCase				)
     else:
       raise ValueError('''Either arr or size must be specified'''				)



   def    __UpperCAmelCase    (       self    :	Any							,       __lowerCamelCase    :	list[int]				):



     """simple docstring"""
     _snake_case   	=      len(__lowerCamelCase				)
     _snake_case   	=      deepcopy(__lowerCamelCase				)
     for i in range(1							,       self.size				):
       _snake_case   	=      self.next_(__lowerCamelCase				)
       if j < self.size:
         self.tree[j] += self.tree[i]



   def    __UpperCAmelCase    (       self    :	Union[str, Any]				):



     """simple docstring"""
     _snake_case   	=      self.tree[:]
     for i in range(self.size - 1							,       0							,       -1				):
       _snake_case   	=      self.next_(__lowerCamelCase				)
       if j < self.size:
         arr[j] -= arr[i]
     return arr



   @staticmethod
   def    __UpperCAmelCase    (       __lowerCamelCase    :	int				):



     """simple docstring"""
     return index + (index & (-index))



   @staticmethod
   def    __UpperCAmelCase    (       __lowerCamelCase    :	int				):



     """simple docstring"""
     return index - (index & (-index))



   def    __UpperCAmelCase    (       self    :	Dict							,       __lowerCamelCase    :	int							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     if index == 0:
       self.tree[0] += value
       return
     while index < self.size:
       self.tree[index] += value
       _snake_case   	=      self.next_(__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	List[str]							,       __lowerCamelCase    :	int							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     self.add(__lowerCamelCase							,       value - self.get(__lowerCamelCase				)				)



   def    __UpperCAmelCase    (       self    :	Optional[Any]							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     if right == 0:
       return 0
     _snake_case   	=      self.tree[0]
     right -= 1  # make right inclusive
     while right > 0:
       result += self.tree[right]
       _snake_case   	=      self.prev(__lowerCamelCase				)
     return result



   def    __UpperCAmelCase    (       self    :	str							,       __lowerCamelCase    :	int							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     return self.prefix(__lowerCamelCase				) - self.prefix(__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	List[str]							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     return self.query(__lowerCamelCase							,       index + 1				)





   def    __UpperCAmelCase    (       self    :	Optional[int]							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     value -= self.tree[0]
     if value < 0:
       return -1

     _snake_case   	=      1  # Largest power of 2 <= size
     while j * 2 < self.size:
       j *= 2

     _snake_case   	=      0

     while j > 0:
       if i + j < self.size and self.tree[i + j] <= value:
         value -= self.tree[i + j]
         i += j
       j //= 2
     return i


if __name__ == "__main__":
     import doctest

     doctest.testmod()





