






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


lowerCAmelCase_           = logging.get_logger(__name__)

lowerCAmelCase_           = {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase_           = {
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

lowerCAmelCase_           = {
    'junnyu/roformer_chinese_small': 1_536,
    'junnyu/roformer_chinese_base': 1_536,
    'junnyu/roformer_chinese_char_small': 512,
    'junnyu/roformer_chinese_char_base': 512,
    'junnyu/roformer_small_discriminator': 128,
    'junnyu/roformer_small_generator': 128,
}


lowerCAmelCase_           = {
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}
class      UpperCamelCase (				snake_case__ ):







		"""simple docstring"""



		snake_case    =   VOCAB_FILES_NAMES
		snake_case    =   PRETRAINED_VOCAB_FILES_MAP
		snake_case    =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		snake_case    =   PRETRAINED_INIT_CONFIGURATION
		snake_case    =   RoFormerTokenizer
		def __init__(  self		:       Dict						,_SCREAMING_SNAKE_CASE		:       str=None						,_SCREAMING_SNAKE_CASE		:       Dict=None						,_SCREAMING_SNAKE_CASE		:       Optional[Any]=True						,_SCREAMING_SNAKE_CASE		:       str="[UNK]"						,_SCREAMING_SNAKE_CASE		:       int="[SEP]"						,_SCREAMING_SNAKE_CASE		:       str="[PAD]"						,_SCREAMING_SNAKE_CASE		:       List[Any]="[CLS]"						,_SCREAMING_SNAKE_CASE		:       Optional[int]="[MASK]"						,_SCREAMING_SNAKE_CASE		:       int=True						,_SCREAMING_SNAKE_CASE		:       List[Any]=None						,**_SCREAMING_SNAKE_CASE		:       List[Any]						,)    ->   Dict:


						'''simple docstring'''
						super().__init__(
						    _SCREAMING_SNAKE_CASE						,tokenizer_file=_SCREAMING_SNAKE_CASE						,do_lower_case=_SCREAMING_SNAKE_CASE						,unk_token=_SCREAMING_SNAKE_CASE						,sep_token=_SCREAMING_SNAKE_CASE						,pad_token=_SCREAMING_SNAKE_CASE						,cls_token=_SCREAMING_SNAKE_CASE						,mask_token=_SCREAMING_SNAKE_CASE						,tokenize_chinese_chars=_SCREAMING_SNAKE_CASE						,strip_accents=_SCREAMING_SNAKE_CASE						,**_SCREAMING_SNAKE_CASE						,)

						A	 =      json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
						if (
						    pre_tok_state.get('lowercase'						,_SCREAMING_SNAKE_CASE		) != do_lower_case
						    or pre_tok_state.get('strip_accents'						,_SCREAMING_SNAKE_CASE		) != strip_accents
						):
										A	 =      getattr(_SCREAMING_SNAKE_CASE						,pre_tok_state.pop('type'		)		)
										A	 =      do_lower_case
										A	 =      strip_accents
										A	 =      pre_tok_class(**_SCREAMING_SNAKE_CASE		)

						A	 =      do_lower_case
		def __getstate__(  self		:       Optional[Any]		)    ->   List[str]:


						'''simple docstring'''
						A	 =      self.__dict__.copy()
						A	 =      BertPreTokenizer()
						return state
		def __setstate__(  self		:       Any						,_SCREAMING_SNAKE_CASE		:       List[Any]		)    ->   Tuple:


						'''simple docstring'''
						A	 =      d
						A	 =      self.__dict__['_tokenizer'].get_vocab()
						A	 =      PreTokenizer.custom(JiebaPreTokenizer(_SCREAMING_SNAKE_CASE		)		)
		def       A(  self		:       Any						,_SCREAMING_SNAKE_CASE		:       Union[str, Any]						,_SCREAMING_SNAKE_CASE		:       Any=None		)    ->   Tuple:


						'''simple docstring'''
						A	 =      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

						if token_ids_a:
										output += token_ids_a + [self.sep_token_id]

						return output
		def       A(  self		:       int						,_SCREAMING_SNAKE_CASE		:       List[int]						,_SCREAMING_SNAKE_CASE		:       Optional[List[int]] = None		)    ->   List[int]:


						'''simple docstring'''
						A	 =      [self.sep_token_id]
						A	 =      [self.cls_token_id]
						if token_ids_a is None:
										return len(cls + token_ids_a + sep		) * [0]
						return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]
		def       A(  self		:       List[str]						,_SCREAMING_SNAKE_CASE		:       str						,_SCREAMING_SNAKE_CASE		:       Optional[str] = None		)    ->   Tuple[str]:


						'''simple docstring'''
						A	 =      self._tokenizer.model.save(_SCREAMING_SNAKE_CASE						,name=_SCREAMING_SNAKE_CASE		)
						return tuple(_SCREAMING_SNAKE_CASE		)







		def       A(  self		:       Optional[int]						,_SCREAMING_SNAKE_CASE		:       Tuple						,_SCREAMING_SNAKE_CASE		:       Dict=None						,_SCREAMING_SNAKE_CASE		:       List[str]=None						,_SCREAMING_SNAKE_CASE		:       Any=False						,**_SCREAMING_SNAKE_CASE		:       str						,)    ->   Optional[int]:


						'''simple docstring'''
						A	 =      BertPreTokenizer()
						return super().save_pretrained(_SCREAMING_SNAKE_CASE						,_SCREAMING_SNAKE_CASE						,_SCREAMING_SNAKE_CASE						,_SCREAMING_SNAKE_CASE						,**_SCREAMING_SNAKE_CASE		)














from collections import deque

from .hash_table import HashTable
class      UpperCamelCase (				snake_case__ ):







		"""simple docstring"""



		def __init__(  self		:       Any						,*_SCREAMING_SNAKE_CASE		:       Optional[int]						,**_SCREAMING_SNAKE_CASE		:       Optional[Any]		)    ->   int:


						'''simple docstring'''
						super().__init__(*_SCREAMING_SNAKE_CASE						,**_SCREAMING_SNAKE_CASE		)
		def       A(  self		:       Optional[int]						,_SCREAMING_SNAKE_CASE		:       Optional[Any]						,_SCREAMING_SNAKE_CASE		:       Tuple		)    ->   List[str]:


						'''simple docstring'''
						A	 =      deque([]		) if self.values[key] is None else self.values[key]
						self.values[key].appendleft(_SCREAMING_SNAKE_CASE		)
						A	 =      self.values[key]
		def       A(  self		:       Optional[Any]		)    ->   Union[str, Any]:


						'''simple docstring'''
						return (
						    sum(self.charge_factor - len(_SCREAMING_SNAKE_CASE		) for slot in self.values		)
						    / self.size_table
						    * self.charge_factor
						)







		def       A(  self		:       Dict						,_SCREAMING_SNAKE_CASE		:       Optional[int]						,_SCREAMING_SNAKE_CASE		:       str=None		)    ->   Union[str, Any]:


						'''simple docstring'''
						if not (
						    len(self.values[key]		) == self.charge_factor and self.values.count(_SCREAMING_SNAKE_CASE		) == 0
						):
										return key
						return super()._collision_resolution(_SCREAMING_SNAKE_CASE						,_SCREAMING_SNAKE_CASE		)





