






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


A        =			logging.get_logger(__name__)

A        =			{'vocab_file': 'vocab.txt'}

A        =			{
    'vocab_file': {
        'YituTech/conv-bert-base': 'https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt',
        'YituTech/conv-bert-medium-small': (
            'https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt'
        ),
        'YituTech/conv-bert-small': 'https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt',
    }
}

A        =			{
    'YituTech/conv-bert-base': 512,
    'YituTech/conv-bert-medium-small': 512,
    'YituTech/conv-bert-small': 512,
}


A        =			{
    'YituTech/conv-bert-base': {'do_lower_case': True},
    'YituTech/conv-bert-medium-small': {'do_lower_case': True},
    'YituTech/conv-bert-small': {'do_lower_case': True},
}


class 							__a							(  _snake_case						):





  '''simple docstring'''



  UpperCAmelCase__      :				List[Any]				    =						VOCAB_FILES_NAMES
  UpperCAmelCase__      :				str				    =						PRETRAINED_VOCAB_FILES_MAP
  UpperCAmelCase__      :				Dict				    =						PRETRAINED_INIT_CONFIGURATION
  UpperCAmelCase__      :				Optional[int]				    =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  UpperCAmelCase__      :				List[Any]				    =						ConvBertTokenizer


  def __init__(						self ,    UpperCamelCase__=None ,    UpperCamelCase__=None ,    UpperCamelCase__=True ,    UpperCamelCase__="[UNK]" ,    UpperCamelCase__="[SEP]" ,    UpperCamelCase__="[PAD]" ,    UpperCamelCase__="[CLS]" ,    UpperCamelCase__="[MASK]" ,    UpperCamelCase__=True ,    UpperCamelCase__=None ,    **UpperCamelCase__ ,    ):
      super().__init__(
          UpperCamelCase__ ,    tokenizer_file=UpperCamelCase__ ,    do_lower_case=UpperCamelCase__ ,    unk_token=UpperCamelCase__ ,    sep_token=UpperCamelCase__ ,    pad_token=UpperCamelCase__ ,    cls_token=UpperCamelCase__ ,    mask_token=UpperCamelCase__ ,    tokenize_chinese_chars=UpperCamelCase__ ,    strip_accents=UpperCamelCase__ ,    **UpperCamelCase__ ,    )

      SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
      if (
          normalizer_state.get('lowercase' ,    UpperCamelCase__       ) != do_lower_case
          or normalizer_state.get('strip_accents' ,    UpperCamelCase__       ) != strip_accents
          or normalizer_state.get('handle_chinese_chars' ,    UpperCamelCase__       ) != tokenize_chinese_chars
      ):
          SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   getattr(UpperCamelCase__ ,    normalizer_state.pop('type'       )       )
          SCREAMING_SNAKE_CASE_   :	Tuple													=   do_lower_case
          SCREAMING_SNAKE_CASE_   :	str													=   strip_accents
          SCREAMING_SNAKE_CASE_   :	List[Any]													=   tokenize_chinese_chars
          SCREAMING_SNAKE_CASE_   :	List[Any]													=   normalizer_class(**UpperCamelCase__       )

      SCREAMING_SNAKE_CASE_   :	List[str]													=   do_lower_case


  def   __snake_case			(						self ,    UpperCamelCase__ ,    UpperCamelCase__=None       ):
      SCREAMING_SNAKE_CASE_   :	List[Any]													=   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
          output += token_ids_a + [self.sep_token_id]

      return output


  def   __snake_case			(						self ,    UpperCamelCase__ ,    UpperCamelCase__ = None       ):
      SCREAMING_SNAKE_CASE_   :	List[Any]													=   [self.sep_token_id]
      SCREAMING_SNAKE_CASE_   :	List[Any]													=   [self.cls_token_id]
      if token_ids_a is None:
          return len(cls + token_ids_a + sep       ) * [0]
      return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]






  def   __snake_case			(						self ,    UpperCamelCase__ ,    UpperCamelCase__ = None       ):
      SCREAMING_SNAKE_CASE_   :	Any													=   self._tokenizer.model.save(UpperCamelCase__ ,    name=UpperCamelCase__       )
      return tuple(UpperCamelCase__       )







from timeit import timeit



def 					_lowerCamelCase(			lowerCAmelCase__				:		int      ):





				'''simple docstring'''







				if number < 0:
								raise ValueError('the value of input must not be negative'      )
				SCREAMING_SNAKE_CASE_   :	Tuple													=   0
				while number:
								number &= number - 1
								result += 1
				return result



def 					_lowerCamelCase(			lowerCAmelCase__				:		int      ):





				'''simple docstring'''







				if number < 0:
								raise ValueError('the value of input must not be negative'      )
				SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   0
				while number:
								if number % 2 == 1:
												result += 1
								number >>= 1
				return result



def 					_lowerCamelCase(			):





				'''simple docstring'''







				def do_benchmark(lowerCAmelCase__				:		int      ) -> None:
								SCREAMING_SNAKE_CASE_   :	int													=   'import __main__ as z'
								print(F'''Benchmark when {number = }:'''      )
								print(F'''{get_set_bits_count_using_modulo_operator(lowerCAmelCase__      ) = }'''      )
								SCREAMING_SNAKE_CASE_   :	int													=   timeit('z.get_set_bits_count_using_modulo_operator(25)'    ,    setup=lowerCAmelCase__      )
								print(F'''timeit() runs in {timing} seconds'''      )
								print(F'''{get_set_bits_count_using_brian_kernighans_algorithm(lowerCAmelCase__      ) = }'''      )
								SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   timeit(
								    'z.get_set_bits_count_using_brian_kernighans_algorithm(25)'    ,    setup=lowerCAmelCase__    ,    )
								print(F'''timeit() runs in {timing} seconds'''      )

				for number in (25, 37, 58, 0):
								do_benchmark(lowerCAmelCase__      )
								print()


if __name__ == "__main__":
				import doctest

				doctest.testmod()
				benchmark()