





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


UpperCAmelCase_      							=     logging.get_logger(__name__)

UpperCAmelCase_      							=     {"""vocab_file""": """vocab.txt"""}

UpperCAmelCase_      							=     {
    """vocab_file""": {
        """YituTech/conv-bert-base""": """https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt""",
        """YituTech/conv-bert-medium-small""": (
            """https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt"""
        ),
        """YituTech/conv-bert-small""": """https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt""",
    }
}

UpperCAmelCase_      							=     {
    """YituTech/conv-bert-base""": 512,
    """YituTech/conv-bert-medium-small""": 512,
    """YituTech/conv-bert-small""": 512,
}


UpperCAmelCase_      							=     {
    """YituTech/conv-bert-base""": {"""do_lower_case""": True},
    """YituTech/conv-bert-medium-small""": {"""do_lower_case""": True},
    """YituTech/conv-bert-small""": {"""do_lower_case""": True},
}
class 			UpperCamelCase_		(  _lowerCamelCase					):
					lowerCAmelCase_							=	VOCAB_FILES_NAMES
					lowerCAmelCase_							=	PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_							=	PRETRAINED_INIT_CONFIGURATION
					lowerCAmelCase_							=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_							=	ConvBertTokenizer




					def __init__(		self				,						lowerCAmelCase_=None				,						lowerCAmelCase_=None				,						lowerCAmelCase_=True				,						lowerCAmelCase_="[UNK]"				,						lowerCAmelCase_="[SEP]"				,						lowerCAmelCase_="[PAD]"				,						lowerCAmelCase_="[CLS]"				,						lowerCAmelCase_="[MASK]"				,						lowerCAmelCase_=True				,						lowerCAmelCase_=None				,						**lowerCAmelCase_				,						)		->							Dict:
										super().__init__(
										    lowerCAmelCase_				,						tokenizer_file=lowerCAmelCase_				,						do_lower_case=lowerCAmelCase_				,						unk_token=lowerCAmelCase_				,						sep_token=lowerCAmelCase_				,						pad_token=lowerCAmelCase_				,						cls_token=lowerCAmelCase_				,						mask_token=lowerCAmelCase_				,						tokenize_chinese_chars=lowerCAmelCase_				,						strip_accents=lowerCAmelCase_				,						**lowerCAmelCase_				,						)

										_snake_case									=				json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
										if (
										    normalizer_state.get('lowercase'				,						lowerCAmelCase_  ) != do_lower_case
										    or normalizer_state.get('strip_accents'				,						lowerCAmelCase_  ) != strip_accents
										    or normalizer_state.get('handle_chinese_chars'				,						lowerCAmelCase_  ) != tokenize_chinese_chars
										):
															_snake_case									=				getattr(lowerCAmelCase_				,						normalizer_state.pop('type'  )  )
															_snake_case									=				do_lower_case
															_snake_case									=				strip_accents
															_snake_case									=				tokenize_chinese_chars
															_snake_case									=				normalizer_class(**lowerCAmelCase_  )

										_snake_case									=				do_lower_case




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_=None  )		->							Optional[Any]:
										_snake_case									=				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

										if token_ids_a:
															output += token_ids_a + [self.sep_token_id]

										return output




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_ = None  )		->							List[int]:
										_snake_case									=				[self.sep_token_id]
										_snake_case									=				[self.cls_token_id]
										if token_ids_a is None:
															return len(cls + token_ids_a + sep  ) * [0]
										return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]






					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_ = None  )		->							Tuple[str]:
										_snake_case									=				self._tokenizer.model.save(lowerCAmelCase_				,						name=lowerCAmelCase_  )
										return tuple(lowerCAmelCase_  )










from collections import deque

from .hash_table import HashTable
class 			UpperCamelCase_		(  _lowerCamelCase					):




					def __init__(		self				,						*lowerCAmelCase_				,						**lowerCAmelCase_  )		->							Any:
										super().__init__(*lowerCAmelCase_				,						**lowerCAmelCase_  )




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_  )		->							Optional[int]:
										_snake_case									=				deque([]  ) if self.values[key] is None else self.values[key]
										self.values[key].appendleft(lowerCAmelCase_  )
										_snake_case									=				self.values[key]




					def 	lowerCAmelCase			(		self  )		->							Optional[int]:
										return (
										    sum(self.charge_factor - len(lowerCAmelCase_  ) for slot in self.values  )
										    / self.size_table
										    * self.charge_factor
										)






					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_=None  )		->							str:
										if not (
										    len(self.values[key]  ) == self.charge_factor and self.values.count(lowerCAmelCase_  ) == 0
										):
															return key
										return super()._collision_resolution(lowerCAmelCase_				,						lowerCAmelCase_  )



