

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


lowerCAmelCase				:    Optional[Any]        =		logging.get_logger(__name__)

lowerCAmelCase				:    Tuple        =		{'vocab_file': 'vocab.txt'}

lowerCAmelCase				:    Union[str, Any]        =		{
    'vocab_file': {
        'YituTech/conv-bert-base': 'https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt',
        'YituTech/conv-bert-medium-small': (
            'https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt'
        ),
        'YituTech/conv-bert-small': 'https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt',
    }
}

lowerCAmelCase				:    Any        =		{
    'YituTech/conv-bert-base': 512,
    'YituTech/conv-bert-medium-small': 512,
    'YituTech/conv-bert-small': 512,
}


lowerCAmelCase				:    List[str]        =		{
    'YituTech/conv-bert-base': {'do_lower_case': True},
    'YituTech/conv-bert-medium-small': {'do_lower_case': True},
    'YituTech/conv-bert-small': {'do_lower_case': True},
}



class     __lowercase							(					__lowercase      ):




			"""simple docstring"""







			_UpperCAmelCase  :				List[Any]     				=	VOCAB_FILES_NAMES
			_UpperCAmelCase  :				Dict     				=	PRETRAINED_VOCAB_FILES_MAP
			_UpperCAmelCase  :				Dict     				=	PRETRAINED_INIT_CONFIGURATION
			_UpperCAmelCase  :				List[str]     				=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCAmelCase  :				List[Any]     				=	ConvBertTokenizer



			def __init__(					self    :	str  ,		lowerCAmelCase__    :	Union[str, Any]=None  ,		lowerCAmelCase__    :	List[Any]=None  ,		lowerCAmelCase__    :	Any=True  ,		lowerCAmelCase__    :	Tuple="[UNK]"  ,		lowerCAmelCase__    :	Tuple="[SEP]"  ,		lowerCAmelCase__    :	List[Any]="[PAD]"  ,		lowerCAmelCase__    :	List[str]="[CLS]"  ,		lowerCAmelCase__    :	Tuple="[MASK]"  ,		lowerCAmelCase__    :	List[Any]=True  ,		lowerCAmelCase__    :	Dict=None  ,		**lowerCAmelCase__    :	List[Any]  ,		):
									super().__init__(
									    lowerCAmelCase__  ,		tokenizer_file=lowerCAmelCase__  ,		do_lower_case=lowerCAmelCase__  ,		unk_token=lowerCAmelCase__  ,		sep_token=lowerCAmelCase__  ,		pad_token=lowerCAmelCase__  ,		cls_token=lowerCAmelCase__  ,		mask_token=lowerCAmelCase__  ,		tokenize_chinese_chars=lowerCAmelCase__  ,		strip_accents=lowerCAmelCase__  ,		**lowerCAmelCase__  ,		)

									SCREAMING_SNAKE_CASE_:					Dict    						=			json.loads(self.backend_tokenizer.normalizer.__getstate__())
									if (
									    normalizer_state.get("lowercase"  ,		lowerCAmelCase__) != do_lower_case
									    or normalizer_state.get("strip_accents"  ,		lowerCAmelCase__) != strip_accents
									    or normalizer_state.get("handle_chinese_chars"  ,		lowerCAmelCase__) != tokenize_chinese_chars
									):
															SCREAMING_SNAKE_CASE_:					Tuple    						=			getattr(lowerCAmelCase__  ,		normalizer_state.pop("type"))
															SCREAMING_SNAKE_CASE_:					Any    						=			do_lower_case
															SCREAMING_SNAKE_CASE_:					Union[str, Any]    						=			strip_accents
															SCREAMING_SNAKE_CASE_:					Dict    						=			tokenize_chinese_chars
															SCREAMING_SNAKE_CASE_:					Dict    						=			normalizer_class(**lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_:					List[Any]    						=			do_lower_case



			def 			_SCREAMING_SNAKE_CASE     (					self    :	Dict  ,		lowerCAmelCase__    :	Dict  ,		lowerCAmelCase__    :	Optional[Any]=None):
									SCREAMING_SNAKE_CASE_:					List[str]    						=			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
															output += token_ids_a + [self.sep_token_id]

									return output



			def 			_SCREAMING_SNAKE_CASE     (					self    :	Dict  ,		lowerCAmelCase__    :	List[int]  ,		lowerCAmelCase__    :	Optional[List[int]] = None):
									SCREAMING_SNAKE_CASE_:					Dict    						=			[self.sep_token_id]
									SCREAMING_SNAKE_CASE_:					Union[str, Any]    						=			[self.cls_token_id]
									if token_ids_a is None:
															return len(cls + token_ids_a + sep) * [0]
									return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]




			def 			_SCREAMING_SNAKE_CASE     (					self    :	List[str]  ,		lowerCAmelCase__    :	str  ,		lowerCAmelCase__    :	Optional[str] = None):
									SCREAMING_SNAKE_CASE_:					Optional[int]    						=			self._tokenizer.model.save(lowerCAmelCase__  ,		name=lowerCAmelCase__)
									return tuple(lowerCAmelCase__)








import warnings

from ...utils import logging
from .image_processing_perceiver import PerceiverImageProcessor


lowercase    :   List[str]						=			logging.get_logger(__name__)




class 					lowerCamelCase__				(							__lowercase):




	'''simple docstring'''




	def __init__(   self						:Dict	,  *a						:Tuple	,  **a						:List[Any]		)						->					None:
								warnings.warn(
								    "The class PerceiverFeatureExtractor is deprecated and will be removed in version 5 of Transformers."
								    " Please use PerceiverImageProcessor instead."	,  a	,  )
								super().__init__(*a	,  **a		)