


'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ....tokenization_utils_fast import PreTrainedTokenizerFast
from ....utils import logging
from .tokenization_retribert import RetriBertTokenizer


A   : Dict             =			logging.get_logger(__name__)

A   : List[Any]             =			{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

A   : str             =			{
    'vocab_file': {
        'yjernite/retribert-base-uncased': (
            'https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'yjernite/retribert-base-uncased': (
            'https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/tokenizer.json'
        ),
    },
}

A   : List[str]             =			{
    'yjernite/retribert-base-uncased': 512,
}


A   : int             =			{
    'yjernite/retribert-base-uncased': {'do_lower_case': True},
}







class 			lowerCamelCase					(						__snake_case  ):
					_SCREAMING_SNAKE_CASE							   =    VOCAB_FILES_NAMES
					_SCREAMING_SNAKE_CASE							   =    PRETRAINED_VOCAB_FILES_MAP
					_SCREAMING_SNAKE_CASE							   =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					_SCREAMING_SNAKE_CASE							   =    PRETRAINED_INIT_CONFIGURATION
					_SCREAMING_SNAKE_CASE							   =    RetriBertTokenizer
					_SCREAMING_SNAKE_CASE							   =    ["input_ids", "attention_mask"]


					def __init__(					self	:  Any       ,     __snake_case	:  Optional[int]=None       ,     __snake_case	:  Tuple=None       ,     __snake_case	:  Tuple=True       ,     __snake_case	:  Dict="[UNK]"       ,     __snake_case	:  Any="[SEP]"       ,     __snake_case	:  Any="[PAD]"       ,     __snake_case	:  List[Any]="[CLS]"       ,     __snake_case	:  str="[MASK]"       ,     __snake_case	:  List[Any]=True       ,     __snake_case	:  List[Any]=None       ,     **__snake_case	:  Dict       ,     ):





											'''simple docstring'''





											super().__init__(
											    _lowercase       ,     tokenizer_file=_lowercase       ,     do_lower_case=_lowercase       ,     unk_token=_lowercase       ,     sep_token=_lowercase       ,     pad_token=_lowercase       ,     cls_token=_lowercase       ,     mask_token=_lowercase       ,     tokenize_chinese_chars=_lowercase       ,     strip_accents=_lowercase       ,     **_lowercase       ,     )

											_snake_case:   Tuple  					=		json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
											if (
											    normalizer_state.get('lowercase'       ,     _lowercase       ) != do_lower_case
											    or normalizer_state.get('strip_accents'       ,     _lowercase       ) != strip_accents
											    or normalizer_state.get('handle_chinese_chars'       ,     _lowercase       ) != tokenize_chinese_chars
											):
																	_snake_case:   str  					=		getattr(_lowercase       ,     normalizer_state.pop('type'       )       )
																	_snake_case:   str  					=		do_lower_case
																	_snake_case:   str  					=		strip_accents
																	_snake_case:   List[str]  					=		tokenize_chinese_chars
																	_snake_case:   Optional[Any]  					=		normalizer_class(**_lowercase       )

											_snake_case:   List[Any]  					=		do_lower_case


					def    SCREAMING_SNAKE_CASE_		(					self	:  List[str]       ,     __snake_case	:  Optional[Any]       ,     __snake_case	:  Optional[int]=None       ):





											'''simple docstring'''





											_snake_case:   Optional[Any]  					=		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

											if token_ids_a:
																	output += token_ids_a + [self.sep_token_id]

											return output


					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[Any]       ,     __snake_case	:  List[Any]       ,     __snake_case	:  str = None       ):





											'''simple docstring'''





											_snake_case:   Union[str, Any]  					=		[self.sep_token_id]
											_snake_case:   int  					=		[self.cls_token_id]
											if token_ids_a is None:
																	return len(cls + token_ids_a + sep       ) * [0]
											return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]




					def    SCREAMING_SNAKE_CASE_		(					self	:  Any       ,     __snake_case	:  Any       ,     __snake_case	:  List[str] = None       ):





											'''simple docstring'''





											_snake_case:   List[Any]  					=		self._tokenizer.model.save(_lowercase       ,     name=_lowercase       )
											return tuple(_lowercase       )












'''simple docstring'''






from ..utils import DummyObject, requires_backends







class 			lowerCamelCase					(						metaclass=__UpperCAmelCase  ):
					_SCREAMING_SNAKE_CASE							   =    ["torch", "scipy"]


					def __init__(					self	:  Tuple       ,     *__snake_case	:  List[Any]       ,     **__snake_case	:  List[Any]       ):





											'''simple docstring'''





											requires_backends(self       ,     ['torch', 'scipy']       )


					@classmethod
					def    SCREAMING_SNAKE_CASE_		(					cls	:  Optional[int]       ,     *__snake_case	:  Optional[Any]       ,     **__snake_case	:  Tuple       ):





											'''simple docstring'''





											requires_backends(cls       ,     ['torch', 'scipy']       )




					@classmethod
					def    SCREAMING_SNAKE_CASE_		(					cls	:  List[Any]       ,     *__snake_case	:  Optional[Any]       ,     **__snake_case	:  str       ):





											'''simple docstring'''





											requires_backends(cls       ,     ['torch', 'scipy']       )





