






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


__a					:				Union[str, Any]      =      logging.get_logger(__name__)

__a					:				int      =      {"vocab_file": "vocab.txt"}

__a					:				int      =      {
    "vocab_file": {
        "tau/splinter-base": "https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt",
        "tau/splinter-base-qass": "https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt",
        "tau/splinter-large": "https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt",
        "tau/splinter-large-qass": "https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt",
    }
}

__a					:				Optional[int]      =      {
    "tau/splinter-base": 512,
    "tau/splinter-base-qass": 512,
    "tau/splinter-large": 512,
    "tau/splinter-large-qass": 512,
}

__a					:				Optional[Any]      =      {
    "tau/splinter-base": {"do_lower_case": False},
    "tau/splinter-base-qass": {"do_lower_case": False},
    "tau/splinter-large": {"do_lower_case": False},
    "tau/splinter-large-qass": {"do_lower_case": False},
}


class 					__lowercase					(		snake_case__     ):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE        = VOCAB_FILES_NAMES
     SCREAMING_SNAKE_CASE        = PRETRAINED_VOCAB_FILES_MAP
     SCREAMING_SNAKE_CASE        = PRETRAINED_INIT_CONFIGURATION
     SCREAMING_SNAKE_CASE        = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     SCREAMING_SNAKE_CASE        = SplinterTokenizer




     def __init__( self				:	int       ,		UpperCamelCase_				:	Optional[int]=None       ,		UpperCamelCase_				:	Dict=None       ,		UpperCamelCase_				:	Any=True       ,		UpperCamelCase_				:	str="[UNK]"       ,		UpperCamelCase_				:	Optional[int]="[SEP]"       ,		UpperCamelCase_				:	Any="[PAD]"       ,		UpperCamelCase_				:	Any="[CLS]"       ,		UpperCamelCase_				:	Dict="[MASK]"       ,		UpperCamelCase_				:	Optional[Any]="[QUESTION]"       ,		UpperCamelCase_				:	str=True       ,		UpperCamelCase_				:	Optional[int]=None       ,		**UpperCamelCase_				:	Dict       ,		):


         """simple docstring"""



         super().__init__(
             __UpperCAmelCase       ,		tokenizer_file=__UpperCAmelCase       ,		do_lower_case=__UpperCAmelCase       ,		unk_token=__UpperCAmelCase       ,		sep_token=__UpperCAmelCase       ,		pad_token=__UpperCAmelCase       ,		cls_token=__UpperCAmelCase       ,		mask_token=__UpperCAmelCase       ,		tokenize_chinese_chars=__UpperCAmelCase       ,		strip_accents=__UpperCAmelCase       ,		additional_special_tokens=(question_token,)       ,		**__UpperCAmelCase       ,		)

         __A   			=		json.loads(self.backend_tokenizer.normalizer.__getstate__() )
         if (
             pre_tok_state.get("""lowercase"""       ,		__UpperCAmelCase ) != do_lower_case
             or pre_tok_state.get("""strip_accents"""       ,		__UpperCAmelCase ) != strip_accents
         ):
             __A   			=		getattr(__UpperCAmelCase       ,		pre_tok_state.pop("""type""" ) )
             __A   			=		do_lower_case
             __A   			=		strip_accents
             __A   			=		pre_tok_class(**__UpperCAmelCase )

         __A   			=		do_lower_case




     @property
     def 						lowerCAmelCase_   ( self				:	Union[str, Any] ):


         """simple docstring"""



         return self.convert_tokens_to_ids(self.question_token )




     def 						lowerCAmelCase_   ( self				:	Tuple       ,		UpperCamelCase_				:	List[int]       ,		UpperCamelCase_				:	Optional[List[int]] = None ):


         """simple docstring"""



         if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

         __A   			=		[self.cls_token_id]
         __A   			=		[self.sep_token_id]
         __A   			=		[self.question_token_id] + [self.convert_tokens_to_ids(""".""" )]
         if self.padding_side == "right":
             # Input is question-then-context
             return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
         else:
             # Input is context-then-question
             return cls + token_ids_a + sep + token_ids_a + question_suffix + sep




     def 						lowerCAmelCase_   ( self				:	Dict       ,		UpperCamelCase_				:	List[int]       ,		UpperCamelCase_				:	Optional[List[int]] = None ):


         """simple docstring"""



         __A   			=		[self.sep_token_id]
         __A   			=		[self.cls_token_id]
         __A   			=		[self.question_token_id] + [self.convert_tokens_to_ids(""".""" )]
         if token_ids_a is None:
             return len(cls + token_ids_a + sep ) * [0]

         if self.padding_side == "right":
             # Input is question-then-context
             return len(cls + token_ids_a + question_suffix + sep ) * [0] + len(token_ids_a + sep ) * [1]
         else:
             # Input is context-then-question
             return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + question_suffix + sep ) * [1]





     def 						lowerCAmelCase_   ( self				:	List[str]       ,		UpperCamelCase_				:	str       ,		UpperCamelCase_				:	Optional[str] = None ):


         """simple docstring"""



         __A   			=		self._tokenizer.model.save(__UpperCAmelCase       ,		name=__UpperCAmelCase )
         return tuple(__UpperCAmelCase )









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__A				    =						{
    '''configuration_mpnet''': ['''MPNET_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''MPNetConfig'''],
    '''tokenization_mpnet''': ['''MPNetTokenizer'''],
}

try:
	if not is_tokenizers_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__A				    =						['''MPNetTokenizerFast''']

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__A				    =						[
	    '''MPNET_PRETRAINED_MODEL_ARCHIVE_LIST''',
	    '''MPNetForMaskedLM''',
	    '''MPNetForMultipleChoice''',
	    '''MPNetForQuestionAnswering''',
	    '''MPNetForSequenceClassification''',
	    '''MPNetForTokenClassification''',
	    '''MPNetLayer''',
	    '''MPNetModel''',
	    '''MPNetPreTrainedModel''',
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__A				    =						[
	    '''TF_MPNET_PRETRAINED_MODEL_ARCHIVE_LIST''',
	    '''TFMPNetEmbeddings''',
	    '''TFMPNetForMaskedLM''',
	    '''TFMPNetForMultipleChoice''',
	    '''TFMPNetForQuestionAnswering''',
	    '''TFMPNetForSequenceClassification''',
	    '''TFMPNetForTokenClassification''',
	    '''TFMPNetMainLayer''',
	    '''TFMPNetModel''',
	    '''TFMPNetPreTrainedModel''',
	]


if TYPE_CHECKING:
	from .configuration_mpnet import MPNET_PRETRAINED_CONFIG_ARCHIVE_MAP, MPNetConfig
	from .tokenization_mpnet import MPNetTokenizer

	try:
		if not is_tokenizers_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_mpnet_fast import MPNetTokenizerFast

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_mpnet import (
		    MPNET_PRETRAINED_MODEL_ARCHIVE_LIST,
		    MPNetForMaskedLM,
		    MPNetForMultipleChoice,
		    MPNetForQuestionAnswering,
		    MPNetForSequenceClassification,
		    MPNetForTokenClassification,
		    MPNetLayer,
		    MPNetModel,
		    MPNetPreTrainedModel,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_mpnet import (
		    TF_MPNET_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFMPNetEmbeddings,
		    TFMPNetForMaskedLM,
		    TFMPNetForMultipleChoice,
		    TFMPNetForQuestionAnswering,
		    TFMPNetForSequenceClassification,
		    TFMPNetForTokenClassification,
		    TFMPNetMainLayer,
		    TFMPNetModel,
		    TFMPNetPreTrainedModel,
		)

else:
	import sys

	__A				    =						_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




