



"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


__SCREAMING_SNAKE_CASE     :  List[str]										=					logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE     :  str										=					{'''vocab_file''': '''vocab.txt'''}

__SCREAMING_SNAKE_CASE     :  Optional[int]										=					{
    '''vocab_file''': {
        '''tau/splinter-base''': '''https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt''',
        '''tau/splinter-base-qass''': '''https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt''',
        '''tau/splinter-large''': '''https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt''',
        '''tau/splinter-large-qass''': '''https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt''',
    }
}

__SCREAMING_SNAKE_CASE     :  List[str]										=					{
    '''tau/splinter-base''': 5_1_2,
    '''tau/splinter-base-qass''': 5_1_2,
    '''tau/splinter-large''': 5_1_2,
    '''tau/splinter-large-qass''': 5_1_2,
}

__SCREAMING_SNAKE_CASE     :  Any										=					{
    '''tau/splinter-base''': {'''do_lower_case''': False},
    '''tau/splinter-base-qass''': {'''do_lower_case''': False},
    '''tau/splinter-large''': {'''do_lower_case''': False},
    '''tau/splinter-large-qass''': {'''do_lower_case''': False},
}


class 							lowerCamelCase_(      A__			):





					'''simple docstring'''



					lowercase__      :				List[str]        =       VOCAB_FILES_NAMES
					lowercase__      :				Any        =       PRETRAINED_VOCAB_FILES_MAP
					lowercase__      :				int        =       PRETRAINED_INIT_CONFIGURATION
					lowercase__      :				Tuple        =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowercase__      :				int        =       SplinterTokenizer

					def __init__(			self ,				lowerCamelCase__=None ,				lowerCamelCase__=None ,				lowerCamelCase__=True ,				lowerCamelCase__="[UNK]" ,				lowerCamelCase__="[SEP]" ,				lowerCamelCase__="[PAD]" ,				lowerCamelCase__="[CLS]" ,				lowerCamelCase__="[MASK]" ,				lowerCamelCase__="[QUESTION]" ,				lowerCamelCase__=True ,				lowerCamelCase__=None ,				**lowerCamelCase__ ,				):
									super().__init__(
									    lowerCamelCase__ ,				tokenizer_file=lowerCamelCase__ ,				do_lower_case=lowerCamelCase__ ,				unk_token=lowerCamelCase__ ,				sep_token=lowerCamelCase__ ,				pad_token=lowerCamelCase__ ,				cls_token=lowerCamelCase__ ,				mask_token=lowerCamelCase__ ,				tokenize_chinese_chars=lowerCamelCase__ ,				strip_accents=lowerCamelCase__ ,				additional_special_tokens=(question_token,) ,				**lowerCamelCase__ ,				)

									_lowerCamelCase					     = json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
									if (
									    pre_tok_state.get('''lowercase''' ,				lowerCamelCase__      ) != do_lower_case
									    or pre_tok_state.get('''strip_accents''' ,				lowerCamelCase__      ) != strip_accents
									):
													_lowerCamelCase					     = getattr(lowerCamelCase__ ,				pre_tok_state.pop('''type'''      )      )
													_lowerCamelCase					     = do_lower_case
													_lowerCamelCase					     = strip_accents
													_lowerCamelCase					     = pre_tok_class(**lowerCamelCase__      )

									_lowerCamelCase					     = do_lower_case

					@property
					def 	snake_case__					(			self      ):
									return self.convert_tokens_to_ids(self.question_token      )

					def 	snake_case__					(			self ,				lowerCamelCase__ ,				lowerCamelCase__ = None      ):
									if token_ids_a is None:
													return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

									_lowerCamelCase					     = [self.cls_token_id]
									_lowerCamelCase					     = [self.sep_token_id]
									_lowerCamelCase					     = [self.question_token_id] + [self.convert_tokens_to_ids('''.'''      )]
									if self.padding_side == "right":
													# Input is question-then-context
													return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
									else:
													# Input is context-then-question
													return cls + token_ids_a + sep + token_ids_a + question_suffix + sep

					def 	snake_case__					(			self ,				lowerCamelCase__ ,				lowerCamelCase__ = None      ):
									_lowerCamelCase					     = [self.sep_token_id]
									_lowerCamelCase					     = [self.cls_token_id]
									_lowerCamelCase					     = [self.question_token_id] + [self.convert_tokens_to_ids('''.'''      )]
									if token_ids_a is None:
													return len(cls + token_ids_a + sep      ) * [0]

									if self.padding_side == "right":
													# Input is question-then-context
													return len(cls + token_ids_a + question_suffix + sep      ) * [0] + len(token_ids_a + sep      ) * [1]
									else:
													# Input is context-then-question
													return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + question_suffix + sep      ) * [1]





					def 	snake_case__					(			self ,				lowerCamelCase__ ,				lowerCamelCase__ = None      ):
									_lowerCamelCase					     = self._tokenizer.model.save(lowerCamelCase__ ,				name=lowerCamelCase__      )
									return tuple(lowerCamelCase__      )

from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowerCamelCase__   = logging.get_logger(__name__)

lowerCamelCase__   = {
    '''nghuyong/ernie-1.0-base-zh''': '''https://huggingface.co/nghuyong/ernie-1.0-base-zh/resolve/main/config.json''',
    '''nghuyong/ernie-2.0-base-en''': '''https://huggingface.co/nghuyong/ernie-2.0-base-en/resolve/main/config.json''',
    '''nghuyong/ernie-2.0-large-en''': '''https://huggingface.co/nghuyong/ernie-2.0-large-en/resolve/main/config.json''',
    '''nghuyong/ernie-3.0-base-zh''': '''https://huggingface.co/nghuyong/ernie-3.0-base-zh/resolve/main/config.json''',
    '''nghuyong/ernie-3.0-medium-zh''': '''https://huggingface.co/nghuyong/ernie-3.0-medium-zh/resolve/main/config.json''',
    '''nghuyong/ernie-3.0-mini-zh''': '''https://huggingface.co/nghuyong/ernie-3.0-mini-zh/resolve/main/config.json''',
    '''nghuyong/ernie-3.0-micro-zh''': '''https://huggingface.co/nghuyong/ernie-3.0-micro-zh/resolve/main/config.json''',
    '''nghuyong/ernie-3.0-nano-zh''': '''https://huggingface.co/nghuyong/ernie-3.0-nano-zh/resolve/main/config.json''',
    '''nghuyong/ernie-gram-zh''': '''https://huggingface.co/nghuyong/ernie-gram-zh/resolve/main/config.json''',
    '''nghuyong/ernie-health-zh''': '''https://huggingface.co/nghuyong/ernie-health-zh/resolve/main/config.json''',
}




class  _UpperCAmelCase       (			lowerCAmelCase   ):



 '''simple docstring'''




 __A           = '''ernie'''







 def __init__(						self							:					Dict				,      lowercase_							:					List[Any]=30522				,      lowercase_							:					Optional[int]=768				,      lowercase_							:					Tuple=12				,      lowercase_							:					Optional[int]=12				,      lowercase_							:					Any=3072				,      lowercase_							:					int="gelu"				,      lowercase_							:					List[Any]=0.1				,      lowercase_							:					Any=0.1				,      lowercase_							:					str=512				,      lowercase_							:					Any=2				,      lowercase_							:					Any=3				,      lowercase_							:					List[Any]=False				,      lowercase_							:					List[Any]=0.02				,      lowercase_							:					str=1e-1_2				,      lowercase_							:					int=0				,      lowercase_							:					str="absolute"				,      lowercase_							:					Dict=True				,      lowercase_							:					List[str]=None				,      **lowercase_							:					Dict				,      )		->     Union[str, Any]:


   """simple docstring"""







   super().__init__(pad_token_id=lowercase_				,      **lowercase_)

   _UpperCamelCase				=  vocab_size
   _UpperCamelCase				=  hidden_size
   _UpperCamelCase				=  num_hidden_layers
   _UpperCamelCase				=  num_attention_heads
   _UpperCamelCase				=  hidden_act
   _UpperCamelCase				=  intermediate_size
   _UpperCamelCase				=  hidden_dropout_prob
   _UpperCamelCase				=  attention_probs_dropout_prob
   _UpperCamelCase				=  max_position_embeddings
   _UpperCamelCase				=  type_vocab_size
   _UpperCamelCase				=  task_type_vocab_size
   _UpperCamelCase				=  use_task_id
   _UpperCamelCase				=  initializer_range
   _UpperCamelCase				=  layer_norm_eps
   _UpperCamelCase				=  position_embedding_type
   _UpperCamelCase				=  use_cache
   _UpperCamelCase				=  classifier_dropout




class  _UpperCAmelCase       (			lowerCAmelCase   ):



 '''simple docstring'''




 @property
 def 						__UpperCAmelCase							(						self							:					Optional[int])		->     Mapping[str, Mapping[int, str]]:


   """simple docstring"""







   if self.task == "multiple-choice":
     _UpperCamelCase				=  {0: "batch", 1: "choice", 2: "sequence"}
   else:
     _UpperCamelCase				=  {0: "batch", 1: "sequence"}
   return OrderedDict(
       [
           ("input_ids", dynamic_axis),
           ("attention_mask", dynamic_axis),
           ("token_type_ids", dynamic_axis),
           ("task_type_ids", dynamic_axis),
       ])




