"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


SCREAMING_SNAKE_CASE__      						=  logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__      						=  {"vocab_file": "vocab.txt"}

SCREAMING_SNAKE_CASE__      						=  {
    "vocab_file": {
        "YituTech/conv-bert-base": "https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt",
        "YituTech/conv-bert-medium-small": (
            "https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt"
        ),
        "YituTech/conv-bert-small": "https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt",
    }
}

SCREAMING_SNAKE_CASE__      						=  {
    "YituTech/conv-bert-base": 512,
    "YituTech/conv-bert-medium-small": 512,
    "YituTech/conv-bert-small": 512,
}


SCREAMING_SNAKE_CASE__      						=  {
    "YituTech/conv-bert-base": {"do_lower_case": True},
    "YituTech/conv-bert-medium-small": {"do_lower_case": True},
    "YituTech/conv-bert-small": {"do_lower_case": True},
}



class   lowerCAmelCase_    (			lowerCAmelCase					):



    """simple docstring"""
    _lowerCAmelCase     :    List[str]			  =						VOCAB_FILES_NAMES
    _lowerCAmelCase     :    Optional[Any]			  =						PRETRAINED_VOCAB_FILES_MAP
    _lowerCAmelCase     :    Union[str, Any]			  =						PRETRAINED_INIT_CONFIGURATION
    _lowerCAmelCase     :    Optional[int]			  =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    _lowerCAmelCase     :    Optional[int]			  =						ConvBertTokenizer

    def __init__(				self		,    lowerCAmelCase=None		,    lowerCAmelCase=None		,    lowerCAmelCase=True		,    lowerCAmelCase="[UNK]"		,    lowerCAmelCase="[SEP]"		,    lowerCAmelCase="[PAD]"		,    lowerCAmelCase="[CLS]"		,    lowerCAmelCase="[MASK]"		,    lowerCAmelCase=True		,    lowerCAmelCase=None		,    **lowerCAmelCase		,    ):






          """simple docstring"""

          super().__init__(
              lowerCAmelCase		,    tokenizer_file=lowerCAmelCase		,    do_lower_case=lowerCAmelCase		,    unk_token=lowerCAmelCase		,    sep_token=lowerCAmelCase		,    pad_token=lowerCAmelCase		,    cls_token=lowerCAmelCase		,    mask_token=lowerCAmelCase		,    tokenize_chinese_chars=lowerCAmelCase		,    strip_accents=lowerCAmelCase		,    **lowerCAmelCase		,    )

          snake_case								=			json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
          if (
              normalizer_state.get('lowercase'		,    lowerCAmelCase    ) != do_lower_case
              or normalizer_state.get('strip_accents'		,    lowerCAmelCase    ) != strip_accents
              or normalizer_state.get('handle_chinese_chars'		,    lowerCAmelCase    ) != tokenize_chinese_chars
          ):
                snake_case								=			getattr(lowerCAmelCase		,    normalizer_state.pop('type'    )    )
                snake_case								=			do_lower_case
                snake_case								=			strip_accents
                snake_case								=			tokenize_chinese_chars
                snake_case								=			normalizer_class(**lowerCAmelCase    )

          snake_case								=			do_lower_case

    def 		snake_case (				self		,    lowerCAmelCase		,    lowerCAmelCase=None    ):






          """simple docstring"""

          snake_case								=			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
                output += token_ids_a + [self.sep_token_id]

          return output

    def 		snake_case (				self		,    lowerCAmelCase		,    lowerCAmelCase = None    ):






          """simple docstring"""

          snake_case								=			[self.sep_token_id]
          snake_case								=			[self.cls_token_id]
          if token_ids_a is None:
                return len(cls + token_ids_a + sep    ) * [0]
          return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]

    def 		snake_case (				self		,    lowerCAmelCase		,    lowerCAmelCase = None    ):






          """simple docstring"""

          snake_case								=			self._tokenizer.model.save(lowerCAmelCase		,    name=lowerCAmelCase    )
          return tuple(lowerCAmelCase    )



"""simple docstring"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE__      						=  logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__      						=  {
    "weiweishi/roc-bert-base-zh": "https://huggingface.co/weiweishi/roc-bert-base-zh/resolve/main/config.json",
}



class   lowerCAmelCase_    (			lowerCAmelCase					):



				"""simple docstring"""
				_lowerCAmelCase     :    Dict			  =						"""roc_bert"""

				def __init__(				self		,    lowerCAmelCase=3_05_22		,    lowerCAmelCase=7_68		,    lowerCAmelCase=12		,    lowerCAmelCase=12		,    lowerCAmelCase=30_72		,    lowerCAmelCase="gelu"		,    lowerCAmelCase=0.1		,    lowerCAmelCase=0.1		,    lowerCAmelCase=5_12		,    lowerCAmelCase=2		,    lowerCAmelCase=0.02		,    lowerCAmelCase=1E-12		,    lowerCAmelCase=True		,    lowerCAmelCase=0		,    lowerCAmelCase="absolute"		,    lowerCAmelCase=None		,    lowerCAmelCase=True		,    lowerCAmelCase=True		,    lowerCAmelCase=7_68		,    lowerCAmelCase=9_10		,    lowerCAmelCase=5_12		,    lowerCAmelCase=2_48_58		,    lowerCAmelCase=True		,    **lowerCAmelCase		,    ):






										"""simple docstring"""

										snake_case								=			vocab_size
										snake_case								=			max_position_embeddings
										snake_case								=			hidden_size
										snake_case								=			num_hidden_layers
										snake_case								=			num_attention_heads
										snake_case								=			intermediate_size
										snake_case								=			hidden_act
										snake_case								=			hidden_dropout_prob
										snake_case								=			attention_probs_dropout_prob
										snake_case								=			initializer_range
										snake_case								=			type_vocab_size
										snake_case								=			layer_norm_eps
										snake_case								=			use_cache
										snake_case								=			enable_pronunciation
										snake_case								=			enable_shape
										snake_case								=			pronunciation_embed_dim
										snake_case								=			pronunciation_vocab_size
										snake_case								=			shape_embed_dim
										snake_case								=			shape_vocab_size
										snake_case								=			concat_input
										snake_case								=			position_embedding_type
										snake_case								=			classifier_dropout
										super().__init__(pad_token_id=lowerCAmelCase		,    **lowerCAmelCase    )


