






'''simple docstring'''





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


__lowerCamelCase						:    Union[str, Any]     				=  {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__lowerCamelCase						:    Any     				=  {
    "vocab_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"
        ),
        "google/electra-base-generator": "https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt",
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-generator": (
            "https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"
        ),
    },
}

__lowerCamelCase						:    Optional[Any]     				=  {
    "google/electra-small-generator": 512,
    "google/electra-base-generator": 512,
    "google/electra-large-generator": 512,
    "google/electra-small-discriminator": 512,
    "google/electra-base-discriminator": 512,
    "google/electra-large-discriminator": 512,
}

__lowerCamelCase						:    Any     				=  {
    "google/electra-small-generator": {"do_lower_case": True},
    "google/electra-base-generator": {"do_lower_case": True},
    "google/electra-large-generator": {"do_lower_case": True},
    "google/electra-small-discriminator": {"do_lower_case": True},
    "google/electra-base-discriminator": {"do_lower_case": True},
    "google/electra-large-discriminator": {"do_lower_case": True},
}



class   lowerCAmelCase__ ( UpperCamelCase__  ):
			A     				= VOCAB_FILES_NAMES
			A     				= PRETRAINED_VOCAB_FILES_MAP
			A     				= PRETRAINED_INIT_CONFIGURATION
			A     				= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			A     				= ElectraTokenizer

			def __init__(					self					:		Tuple     ,		UpperCamelCase_					:		List[Any]=None     ,		UpperCamelCase_					:		Dict=None     ,		UpperCamelCase_					:		Tuple=True     ,		UpperCamelCase_					:		Optional[Any]="[UNK]"     ,		UpperCamelCase_					:		List[str]="[SEP]"     ,		UpperCamelCase_					:		Union[str, Any]="[PAD]"     ,		UpperCamelCase_					:		Any="[CLS]"     ,		UpperCamelCase_					:		Any="[MASK]"     ,		UpperCamelCase_					:		List[Any]=True     ,		UpperCamelCase_					:		int=None     ,		**UpperCamelCase_					:		Any     ,		) ->  str:







				"""simple docstring"""






				super().__init__(
				    lowercase_     ,		tokenizer_file=lowercase_     ,		do_lower_case=lowercase_     ,		unk_token=lowercase_     ,		sep_token=lowercase_     ,		pad_token=lowercase_     ,		cls_token=lowercase_     ,		mask_token=lowercase_     ,		tokenize_chinese_chars=lowercase_     ,		strip_accents=lowercase_     ,		**lowercase_     ,		)

				lowerCamelCase_					:  Optional[int]						=    json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
				if (
				    normalizer_state.get('''lowercase'''     ,		lowercase_	) != do_lower_case
				    or normalizer_state.get('''strip_accents'''     ,		lowercase_	) != strip_accents
				    or normalizer_state.get('''handle_chinese_chars'''     ,		lowercase_	) != tokenize_chinese_chars
				):
					lowerCamelCase_					:  Optional[Any]						=    getattr(lowercase_     ,		normalizer_state.pop('''type'''	)	)
					lowerCamelCase_					:  int						=    do_lower_case
					lowerCamelCase_					:  Optional[Any]						=    strip_accents
					lowerCamelCase_					:  List[Any]						=    tokenize_chinese_chars
					lowerCamelCase_					:  int						=    normalizer_class(**lowercase_	)

				lowerCamelCase_					:  Tuple						=    do_lower_case

			def 						__UpperCamelCase				(					self					:		List[Any]     ,		UpperCamelCase_					:		Tuple     ,		UpperCamelCase_					:		Any=None	) ->  List[Any]:







				"""simple docstring"""






				lowerCamelCase_					:  int						=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
					output += token_ids_a + [self.sep_token_id]

				return output

			def 						__UpperCamelCase				(					self					:		Dict     ,		UpperCamelCase_					:		List[int]     ,		UpperCamelCase_					:		Optional[List[int]] = None	) ->  List[int]:







				"""simple docstring"""






				lowerCamelCase_					:  Union[str, Any]						=    [self.sep_token_id]
				lowerCamelCase_					:  Any						=    [self.cls_token_id]
				if token_ids_a is None:
					return len(cls + token_ids_a + sep	) * [0]
				return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]





			def 						__UpperCamelCase				(					self					:		List[Any]     ,		UpperCamelCase_					:		str     ,		UpperCamelCase_					:		Optional[str] = None	) ->  Tuple[str]:







				"""simple docstring"""






				lowerCamelCase_					:  List[Any]						=    self._tokenizer.model.save(lowercase_     ,		name=lowercase_	)
				return tuple(lowercase_	)













'''simple docstring'''

# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__lowerCAmelCase				:      Union[str, Any]						      ={"configuration_mra": ["MRA_PRETRAINED_CONFIG_ARCHIVE_MAP", "MraConfig"]}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __lowerCAmelCase				:      int						      =[
      "MRA_PRETRAINED_MODEL_ARCHIVE_LIST",
      "MraForMaskedLM",
      "MraForMultipleChoice",
      "MraForQuestionAnswering",
      "MraForSequenceClassification",
      "MraForTokenClassification",
      "MraLayer",
      "MraModel",
      "MraPreTrainedModel",
  ]


if TYPE_CHECKING:
  from .configuration_mra import MRA_PRETRAINED_CONFIG_ARCHIVE_MAP, MraConfig

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_mra import (
        MRA_PRETRAINED_MODEL_ARCHIVE_LIST,
        MraForMaskedLM,
        MraForMultipleChoice,
        MraForQuestionAnswering,
        MraForSequenceClassification,
        MraForTokenClassification,
        MraLayer,
        MraModel,
        MraPreTrainedModel,
    )
else:
  import sys

  __lowerCAmelCase				:      Dict						      =_LazyModule(__name__, globals()["__file__"], _import_structure)



