





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


A      = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

A      = {
    "vocab_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"
        ),
        "google/electra-base-generator": "https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt",
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-generator": (
            "https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"
        ),
    },
}

A      = {
    "google/electra-small-generator": 512,
    "google/electra-base-generator": 512,
    "google/electra-large-generator": 512,
    "google/electra-small-discriminator": 512,
    "google/electra-base-discriminator": 512,
    "google/electra-large-discriminator": 512,
}

A      = {
    "google/electra-small-generator": {"do_lower_case": True},
    "google/electra-base-generator": {"do_lower_case": True},
    "google/electra-large-generator": {"do_lower_case": True},
    "google/electra-small-discriminator": {"do_lower_case": True},
    "google/electra-base-discriminator": {"do_lower_case": True},
    "google/electra-large-discriminator": {"do_lower_case": True},
}


class     lowercase__  (					__SCREAMING_SNAKE_CASE		):
  A__=  VOCAB_FILES_NAMES
  A__=  PRETRAINED_VOCAB_FILES_MAP
  A__=  PRETRAINED_INIT_CONFIGURATION
  A__=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  A__=  ElectraTokenizer






  def __init__(					self      :					Optional[int]		,						_lowercase      :					Tuple=None		,						_lowercase      :					int=None		,						_lowercase      :					Union[str, Any]=True		,						_lowercase      :					int="[UNK]"		,						_lowercase      :					List[str]="[SEP]"		,						_lowercase      :					Tuple="[PAD]"		,						_lowercase      :					Optional[int]="[CLS]"		,						_lowercase      :					List[Any]="[MASK]"		,						_lowercase      :					Optional[int]=True		,						_lowercase      :					Dict=None		,						**_lowercase      :					Any		,						):




         """simple docstring"""







         super().__init__(
             _lowercase		,						tokenizer_file=_lowercase		,						do_lower_case=_lowercase		,						unk_token=_lowercase		,						sep_token=_lowercase		,						pad_token=_lowercase		,						cls_token=_lowercase		,						mask_token=_lowercase		,						tokenize_chinese_chars=_lowercase		,						strip_accents=_lowercase		,						**_lowercase		,						)

         UpperCAmelCase__										=     json.loads(self.backend_tokenizer.normalizer.__getstate__() )
         if (
             normalizer_state.get("lowercase"		,						_lowercase ) != do_lower_case
             or normalizer_state.get("strip_accents"		,						_lowercase ) != strip_accents
             or normalizer_state.get("handle_chinese_chars"		,						_lowercase ) != tokenize_chinese_chars
         ):
                UpperCAmelCase__										=     getattr(_lowercase		,						normalizer_state.pop("type" ) )
                UpperCAmelCase__										=     do_lower_case
                UpperCAmelCase__										=     strip_accents
                UpperCAmelCase__										=     tokenize_chinese_chars
                UpperCAmelCase__										=     normalizer_class(**_lowercase )

         UpperCAmelCase__										=     do_lower_case






  def 			_UpperCAmelCase  (					self      :					List[str]		,						_lowercase      :					Optional[int]		,						_lowercase      :					Optional[Any]=None ):




         """simple docstring"""







         UpperCAmelCase__										=     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
                output += token_ids_a + [self.sep_token_id]

         return output






  def 			_UpperCAmelCase  (					self      :					Optional[Any]		,						_lowercase      :					List[int]		,						_lowercase      :					Optional[List[int]] = None ):




         """simple docstring"""







         UpperCAmelCase__										=     [self.sep_token_id]
         UpperCAmelCase__										=     [self.cls_token_id]
         if token_ids_a is None:
                return len(cls + token_ids_a + sep ) * [0]
         return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]






  def 			_UpperCAmelCase  (					self      :					List[str]		,						_lowercase      :					str		,						_lowercase      :					Optional[str] = None ):




         """simple docstring"""







         UpperCAmelCase__										=     self._tokenizer.model.save(_lowercase		,						name=_lowercase )
         return tuple(_lowercase )









class     lowercase__  :






  def __init__(					self      :					List[str]		,						_lowercase      :					list ):




         """simple docstring"""







         UpperCAmelCase__										=     set_counts
         UpperCAmelCase__										=     max(_lowercase )
         UpperCAmelCase__										=     len(_lowercase )
         UpperCAmelCase__										=     [1] * num_sets
         UpperCAmelCase__										=     list(range(_lowercase ) )






  def 			_UpperCAmelCase  (					self      :					Dict		,						_lowercase      :					int		,						_lowercase      :					int ):




         """simple docstring"""







         UpperCAmelCase__										=     self.get_parent(_lowercase )
         UpperCAmelCase__										=     self.get_parent(_lowercase )

         if src_parent == dst_parent:
                return False

         if self.ranks[dst_parent] >= self.ranks[src_parent]:
                self.set_counts[dst_parent] += self.set_counts[src_parent]
                UpperCAmelCase__										=     0
                UpperCAmelCase__										=     dst_parent
                if self.ranks[dst_parent] == self.ranks[src_parent]:
                       self.ranks[dst_parent] += 1
                UpperCAmelCase__										=     self.set_counts[dst_parent]
         else:
                self.set_counts[src_parent] += self.set_counts[dst_parent]
                UpperCAmelCase__										=     0
                UpperCAmelCase__										=     src_parent
                UpperCAmelCase__										=     self.set_counts[src_parent]

         UpperCAmelCase__										=     max(self.max_set		,						_lowercase )
         return True






  def 			_UpperCAmelCase  (					self      :					Optional[Any]		,						_lowercase      :					int ):




         """simple docstring"""







         if self.parents[disj_set] == disj_set:
                return disj_set
         UpperCAmelCase__										=     self.get_parent(self.parents[disj_set] )
         return self.parents[disj_set]





