



"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


lowercase__		:    Tuple				   =     logging.get_logger(__name__)

lowercase__		:    Tuple				   =     {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowercase__		:    Union[str, Any]				   =     {
    """vocab_file""": {
        """bert-base-uncased""": """https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt""",
        """bert-large-uncased""": """https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt""",
        """bert-base-cased""": """https://huggingface.co/bert-base-cased/resolve/main/vocab.txt""",
        """bert-large-cased""": """https://huggingface.co/bert-large-cased/resolve/main/vocab.txt""",
        """bert-base-multilingual-uncased""": (
            """https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"""
        ),
        """bert-base-multilingual-cased""": """https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt""",
        """bert-base-chinese""": """https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt""",
        """bert-base-german-cased""": """https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt""",
        """bert-large-uncased-whole-word-masking""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"""
        ),
        """bert-large-cased-whole-word-masking""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"""
        ),
        """bert-large-uncased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"""
        ),
        """bert-large-cased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"""
        ),
        """bert-base-cased-finetuned-mrpc""": (
            """https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"""
        ),
        """bert-base-german-dbmdz-cased""": """https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt""",
        """bert-base-german-dbmdz-uncased""": (
            """https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"""
        ),
        """TurkuNLP/bert-base-finnish-cased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"""
        ),
        """TurkuNLP/bert-base-finnish-uncased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"""
        ),
        """wietsedv/bert-base-dutch-cased""": (
            """https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """bert-base-uncased""": """https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json""",
        """bert-large-uncased""": """https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json""",
        """bert-base-cased""": """https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json""",
        """bert-large-cased""": """https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json""",
        """bert-base-multilingual-uncased""": (
            """https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"""
        ),
        """bert-base-multilingual-cased""": (
            """https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"""
        ),
        """bert-base-chinese""": """https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json""",
        """bert-base-german-cased""": """https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json""",
        """bert-large-uncased-whole-word-masking""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"""
        ),
        """bert-large-cased-whole-word-masking""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"""
        ),
        """bert-large-uncased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"""
        ),
        """bert-large-cased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"""
        ),
        """bert-base-cased-finetuned-mrpc""": (
            """https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"""
        ),
        """bert-base-german-dbmdz-cased""": (
            """https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"""
        ),
        """bert-base-german-dbmdz-uncased""": (
            """https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"""
        ),
        """TurkuNLP/bert-base-finnish-cased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"""
        ),
        """TurkuNLP/bert-base-finnish-uncased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"""
        ),
        """wietsedv/bert-base-dutch-cased""": (
            """https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"""
        ),
    },
}

lowercase__		:    Optional[Any]				   =     {
    """bert-base-uncased""": 5_1_2,
    """bert-large-uncased""": 5_1_2,
    """bert-base-cased""": 5_1_2,
    """bert-large-cased""": 5_1_2,
    """bert-base-multilingual-uncased""": 5_1_2,
    """bert-base-multilingual-cased""": 5_1_2,
    """bert-base-chinese""": 5_1_2,
    """bert-base-german-cased""": 5_1_2,
    """bert-large-uncased-whole-word-masking""": 5_1_2,
    """bert-large-cased-whole-word-masking""": 5_1_2,
    """bert-large-uncased-whole-word-masking-finetuned-squad""": 5_1_2,
    """bert-large-cased-whole-word-masking-finetuned-squad""": 5_1_2,
    """bert-base-cased-finetuned-mrpc""": 5_1_2,
    """bert-base-german-dbmdz-cased""": 5_1_2,
    """bert-base-german-dbmdz-uncased""": 5_1_2,
    """TurkuNLP/bert-base-finnish-cased-v1""": 5_1_2,
    """TurkuNLP/bert-base-finnish-uncased-v1""": 5_1_2,
    """wietsedv/bert-base-dutch-cased""": 5_1_2,
}

lowercase__		:    List[Any]				   =     {
    """bert-base-uncased""": {"""do_lower_case""": True},
    """bert-large-uncased""": {"""do_lower_case""": True},
    """bert-base-cased""": {"""do_lower_case""": False},
    """bert-large-cased""": {"""do_lower_case""": False},
    """bert-base-multilingual-uncased""": {"""do_lower_case""": True},
    """bert-base-multilingual-cased""": {"""do_lower_case""": False},
    """bert-base-chinese""": {"""do_lower_case""": False},
    """bert-base-german-cased""": {"""do_lower_case""": False},
    """bert-large-uncased-whole-word-masking""": {"""do_lower_case""": True},
    """bert-large-cased-whole-word-masking""": {"""do_lower_case""": False},
    """bert-large-uncased-whole-word-masking-finetuned-squad""": {"""do_lower_case""": True},
    """bert-large-cased-whole-word-masking-finetuned-squad""": {"""do_lower_case""": False},
    """bert-base-cased-finetuned-mrpc""": {"""do_lower_case""": False},
    """bert-base-german-dbmdz-cased""": {"""do_lower_case""": False},
    """bert-base-german-dbmdz-uncased""": {"""do_lower_case""": True},
    """TurkuNLP/bert-base-finnish-cased-v1""": {"""do_lower_case""": False},
    """TurkuNLP/bert-base-finnish-uncased-v1""": {"""do_lower_case""": True},
    """wietsedv/bert-base-dutch-cased""": {"""do_lower_case""": False},
}







class 				UpperCamelCase__       (     lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE      =					VOCAB_FILES_NAMES
      _SCREAMING_SNAKE_CASE      =					PRETRAINED_VOCAB_FILES_MAP
      _SCREAMING_SNAKE_CASE      =					PRETRAINED_INIT_CONFIGURATION
      _SCREAMING_SNAKE_CASE      =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _SCREAMING_SNAKE_CASE      =					BertTokenizer



      def __init__(				self    :				Optional[int]   ,      SCREAMING_SNAKE_CASE_    :				Tuple=None   ,      SCREAMING_SNAKE_CASE_    :				str=None   ,      SCREAMING_SNAKE_CASE_    :				List[Any]=True   ,      SCREAMING_SNAKE_CASE_    :				int="[UNK]"   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]="[SEP]"   ,      SCREAMING_SNAKE_CASE_    :				Optional[int]="[PAD]"   ,      SCREAMING_SNAKE_CASE_    :				Tuple="[CLS]"   ,      SCREAMING_SNAKE_CASE_    :				List[str]="[MASK]"   ,      SCREAMING_SNAKE_CASE_    :				List[str]=True   ,      SCREAMING_SNAKE_CASE_    :				Any=None   ,      **SCREAMING_SNAKE_CASE_    :				Dict   ,      ):
        super().__init__(
            _SCREAMING_SNAKE_CASE   ,      tokenizer_file=_SCREAMING_SNAKE_CASE   ,      do_lower_case=_SCREAMING_SNAKE_CASE   ,      unk_token=_SCREAMING_SNAKE_CASE   ,      sep_token=_SCREAMING_SNAKE_CASE   ,      pad_token=_SCREAMING_SNAKE_CASE   ,      cls_token=_SCREAMING_SNAKE_CASE   ,      mask_token=_SCREAMING_SNAKE_CASE   ,      tokenize_chinese_chars=_SCREAMING_SNAKE_CASE   ,      strip_accents=_SCREAMING_SNAKE_CASE   ,      **_SCREAMING_SNAKE_CASE   ,      )

        lowerCAmelCase_       :    Union[str, Any]								=			json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
        if (
            normalizer_state.get('lowercase'   ,      _SCREAMING_SNAKE_CASE	) != do_lower_case
            or normalizer_state.get('strip_accents'   ,      _SCREAMING_SNAKE_CASE	) != strip_accents
            or normalizer_state.get('handle_chinese_chars'   ,      _SCREAMING_SNAKE_CASE	) != tokenize_chinese_chars
        ):
          lowerCAmelCase_       :    Tuple								=			getattr(_SCREAMING_SNAKE_CASE   ,      normalizer_state.pop('type'	)	)
          lowerCAmelCase_       :    List[Any]								=			do_lower_case
          lowerCAmelCase_       :    Tuple								=			strip_accents
          lowerCAmelCase_       :    Union[str, Any]								=			tokenize_chinese_chars
          lowerCAmelCase_       :    List[str]								=			normalizer_class(**_SCREAMING_SNAKE_CASE	)

        lowerCAmelCase_       :    int								=			do_lower_case



      def 					SCREAMING_SNAKE_CASE__			(				self    :				str   ,      SCREAMING_SNAKE_CASE_    :				List[str]   ,      SCREAMING_SNAKE_CASE_    :				str=None	):
        lowerCAmelCase_       :    Tuple								=			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
          output += token_ids_a + [self.sep_token_id]

        return output



      def 					SCREAMING_SNAKE_CASE__			(				self    :				List[str]   ,      SCREAMING_SNAKE_CASE_    :				Dict   ,      SCREAMING_SNAKE_CASE_    :				Any = None	):
        lowerCAmelCase_       :    List[Any]								=			[self.sep_token_id]
        lowerCAmelCase_       :    int								=			[self.cls_token_id]
        if token_ids_a is None:
          return len(cls + token_ids_a + sep	) * [0]
        return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]





      def 					SCREAMING_SNAKE_CASE__			(				self    :				Union[str, Any]   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]   ,      SCREAMING_SNAKE_CASE_    :				List[str] = None	):
        lowerCAmelCase_       :    int								=			self._tokenizer.model.save(_SCREAMING_SNAKE_CASE   ,      name=_SCREAMING_SNAKE_CASE	)
        return tuple(_SCREAMING_SNAKE_CASE	)










"""simple docstring"""







from collections.abc import Iterable
from typing import Generic, TypeVar

lowercase__		:    Any				   =     TypeVar("""_T""")







class 				UpperCamelCase__       (     Generic[_T]  ):




      """simple docstring"""




      def __init__(				self    :				Any   ,      SCREAMING_SNAKE_CASE_    :				Iterable[_T] | None = None	):
        lowerCAmelCase_       :    list[_T]								=			list(iterable or []	)
        lowerCAmelCase_       :    list[_T]								=			[]



      def __len__(				self    :				List[str]	):
        return len(self._stacka	) + len(self._stacka	)



      def __repr__(				self    :				int	):
        return F"Queue({tuple(self._stacka[::-1] + self._stacka	)})"



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Tuple   ,      SCREAMING_SNAKE_CASE_    :				_T	):
        self._stacka.append(SCREAMING_SNAKE_CASE_	)





      def 					SCREAMING_SNAKE_CASE__			(				self    :				str	):
        lowerCAmelCase_       :    List[str]								=			self._stacka.pop
        lowerCAmelCase_       :    int								=			self._stacka.append

        if not self._stacka:
          while self._stacka:
            stacka_append(stacka_pop()	)

        if not self._stacka:
          raise IndexError('Queue is empty'	)
        return self._stacka.pop()


if __name__ == "__main__":
     from doctest import testmod

     testmod()





