






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


_a    :   Optional[int]        =			logging.get_logger(__name__)

_a    :   int        =			{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_a    :   Optional[Any]        =			{
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

_a    :   Dict        =			{
    """google/realm-cc-news-pretrained-embedder""": 512,
    """google/realm-cc-news-pretrained-encoder""": 512,
    """google/realm-cc-news-pretrained-scorer""": 512,
    """google/realm-cc-news-pretrained-openqa""": 512,
    """google/realm-orqa-nq-openqa""": 512,
    """google/realm-orqa-nq-reader""": 512,
    """google/realm-orqa-wq-openqa""": 512,
    """google/realm-orqa-wq-reader""": 512,
}

_a    :   Any        =			{
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}







class 	_UpperCAmelCase  (  A_		):




      """simple docstring"""



      A          =    VOCAB_FILES_NAMES
      A          =    PRETRAINED_VOCAB_FILES_MAP
      A          =    PRETRAINED_INIT_CONFIGURATION
      A          =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      A          =    RealmTokenizer
      def __init__(   self    ,	_lowerCAmelCase=None    ,	_lowerCAmelCase=None    ,	_lowerCAmelCase=True    ,	_lowerCAmelCase="[UNK]"    ,	_lowerCAmelCase="[SEP]"    ,	_lowerCAmelCase="[PAD]"    ,	_lowerCAmelCase="[CLS]"    ,	_lowerCAmelCase="[MASK]"    ,	_lowerCAmelCase=True    ,	_lowerCAmelCase=None    ,	**_lowerCAmelCase    ,	):



          '''simple docstring'''
          super().__init__(
              _lowerCAmelCase    ,	tokenizer_file=_lowerCAmelCase    ,	do_lower_case=_lowerCAmelCase    ,	unk_token=_lowerCAmelCase    ,	sep_token=_lowerCAmelCase    ,	pad_token=_lowerCAmelCase    ,	cls_token=_lowerCAmelCase    ,	mask_token=_lowerCAmelCase    ,	tokenize_chinese_chars=_lowerCAmelCase    ,	strip_accents=_lowerCAmelCase    ,	**_lowerCAmelCase    ,	)

          lowerCAmelCase__   :List[str]      							=				json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
          if (
              normalizer_state.get("lowercase"    ,	_lowerCAmelCase		) != do_lower_case
              or normalizer_state.get("strip_accents"    ,	_lowerCAmelCase		) != strip_accents
              or normalizer_state.get("handle_chinese_chars"    ,	_lowerCAmelCase		) != tokenize_chinese_chars
          ):
              lowerCAmelCase__   :int      							=				getattr(_lowerCAmelCase    ,	normalizer_state.pop("type"		)		)
              lowerCAmelCase__   :Optional[Any]      							=				do_lower_case
              lowerCAmelCase__   :List[str]      							=				strip_accents
              lowerCAmelCase__   :Dict      							=				tokenize_chinese_chars
              lowerCAmelCase__   :Optional[Any]      							=				normalizer_class(**_lowerCAmelCase		)

          lowerCAmelCase__   :Any      							=				do_lower_case
      def 			snake_case_      (   self    ,	_lowerCAmelCase    ,	**_lowerCAmelCase		):



          '''simple docstring'''
          lowerCAmelCase__   :int      							=				PaddingStrategy.MAX_LENGTH

          lowerCAmelCase__   :Optional[int]      							=				text
          lowerCAmelCase__   :Tuple      							=				kwargs.pop("text_pair"    ,	_lowerCAmelCase		)
          lowerCAmelCase__   :List[Any]      							=				kwargs.pop("return_tensors"    ,	_lowerCAmelCase		)

          lowerCAmelCase__   :Tuple      							=				{
              """input_ids""": [],
              """attention_mask""": [],
              """token_type_ids""": [],
          }

          for idx, candidate_text in enumerate(_lowerCAmelCase		):
              if batch_text_pair is not None:
                  lowerCAmelCase__   :List[Any]      							=				batch_text_pair[idx]
              else:
                  lowerCAmelCase__   :Any      							=				None

              lowerCAmelCase__   :List[str]      							=				super().__call__(_lowerCAmelCase    ,	_lowerCAmelCase    ,	return_tensors=_lowerCAmelCase    ,	**_lowerCAmelCase		)

              lowerCAmelCase__   :str      							=				encoded_candidates.get("input_ids"		)
              lowerCAmelCase__   :Union[str, Any]      							=				encoded_candidates.get("attention_mask"		)
              lowerCAmelCase__   :int      							=				encoded_candidates.get("token_type_ids"		)

              if encoded_input_ids is not None:
                  output_data["input_ids"].append(_lowerCAmelCase		)
              if encoded_attention_mask is not None:
                  output_data["attention_mask"].append(_lowerCAmelCase		)
              if encoded_token_type_ids is not None:
                  output_data["token_type_ids"].append(_lowerCAmelCase		)

          lowerCAmelCase__   :List[str]      							=				{key: item for key, item in output_data.items() if len(_lowerCAmelCase		) != 0}

          return BatchEncoding(_lowerCAmelCase    ,	tensor_type=_lowerCAmelCase		)
      def 			snake_case_      (   self    ,	_lowerCAmelCase    ,	_lowerCAmelCase=None		):



          '''simple docstring'''
          lowerCAmelCase__   :Dict      							=				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

          return output
      def 			snake_case_      (   self    ,	_lowerCAmelCase    ,	_lowerCAmelCase = None		):



          '''simple docstring'''
          lowerCAmelCase__   :List[str]      							=				[self.sep_token_id]
          lowerCAmelCase__   :Union[str, Any]      							=				[self.cls_token_id]
          if token_ids_a is None:
              return len(cls + token_ids_a + sep		) * [0]
          return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]



      def 			snake_case_      (   self    ,	_lowerCAmelCase    ,	_lowerCAmelCase = None		):



          '''simple docstring'''
          lowerCAmelCase__   :Any      							=				self._tokenizer.model.save(_lowerCAmelCase    ,	name=_lowerCAmelCase		)
          return tuple(_lowerCAmelCase		)









def 				snake_case__							(							UpperCAmelCase							:  float       ):

    if edge <= 0 or not isinstance(UpperCAmelCase    ,			UpperCAmelCase       ):
        raise ValueError("Length must be a positive."       )
    return 3 * ((2_5 + 1_0 * (5 ** (1 / 2))) ** (1 / 2)) * (edge**2)


def 				snake_case__							(							UpperCAmelCase							:  float       ):

    if edge <= 0 or not isinstance(UpperCAmelCase    ,			UpperCAmelCase       ):
        raise ValueError("Length must be a positive."       )
    return ((1_5 + (7 * (5 ** (1 / 2)))) / 4) * (edge**3)


if __name__ == "__main__":
 import doctest

 doctest.testmod()



