





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


_UpperCAmelCase       : str			     =       logging.get_logger(__name__)

_UpperCAmelCase       : Optional[Any]			     =       {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_UpperCAmelCase       : Dict			     =       {
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

_UpperCAmelCase       : Tuple			     =       {
    """google/realm-cc-news-pretrained-embedder""": 5_12,
    """google/realm-cc-news-pretrained-encoder""": 5_12,
    """google/realm-cc-news-pretrained-scorer""": 5_12,
    """google/realm-cc-news-pretrained-openqa""": 5_12,
    """google/realm-orqa-nq-openqa""": 5_12,
    """google/realm-orqa-nq-reader""": 5_12,
    """google/realm-orqa-wq-openqa""": 5_12,
    """google/realm-orqa-wq-reader""": 5_12,
}

_UpperCAmelCase       : str			     =       {
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}

class      lowerCAmelCase							(      __UpperCamelCase    ):
 UpperCAmelCase__									=	VOCAB_FILES_NAMES
 UpperCAmelCase__									=	PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase__									=	PRETRAINED_INIT_CONFIGURATION
 UpperCAmelCase__									=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase__									=	RealmTokenizer
 def __init__(   self						:   Union[str, Any]    ,  UpperCAmelCase						:   int=None    ,  UpperCAmelCase						:   Dict=None    ,  UpperCAmelCase						:   Tuple=True    ,  UpperCAmelCase						:   Any="[UNK]"    ,  UpperCAmelCase						:   int="[SEP]"    ,  UpperCAmelCase						:   int="[PAD]"    ,  UpperCAmelCase						:   Dict="[CLS]"    ,  UpperCAmelCase						:   List[str]="[MASK]"    ,  UpperCAmelCase						:   Tuple=True    ,  UpperCAmelCase						:   Union[str, Any]=None    ,  **UpperCAmelCase						:   Optional[int]    ,  )    ->			Optional[int]:
     super().__init__(
         UpperCAmelCase    ,  tokenizer_file=UpperCAmelCase    ,  do_lower_case=UpperCAmelCase    ,  unk_token=UpperCAmelCase    ,  sep_token=UpperCAmelCase    ,  pad_token=UpperCAmelCase    ,  cls_token=UpperCAmelCase    ,  mask_token=UpperCAmelCase    ,  tokenize_chinese_chars=UpperCAmelCase    ,  strip_accents=UpperCAmelCase    ,  **UpperCAmelCase    ,  )

     lowerCamelCase__					:   Tuple      				=	json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
     if (
         normalizer_state.get('lowercase'    ,  UpperCAmelCase			) != do_lower_case
         or normalizer_state.get('strip_accents'    ,  UpperCAmelCase			) != strip_accents
         or normalizer_state.get('handle_chinese_chars'    ,  UpperCAmelCase			) != tokenize_chinese_chars
     ):
         lowerCamelCase__					:   str      				=	getattr(UpperCAmelCase    ,  normalizer_state.pop('type'			)			)
         lowerCamelCase__					:   Any      				=	do_lower_case
         lowerCamelCase__					:   Optional[Any]      				=	strip_accents
         lowerCamelCase__					:   List[str]      				=	tokenize_chinese_chars
         lowerCamelCase__					:   Union[str, Any]      				=	normalizer_class(**UpperCAmelCase			)

     lowerCamelCase__					:   Optional[Any]      				=	do_lower_case
 def   A_			(   self						:   List[str]    ,  UpperCAmelCase						:   Optional[int]    ,  **UpperCAmelCase						:   int			)    ->			Any:
     lowerCamelCase__					:   str      				=	PaddingStrategy.MAX_LENGTH

     lowerCamelCase__					:   List[str]      				=	text
     lowerCamelCase__					:   Optional[int]      				=	kwargs.pop('text_pair'    ,  UpperCAmelCase			)
     lowerCamelCase__					:   Any      				=	kwargs.pop('return_tensors'    ,  UpperCAmelCase			)

     lowerCamelCase__					:   List[str]      				=	{
         'input_ids': [],
         'attention_mask': [],
         'token_type_ids': [],
     }

     for idx, candidate_text in enumerate(UpperCAmelCase			):
         if batch_text_pair is not None:
             lowerCamelCase__					:   Any      				=	batch_text_pair[idx]
         else:
             lowerCamelCase__					:   List[Any]      				=	None

         lowerCamelCase__					:   Optional[int]      				=	super().__call__(UpperCAmelCase    ,  UpperCAmelCase    ,  return_tensors=UpperCAmelCase    ,  **UpperCAmelCase			)

         lowerCamelCase__					:   Optional[int]      				=	encoded_candidates.get('input_ids'			)
         lowerCamelCase__					:   Any      				=	encoded_candidates.get('attention_mask'			)
         lowerCamelCase__					:   str      				=	encoded_candidates.get('token_type_ids'			)

         if encoded_input_ids is not None:
             output_data["input_ids"].append(UpperCAmelCase			)
         if encoded_attention_mask is not None:
             output_data["attention_mask"].append(UpperCAmelCase			)
         if encoded_token_type_ids is not None:
             output_data["token_type_ids"].append(UpperCAmelCase			)

     lowerCamelCase__					:   int      				=	{key: item for key, item in output_data.items() if len(UpperCAmelCase			) != 0}

     return BatchEncoding(UpperCAmelCase    ,  tensor_type=UpperCAmelCase			)
 def   A_			(   self						:   Any    ,  UpperCAmelCase						:   int    ,  UpperCAmelCase						:   Any=None			)    ->			Any:
     lowerCamelCase__					:   List[str]      				=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

     return output
 def   A_			(   self						:   Any    ,  UpperCAmelCase						:   List[int]    ,  UpperCAmelCase						:   Optional[List[int]] = None			)    ->			List[int]:
     lowerCamelCase__					:   str      				=	[self.sep_token_id]
     lowerCamelCase__					:   Tuple      				=	[self.cls_token_id]
     if token_ids_a is None:
         return len(cls + token_ids_a + sep			) * [0]
     return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]







 def   A_			(   self						:   Any    ,  UpperCAmelCase						:   str    ,  UpperCAmelCase						:   Optional[str] = None			)    ->			Tuple[str]:
     lowerCamelCase__					:   List[str]      				=	self._tokenizer.model.save(UpperCAmelCase    ,  name=UpperCAmelCase			)
     return tuple(UpperCAmelCase			)












from ..utils import DummyObject, requires_backends

class      lowerCAmelCase							(      metaclass=__UpperCamelCase    ):
 UpperCAmelCase__									=	["""torch""", """transformers""", """onnx"""]
 def __init__(   self						:   Tuple    ,  *UpperCAmelCase						:   str    ,  **UpperCAmelCase						:   Optional[int]			)    ->			List[str]:
     requires_backends(self    ,  ['torch', 'transformers', 'onnx']			)
 @classmethod
 def   A_			(   cls						:   Union[str, Any]    ,  *UpperCAmelCase						:   Dict    ,  **UpperCAmelCase						:   List[str]			)    ->			int:
     requires_backends(cls    ,  ['torch', 'transformers', 'onnx']			)







 @classmethod
 def   A_			(   cls						:   Any    ,  *UpperCAmelCase						:   List[Any]    ,  **UpperCAmelCase						:   Tuple			)    ->			Union[str, Any]:
     requires_backends(cls    ,  ['torch', 'transformers', 'onnx']			)


class      lowerCAmelCase							(      metaclass=__UpperCamelCase    ):
 UpperCAmelCase__									=	["""torch""", """transformers""", """onnx"""]
 def __init__(   self						:   List[Any]    ,  *UpperCAmelCase						:   Union[str, Any]    ,  **UpperCAmelCase						:   Dict			)    ->			Any:
     requires_backends(self    ,  ['torch', 'transformers', 'onnx']			)
 @classmethod
 def   A_			(   cls						:   str    ,  *UpperCAmelCase						:   Union[str, Any]    ,  **UpperCAmelCase						:   Union[str, Any]			)    ->			Tuple:
     requires_backends(cls    ,  ['torch', 'transformers', 'onnx']			)







 @classmethod
 def   A_			(   cls						:   List[Any]    ,  *UpperCAmelCase						:   str    ,  **UpperCAmelCase						:   Dict			)    ->			str:
     requires_backends(cls    ,  ['torch', 'transformers', 'onnx']			)


class      lowerCAmelCase							(      metaclass=__UpperCamelCase    ):
 UpperCAmelCase__									=	["""torch""", """transformers""", """onnx"""]
 def __init__(   self						:   Dict    ,  *UpperCAmelCase						:   Dict    ,  **UpperCAmelCase						:   Dict			)    ->			Optional[int]:
     requires_backends(self    ,  ['torch', 'transformers', 'onnx']			)
 @classmethod
 def   A_			(   cls						:   Optional[Any]    ,  *UpperCAmelCase						:   Union[str, Any]    ,  **UpperCAmelCase						:   Any			)    ->			Any:
     requires_backends(cls    ,  ['torch', 'transformers', 'onnx']			)







 @classmethod
 def   A_			(   cls						:   List[str]    ,  *UpperCAmelCase						:   Any    ,  **UpperCAmelCase						:   Dict			)    ->			Tuple:
     requires_backends(cls    ,  ['torch', 'transformers', 'onnx']			)


class      lowerCAmelCase							(      metaclass=__UpperCamelCase    ):
 UpperCAmelCase__									=	["""torch""", """transformers""", """onnx"""]
 def __init__(   self						:   Union[str, Any]    ,  *UpperCAmelCase						:   Optional[int]    ,  **UpperCAmelCase						:   Optional[int]			)    ->			Any:
     requires_backends(self    ,  ['torch', 'transformers', 'onnx']			)
 @classmethod
 def   A_			(   cls						:   List[Any]    ,  *UpperCAmelCase						:   Dict    ,  **UpperCAmelCase						:   int			)    ->			Dict:
     requires_backends(cls    ,  ['torch', 'transformers', 'onnx']			)







 @classmethod
 def   A_			(   cls						:   Dict    ,  *UpperCAmelCase						:   Dict    ,  **UpperCAmelCase						:   Optional[int]			)    ->			Optional[int]:
     requires_backends(cls    ,  ['torch', 'transformers', 'onnx']			)


class      lowerCAmelCase							(      metaclass=__UpperCamelCase    ):
 UpperCAmelCase__									=	["""torch""", """transformers""", """onnx"""]
 def __init__(   self						:   List[str]    ,  *UpperCAmelCase						:   List[Any]    ,  **UpperCAmelCase						:   List[str]			)    ->			Optional[int]:
     requires_backends(self    ,  ['torch', 'transformers', 'onnx']			)
 @classmethod
 def   A_			(   cls						:   List[Any]    ,  *UpperCAmelCase						:   Optional[Any]    ,  **UpperCAmelCase						:   Optional[Any]			)    ->			str:
     requires_backends(cls    ,  ['torch', 'transformers', 'onnx']			)







 @classmethod
 def   A_			(   cls						:   List[Any]    ,  *UpperCAmelCase						:   Optional[int]    ,  **UpperCAmelCase						:   Any			)    ->			Any:
     requires_backends(cls    ,  ['torch', 'transformers', 'onnx']			)


class      lowerCAmelCase							(      metaclass=__UpperCamelCase    ):
 UpperCAmelCase__									=	["""torch""", """transformers""", """onnx"""]
 def __init__(   self						:   List[str]    ,  *UpperCAmelCase						:   Tuple    ,  **UpperCAmelCase						:   Optional[int]			)    ->			Optional[Any]:
     requires_backends(self    ,  ['torch', 'transformers', 'onnx']			)
 @classmethod
 def   A_			(   cls						:   Union[str, Any]    ,  *UpperCAmelCase						:   List[Any]    ,  **UpperCAmelCase						:   Optional[Any]			)    ->			Tuple:
     requires_backends(cls    ,  ['torch', 'transformers', 'onnx']			)







 @classmethod
 def   A_			(   cls						:   Tuple    ,  *UpperCAmelCase						:   List[str]    ,  **UpperCAmelCase						:   List[str]			)    ->			Tuple:
     requires_backends(cls    ,  ['torch', 'transformers', 'onnx']			)





