'''simple docstring'''





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


a_							=      logging.get_logger(__name__)

a_							=      {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

a_							=      {
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

a_							=      {
    """google/realm-cc-news-pretrained-embedder""": 512,
    """google/realm-cc-news-pretrained-encoder""": 512,
    """google/realm-cc-news-pretrained-scorer""": 512,
    """google/realm-cc-news-pretrained-openqa""": 512,
    """google/realm-orqa-nq-openqa""": 512,
    """google/realm-orqa-nq-reader""": 512,
    """google/realm-orqa-wq-openqa""": 512,
    """google/realm-orqa-wq-reader""": 512,
}

a_							=      {
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}


class 			UpperCAmelCase__    (		lowercase_					):



  """simple docstring"""






  lowerCAmelCase__		:	Union[str, Any]    		= VOCAB_FILES_NAMES
  lowerCAmelCase__		:	Dict    		= PRETRAINED_VOCAB_FILES_MAP
  lowerCAmelCase__		:	str    		= PRETRAINED_INIT_CONFIGURATION
  lowerCAmelCase__		:	Optional[int]    		= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowerCAmelCase__		:	Tuple    		= RealmTokenizer






  def __init__(				self:			Any				,				__lowerCAmelCase:			Optional[int]=None				,				__lowerCAmelCase:			Union[str, Any]=None				,				__lowerCAmelCase:			Optional[Any]=True				,				__lowerCAmelCase:			Dict="[UNK]"				,				__lowerCAmelCase:			Tuple="[SEP]"				,				__lowerCAmelCase:			Union[str, Any]="[PAD]"				,				__lowerCAmelCase:			Optional[int]="[CLS]"				,				__lowerCAmelCase:			Tuple="[MASK]"				,				__lowerCAmelCase:			int=True				,				__lowerCAmelCase:			Union[str, Any]=None				,				**__lowerCAmelCase:			int				,				) -> List[str]:




    '''simple docstring'''




    super().__init__(
        lowerCamelCase_				,				tokenizer_file=lowerCamelCase_				,				do_lower_case=lowerCamelCase_				,				unk_token=lowerCamelCase_				,				sep_token=lowerCamelCase_				,				pad_token=lowerCamelCase_				,				cls_token=lowerCamelCase_				,				mask_token=lowerCamelCase_				,				tokenize_chinese_chars=lowerCamelCase_				,				strip_accents=lowerCamelCase_				,				**lowerCamelCase_				,				)

    __UpperCAmelCase						    =       json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
    if (
        normalizer_state.get("lowercase"				,				lowerCamelCase_   ) != do_lower_case
        or normalizer_state.get("strip_accents"				,				lowerCamelCase_   ) != strip_accents
        or normalizer_state.get("handle_chinese_chars"				,				lowerCamelCase_   ) != tokenize_chinese_chars
    ):
      __UpperCAmelCase						    =       getattr(lowerCamelCase_				,				normalizer_state.pop("type"   )   )
      __UpperCAmelCase						    =       do_lower_case
      __UpperCAmelCase						    =       strip_accents
      __UpperCAmelCase						    =       tokenize_chinese_chars
      __UpperCAmelCase						    =       normalizer_class(**lowerCamelCase_   )

    __UpperCAmelCase						    =       do_lower_case






  def 	_UpperCAmelCase				(				self:			Any				,				__lowerCAmelCase:			Any				,				**__lowerCAmelCase:			int   ) -> int:




    '''simple docstring'''




    __UpperCAmelCase						    =       PaddingStrategy.MAX_LENGTH

    __UpperCAmelCase						    =       text
    __UpperCAmelCase						    =       kwargs.pop("text_pair"				,				lowerCamelCase_   )
    __UpperCAmelCase						    =       kwargs.pop("return_tensors"				,				lowerCamelCase_   )

    __UpperCAmelCase						    =       {
        """input_ids""": [],
        """attention_mask""": [],
        """token_type_ids""": [],
    }

    for idx, candidate_text in enumerate(lowerCamelCase_   ):
      if batch_text_pair is not None:
        __UpperCAmelCase						    =       batch_text_pair[idx]
      else:
        __UpperCAmelCase						    =       None

      __UpperCAmelCase						    =       super().__call__(lowerCamelCase_				,				lowerCamelCase_				,				return_tensors=lowerCamelCase_				,				**lowerCamelCase_   )

      __UpperCAmelCase						    =       encoded_candidates.get("input_ids"   )
      __UpperCAmelCase						    =       encoded_candidates.get("attention_mask"   )
      __UpperCAmelCase						    =       encoded_candidates.get("token_type_ids"   )

      if encoded_input_ids is not None:
        output_data["input_ids"].append(lowerCamelCase_   )
      if encoded_attention_mask is not None:
        output_data["attention_mask"].append(lowerCamelCase_   )
      if encoded_token_type_ids is not None:
        output_data["token_type_ids"].append(lowerCamelCase_   )

    __UpperCAmelCase						    =       {key: item for key, item in output_data.items() if len(lowerCamelCase_   ) != 0}

    return BatchEncoding(lowerCamelCase_				,				tensor_type=lowerCamelCase_   )






  def 	_UpperCAmelCase				(				self:			List[Any]				,				__lowerCAmelCase:			Any				,				__lowerCAmelCase:			Tuple=None   ) -> Optional[int]:




    '''simple docstring'''




    __UpperCAmelCase						    =       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

    if token_ids_a:
      output += token_ids_a + [self.sep_token_id]

    return output






  def 	_UpperCAmelCase				(				self:			str				,				__lowerCAmelCase:			List[int]				,				__lowerCAmelCase:			Optional[List[int]] = None   ) -> Dict:




    '''simple docstring'''




    __UpperCAmelCase						    =       [self.sep_token_id]
    __UpperCAmelCase						    =       [self.cls_token_id]
    if token_ids_a is None:
      return len(cls + token_ids_a + sep   ) * [0]
    return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]






  def 	_UpperCAmelCase				(				self:			List[Any]				,				__lowerCAmelCase:			str				,				__lowerCAmelCase:			Optional[str] = None   ) -> List[str]:




    '''simple docstring'''




    __UpperCAmelCase						    =       self._tokenizer.model.save(lowerCamelCase_				,				name=lowerCamelCase_   )
    return tuple(lowerCamelCase_   )




from ..utils import DummyObject, requires_backends


class 			UpperCAmelCase__    (		metaclass=snake_case					):



  """simple docstring"""






  lowerCAmelCase__		:	List[str]    		= ['transformers', 'torch', 'note_seq']






  def __init__(				self:			List[str]				,				*__lowerCAmelCase:			Optional[int]				,				**__lowerCAmelCase:			List[Any]   ) -> Optional[int]:




    '''simple docstring'''




    requires_backends(self				,				["transformers", "torch", "note_seq"]   )






  @classmethod
  def 	_UpperCAmelCase				(				cls:			Optional[int]				,				*__lowerCAmelCase:			Any				,				**__lowerCAmelCase:			List[str]   ) -> Dict:




    '''simple docstring'''




    requires_backends(cls				,				["transformers", "torch", "note_seq"]   )






  @classmethod
  def 	_UpperCAmelCase				(				cls:			Union[str, Any]				,				*__lowerCAmelCase:			Optional[Any]				,				**__lowerCAmelCase:			Optional[Any]   ) -> Any:




    '''simple docstring'''




    requires_backends(cls				,				["transformers", "torch", "note_seq"]   )



