


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


UpperCamelCase					:       List[str]			    =     logging.get_logger(__name__)

UpperCamelCase					:       List[Any]			    =     {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCamelCase					:       Union[str, Any]			    =     {
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCamelCase					:       Optional[int]			    =     {
    """google/realm-cc-news-pretrained-embedder""": 512,
    """google/realm-cc-news-pretrained-encoder""": 512,
    """google/realm-cc-news-pretrained-scorer""": 512,
    """google/realm-cc-news-pretrained-openqa""": 512,
    """google/realm-orqa-nq-openqa""": 512,
    """google/realm-orqa-nq-reader""": 512,
    """google/realm-orqa-wq-openqa""": 512,
    """google/realm-orqa-wq-reader""": 512,
}

UpperCamelCase					:       List[str]			    =     {
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}



class A__     ( A__  ):



  """simple docstring"""



  _lowercase									=		VOCAB_FILES_NAMES
  _lowercase									=		PRETRAINED_VOCAB_FILES_MAP
  _lowercase									=		PRETRAINED_INIT_CONFIGURATION
  _lowercase									=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _lowercase									=		RealmTokenizer

  def __init__( self			:  str				,				lowerCamelCase__			:  str=None				,				lowerCamelCase__			:  int=None				,				lowerCamelCase__			:  Any=True				,				lowerCamelCase__			:  Any="[UNK]"				,				lowerCamelCase__			:  Optional[int]="[SEP]"				,				lowerCamelCase__			:  Optional[Any]="[PAD]"				,				lowerCamelCase__			:  Optional[int]="[CLS]"				,				lowerCamelCase__			:  Tuple="[MASK]"				,				lowerCamelCase__			:  int=True				,				lowerCamelCase__			:  Optional[int]=None				,				**lowerCamelCase__			:  str				,				):
        super().__init__(
            lowerCamelCase__				,				tokenizer_file=lowerCamelCase__				,				do_lower_case=lowerCamelCase__				,				unk_token=lowerCamelCase__				,				sep_token=lowerCamelCase__				,				pad_token=lowerCamelCase__				,				cls_token=lowerCamelCase__				,				mask_token=lowerCamelCase__				,				tokenize_chinese_chars=lowerCamelCase__				,				strip_accents=lowerCamelCase__				,				**lowerCamelCase__				,				)

        a__	:  Optional[Any]     =   json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
        if (
            normalizer_state.get("lowercase"				,				lowerCamelCase__       ) != do_lower_case
            or normalizer_state.get("strip_accents"				,				lowerCamelCase__       ) != strip_accents
            or normalizer_state.get("handle_chinese_chars"				,				lowerCamelCase__       ) != tokenize_chinese_chars
        ):
              a__	:  Optional[Any]     =   getattr(lowerCamelCase__				,				normalizer_state.pop("type"       )       )
              a__	:  List[str]     =   do_lower_case
              a__	:  List[Any]     =   strip_accents
              a__	:  Optional[Any]     =   tokenize_chinese_chars
              a__	:  Optional[int]     =   normalizer_class(**lowerCamelCase__       )

        a__	:  Union[str, Any]     =   do_lower_case

  def    _UpperCamelCase( self			:  Any				,				lowerCamelCase__			:  int				,				**lowerCamelCase__			:  int       ):
        a__	:  List[str]     =   PaddingStrategy.MAX_LENGTH

        a__	:  Dict     =   text
        a__	:  Optional[int]     =   kwargs.pop("text_pair"				,				lowerCamelCase__       )
        a__	:  Dict     =   kwargs.pop("return_tensors"				,				lowerCamelCase__       )

        a__	:  List[Any]     =   {
            "input_ids": [],
            "attention_mask": [],
            "token_type_ids": [],
        }

        for idx, candidate_text in enumerate(lowerCamelCase__       ):
              if batch_text_pair is not None:
                    a__	:  Dict     =   batch_text_pair[idx]
              else:
                    a__	:  Union[str, Any]     =   None

              a__	:  int     =   super().__call__(lowerCamelCase__				,				lowerCamelCase__				,				return_tensors=lowerCamelCase__				,				**lowerCamelCase__       )

              a__	:  Union[str, Any]     =   encoded_candidates.get("input_ids"       )
              a__	:  List[Any]     =   encoded_candidates.get("attention_mask"       )
              a__	:  int     =   encoded_candidates.get("token_type_ids"       )

              if encoded_input_ids is not None:
                    output_data["input_ids"].append(lowerCamelCase__       )
              if encoded_attention_mask is not None:
                    output_data["attention_mask"].append(lowerCamelCase__       )
              if encoded_token_type_ids is not None:
                    output_data["token_type_ids"].append(lowerCamelCase__       )

        a__	:  Union[str, Any]     =   {key: item for key, item in output_data.items() if len(lowerCamelCase__       ) != 0}

        return BatchEncoding(lowerCamelCase__				,				tensor_type=lowerCamelCase__       )

  def    _UpperCamelCase( self			:  List[str]				,				lowerCamelCase__			:  Optional[Any]				,				lowerCamelCase__			:  str=None       ):
        a__	:  Optional[Any]     =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

        return output

  def    _UpperCamelCase( self			:  Dict				,				lowerCamelCase__			:  List[int]				,				lowerCamelCase__			:  Optional[List[int]] = None       ):
        a__	:  List[Any]     =   [self.sep_token_id]
        a__	:  Optional[int]     =   [self.cls_token_id]
        if token_ids_a is None:
              return len(cls + token_ids_a + sep       ) * [0]
        return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]

  def    _UpperCamelCase( self			:  List[Any]				,				lowerCamelCase__			:  str				,				lowerCamelCase__			:  Optional[str] = None       ):
        a__	:  Tuple     =   self._tokenizer.model.save(lowerCamelCase__				,				name=lowerCamelCase__       )
        return tuple(lowerCamelCase__       )








from __future__ import annotations




def  __UpperCAmelCase(   lowercase_			,     lowercase_ = None			,     lowercase_ = None			,     lowercase_ = False			,     ):
       _lowerCamelCase				:  Tuple						=		cipher_alphabet or [chr(lowercase_   ) for i in range(97			,     1_23   )]

       # If the argument is None or the user provided an empty dictionary
       if not frequencies_dict:
              # Frequencies of letters in the english language (how much they show up)
              _lowerCamelCase				:  Dict						=		{
                  '''a''': 0.0_8_4_9_7,
                  '''b''': 0.0_1_4_9_2,
                  '''c''': 0.0_2_2_0_2,
                  '''d''': 0.0_4_2_5_3,
                  '''e''': 0.1_1_1_6_2,
                  '''f''': 0.0_2_2_2_8,
                  '''g''': 0.0_2_0_1_5,
                  '''h''': 0.0_6_0_9_4,
                  '''i''': 0.0_7_5_4_6,
                  '''j''': 0.0_0_1_5_3,
                  '''k''': 0.0_1_2_9_2,
                  '''l''': 0.0_4_0_2_5,
                  '''m''': 0.0_2_4_0_6,
                  '''n''': 0.0_6_7_4_9,
                  '''o''': 0.0_7_5_0_7,
                  '''p''': 0.0_1_9_2_9,
                  '''q''': 0.0_0_0_9_5,
                  '''r''': 0.0_7_5_8_7,
                  '''s''': 0.0_6_3_2_7,
                  '''t''': 0.0_9_3_5_6,
                  '''u''': 0.0_2_7_5_8,
                  '''v''': 0.0_0_9_7_8,
                  '''w''': 0.0_2_5_6_0,
                  '''x''': 0.0_0_1_5_0,
                  '''y''': 0.0_1_9_9_4,
                  '''z''': 0.0_0_0_7_7,
              }
       else:
              # Custom frequencies dictionary
              _lowerCamelCase				:  Tuple						=		frequencies_dict

       if not case_sensitive:
              _lowerCamelCase				:  Optional[int]						=		ciphertext.lower()

       # Chi squared statistic values
       _lowerCamelCase				:  dict[int, tuple[float, str]]						=		{}

       # cycle through all of the shifts
       for shift in range(len(lowercase_   )   ):
              _lowerCamelCase				:  Any						=		''''''

              # decrypt the message with the shift
              for letter in ciphertext:
                     try:
                            # Try to index the letter in the alphabet
                            _lowerCamelCase				:  int						=		(alphabet_letters.index(letter.lower()   ) - shift) % len(
                                lowercase_   )
                            decrypted_with_shift += (
                                alphabet_letters[new_key].upper()
                                if case_sensitive and letter.isupper()
                                else alphabet_letters[new_key]
                            )
                     except ValueError:
                            # Append the character if it isn't in the alphabet
                            decrypted_with_shift += letter

              _lowerCamelCase				:  Optional[Any]						=		0.0

              # Loop through each letter in the decoded message with the shift
              for letter in decrypted_with_shift:
                     if case_sensitive:
                            _lowerCamelCase				:  Any						=		letter.lower()
                            if letter in frequencies:
                                   # Get the amount of times the letter occurs in the message
                                   _lowerCamelCase				:  List[str]						=		decrypted_with_shift.lower().count(lowercase_   )

                                   # Get the excepcted amount of times the letter should appear based
                                   # on letter frequencies
                                   _lowerCamelCase				:  Optional[Any]						=		frequencies[letter] * occurrences

                                   # Complete the chi squared statistic formula
                                   _lowerCamelCase				:  int						=		((occurrences - expected) ** 2) / expected

                                   # Add the margin of error to the total chi squared statistic
                                   chi_squared_statistic += chi_letter_value
                     else:
                            if letter.lower() in frequencies:
                                   # Get the amount of times the letter occurs in the message
                                   _lowerCamelCase				:  Union[str, Any]						=		decrypted_with_shift.count(lowercase_   )

                                   # Get the excepcted amount of times the letter should appear based
                                   # on letter frequencies
                                   _lowerCamelCase				:  List[str]						=		frequencies[letter] * occurrences

                                   # Complete the chi squared statistic formula
                                   _lowerCamelCase				:  str						=		((occurrences - expected) ** 2) / expected

                                   # Add the margin of error to the total chi squared statistic
                                   chi_squared_statistic += chi_letter_value

        # Add the data to the chi_squared_statistic_values dictionary
              _lowerCamelCase				:  Tuple						=		(
                  chi_squared_statistic,
                  decrypted_with_shift,
              )

       # Get the most likely cipher by finding the cipher with the smallest chi squared
       # statistic
       def chi_squared_statistic_values_sorting_key(lowercase_   ) -> tuple[float, str]:
              return chi_squared_statistic_values[key]

       _lowerCamelCase				:  int						=		min(
           lowercase_			,     key=lowercase_			,     )

       # Get all the data from the most likely cipher (key, decoded message)
       (
           (
           _lowerCamelCase
       ),	(
           _lowerCamelCase
       ),	
       )				:  Optional[int]						=		chi_squared_statistic_values[most_likely_cipher]

       # Return the data on the most likely shift
       return (
           most_likely_cipher,
           most_likely_cipher_chi_squared_value,
           decoded_most_likely_cipher,
       )

