





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


UpperCamelCase__      =logging.get_logger(__name__)

UpperCamelCase__      ={'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCamelCase__      ={
    'vocab_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'
        ),
        'google/realm-orqa-nq-openqa': 'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-nq-reader': 'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt',
        'google/realm-orqa-wq-openqa': 'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-wq-reader': 'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-openqa': (
            'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-reader': (
            'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-openqa': (
            'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-reader': (
            'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'
        ),
    },
}

UpperCamelCase__      ={
    'google/realm-cc-news-pretrained-embedder': 512,
    'google/realm-cc-news-pretrained-encoder': 512,
    'google/realm-cc-news-pretrained-scorer': 512,
    'google/realm-cc-news-pretrained-openqa': 512,
    'google/realm-orqa-nq-openqa': 512,
    'google/realm-orqa-nq-reader': 512,
    'google/realm-orqa-wq-openqa': 512,
    'google/realm-orqa-wq-reader': 512,
}

UpperCamelCase__      ={
    'google/realm-cc-news-pretrained-embedder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-encoder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-scorer': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-reader': {'do_lower_case': True},
    'google/realm-orqa-wq-openqa': {'do_lower_case': True},
    'google/realm-orqa-wq-reader': {'do_lower_case': True},
}





class 			lowerCAmelCase__( __lowercase    ):






       '''simple docstring'''


       __snake_case 			=    VOCAB_FILES_NAMES
       __snake_case 			=    PRETRAINED_VOCAB_FILES_MAP
       __snake_case 			=    PRETRAINED_INIT_CONFIGURATION
       __snake_case 			=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __snake_case 			=    RealmTokenizer




       def __init__(      self    ,						__lowerCamelCase=None    ,						__lowerCamelCase=None    ,						__lowerCamelCase=True    ,						__lowerCamelCase="[UNK]"    ,						__lowerCamelCase="[SEP]"    ,						__lowerCamelCase="[PAD]"    ,						__lowerCamelCase="[CLS]"    ,						__lowerCamelCase="[MASK]"    ,						__lowerCamelCase=True    ,						__lowerCamelCase=None    ,						**__lowerCamelCase    ,						)			->     Dict:
            super().__init__(
                __lowerCamelCase    ,						tokenizer_file=__lowerCamelCase    ,						do_lower_case=__lowerCamelCase    ,						unk_token=__lowerCamelCase    ,						sep_token=__lowerCamelCase    ,						pad_token=__lowerCamelCase    ,						cls_token=__lowerCamelCase    ,						mask_token=__lowerCamelCase    ,						tokenize_chinese_chars=__lowerCamelCase    ,						strip_accents=__lowerCamelCase    ,						**__lowerCamelCase    ,						)

            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
            if (
                normalizer_state.get("lowercase"    ,						__lowerCamelCase							) != do_lower_case
                or normalizer_state.get("strip_accents"    ,						__lowerCamelCase							) != strip_accents
                or normalizer_state.get("handle_chinese_chars"    ,						__lowerCamelCase							) != tokenize_chinese_chars
            ):
                 _SCREAMING_SNAKE_CASE       :	Dict						    =							getattr(__lowerCamelCase    ,						normalizer_state.pop("type"							)							)
                 _SCREAMING_SNAKE_CASE       :	str						    =							do_lower_case
                 _SCREAMING_SNAKE_CASE       :	str						    =							strip_accents
                 _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							tokenize_chinese_chars
                 _SCREAMING_SNAKE_CASE       :	int						    =							normalizer_class(**__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	int						    =							do_lower_case




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase    ,						**__lowerCamelCase							)			->     List[Any]:
            _SCREAMING_SNAKE_CASE       :	Dict						    =							PaddingStrategy.MAX_LENGTH

            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							text
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							kwargs.pop("text_pair"    ,						__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							kwargs.pop("return_tensors"    ,						__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							{
                "input_ids": [],
                "attention_mask": [],
                "token_type_ids": [],
            }

            for idx, candidate_text in enumerate(__lowerCamelCase							):
                 if batch_text_pair is not None:
                      _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							batch_text_pair[idx]
                 else:
                      _SCREAMING_SNAKE_CASE       :	List[str]						    =							None

                 _SCREAMING_SNAKE_CASE       :	Any						    =							super().__call__(__lowerCamelCase    ,						__lowerCamelCase    ,						return_tensors=__lowerCamelCase    ,						**__lowerCamelCase							)

                 _SCREAMING_SNAKE_CASE       :	List[Any]						    =							encoded_candidates.get("input_ids"							)
                 _SCREAMING_SNAKE_CASE       :	str						    =							encoded_candidates.get("attention_mask"							)
                 _SCREAMING_SNAKE_CASE       :	Dict						    =							encoded_candidates.get("token_type_ids"							)

                 if encoded_input_ids is not None:
                      output_data["input_ids"].append(__lowerCamelCase							)
                 if encoded_attention_mask is not None:
                      output_data["attention_mask"].append(__lowerCamelCase							)
                 if encoded_token_type_ids is not None:
                      output_data["token_type_ids"].append(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Any						    =							{key: item for key, item in output_data.items() if len(__lowerCamelCase							) != 0}

            return BatchEncoding(__lowerCamelCase    ,						tensor_type=__lowerCamelCase							)




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase    ,						__lowerCamelCase=None							)			->     Dict:
            _SCREAMING_SNAKE_CASE       :	str						    =							[self.cls_token_id] + token_ids_a + [self.sep_token_id]

            if token_ids_a:
                 output += token_ids_a + [self.sep_token_id]

            return output




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase    ,						__lowerCamelCase = None							)			->     List[int]:
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							[self.sep_token_id]
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							[self.cls_token_id]
            if token_ids_a is None:
                 return len(cls + token_ids_a + sep							) * [0]
            return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







       def     UpperCamelCase_			(      self    ,						__lowerCamelCase    ,						__lowerCamelCase = None							)			->     Tuple[str]:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							self._tokenizer.model.save(__lowerCamelCase    ,						name=__lowerCamelCase							)
            return tuple(__lowerCamelCase							)






import argparse

import torch

from transformers import GPTaConfig, GPTaModel, load_tf_weights_in_gpta
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase	):
     # Construct model
     if gpta_config_file == "":
          _SCREAMING_SNAKE_CASE       :	str						    =							GPTaConfig()
     else:
          _SCREAMING_SNAKE_CASE       :	int						    =							GPTaConfig.from_json_file(__lowerCamelCase	)
     _SCREAMING_SNAKE_CASE       :	Dict						    =							GPTaModel(__lowerCamelCase	)

     # Load weights from numpy
     load_tf_weights_in_gpta(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase	)

     # Save pytorch-model
     _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							pytorch_dump_folder_path + "/" + WEIGHTS_NAME
     _SCREAMING_SNAKE_CASE       :	Tuple						    =							pytorch_dump_folder_path + "/" + CONFIG_NAME
     print(f"""Save PyTorch model to {pytorch_weights_dump_path}"""	)
     torch.save(model.state_dict(),       __lowerCamelCase	)
     print(f"""Save configuration file to {pytorch_config_dump_path}"""	)
     with open(__lowerCamelCase,       "w",       encoding="utf-8"	) as f:
          f.write(config.to_json_string()	)


if __name__ == "__main__":
       UpperCamelCase__      =argparse.ArgumentParser()
       # Required parameters
       parser.add_argument(
           '--gpt2_checkpoint_path', default=None, type=str, required=True, help='Path to the TensorFlow checkpoint path.'
       )
       parser.add_argument(
           '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
       )
       parser.add_argument(
           '--gpt2_config_file',
           default='',
           type=str,
           help=(
               'An optional config json file corresponding to the pre-trained OpenAI model. \n'
               'This specifies the model architecture.'
           ),
       )
       UpperCamelCase__      =parser.parse_args()
       convert_gpta_checkpoint_to_pytorch(args.gpta_checkpoint_path, args.gpta_config_file, args.pytorch_dump_folder_path)