





'''simple docstring'''





import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
       from transformers.pipelines.conversational import Conversation


UpperCamelCase__      =logging.get_logger(__name__)

UpperCamelCase__      ={'vocab_file': 'vocab.txt', 'emoji_file': 'emoji.json'}

UpperCamelCase__      ={
    'vocab_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt',
    },
    'emoji_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json',
    },
}

UpperCamelCase__      ={
    'abeja/gpt-neox-japanese-2.7b': 2048,
}






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase	):







     with open(lowercase__,       "r",       encoding="utf-8"	) as f:
          _SCREAMING_SNAKE_CASE       :	List[Any]						    =							json.loads(f.read()	)

     _SCREAMING_SNAKE_CASE       :	Tuple						    =							collections.OrderedDict()
     _SCREAMING_SNAKE_CASE       :	List[Any]						    =							collections.OrderedDict()
     _SCREAMING_SNAKE_CASE       :	List[Any]						    =							collections.OrderedDict()
     with open(lowercase__,       "r",       encoding="utf-8"	) as f:
          _SCREAMING_SNAKE_CASE       :	Any						    =							f.readlines()
     _SCREAMING_SNAKE_CASE       :	Any						    =							[[t.rstrip("\n"	)] if (t == "," or "," not in t) else t.rstrip("\n"	).split(","	) for t in token]
     for idx, b in enumerate(lowercase__	):
          _SCREAMING_SNAKE_CASE       :	Dict						    =							b
          _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							idx
          for wd in b:
               _SCREAMING_SNAKE_CASE       :	List[Any]						    =							idx

     return vocab, raw_vocab, ids_to_tokens, emoji





class 			lowerCAmelCase__( __a    ):






       '''simple docstring'''


       __snake_case 			=    VOCAB_FILES_NAMES
       __snake_case 			=    PRETRAINED_VOCAB_FILES_MAP
       __snake_case 			=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __snake_case 			=    ["input_ids", "attention_mask"]




       def __init__(      self    ,						__lowerCamelCase    ,						__lowerCamelCase    ,						__lowerCamelCase="<|endoftext|>"    ,						__lowerCamelCase="<|endoftext|>"    ,						__lowerCamelCase="<|startoftext|>"    ,						__lowerCamelCase="<|endoftext|>"    ,						__lowerCamelCase=False    ,						**__lowerCamelCase    ,						)			->     List[Any]:
            super().__init__(
                unk_token=lowerCAmelCase_    ,						pad_token=lowerCAmelCase_    ,						bos_token=lowerCAmelCase_    ,						eos_token=lowerCAmelCase_    ,						do_clean_text=lowerCAmelCase_    ,						**lowerCAmelCase_    ,						)
            if not os.path.isfile(lowerCAmelCase_							):
                 raise ValueError(
                     F"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
                     " model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"							)
            if not os.path.isfile(lowerCAmelCase_							):
                 raise ValueError(
                     F"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
                     " pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"							)
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							do_clean_text
            _SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE       :	str						    =							load_vocab_and_emoji(lowerCAmelCase_    ,						lowerCAmelCase_							)
            _SCREAMING_SNAKE_CASE       :	Dict						    =							SubWordJapaneseTokenizer(
                vocab=self.vocab    ,						ids_to_tokens=self.ids_to_tokens    ,						emoji=self.emoji							)




       @property
       def     UpperCamelCase_			(      self							)			->     int:
            return len(self.raw_vocab							)




       def     UpperCamelCase_			(      self							)			->     Optional[Any]:
            return dict(self.raw_vocab    ,						**self.added_tokens_encoder							)




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase							)			->     List[str]:
            return self.subword_tokenizer.tokenize(lowerCAmelCase_    ,						clean=self.do_clean_text							)




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase							)			->     Optional[int]:
            return self.vocab.get(lowerCAmelCase_    ,						self.vocab.get(self.unk_token							)							)




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase							)			->     List[str]:
            return self.subword_tokenizer.convert_id_to_token(lowerCAmelCase_							)




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase							)			->     Optional[Any]:
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							"".join(lowerCAmelCase_							).strip()
            return out_string




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase							)			->     List[int]:
            _SCREAMING_SNAKE_CASE       :	str						    =							[]
            for is_user, text in conversation.iter_texts():
                 input_ids.extend(self.encode(lowerCAmelCase_    ,						add_special_tokens=lowerCAmelCase_							) + [self.eos_token_id]							)

            if len(lowerCAmelCase_							) > self.model_max_length:
                 _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							input_ids[-self.model_max_length :]
            return input_ids







       def     UpperCamelCase_			(      self    ,						__lowerCamelCase    ,						__lowerCamelCase = None							)			->     Tuple[str]:
            _SCREAMING_SNAKE_CASE       :	str						    =							0
            if os.path.isdir(lowerCAmelCase_							):
                 _SCREAMING_SNAKE_CASE       :	str						    =							os.path.join(
                     lowerCAmelCase_    ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]							)
                 _SCREAMING_SNAKE_CASE       :	Tuple						    =							os.path.join(
                     lowerCAmelCase_    ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["emoji_file"]							)
            else:
                 _SCREAMING_SNAKE_CASE       :	Tuple						    =							(
                     (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["vocab_file"]
                 )
                 _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							(
                     (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["emoji_file"]
                 )
            with open(lowerCAmelCase_    ,						"w"    ,						encoding="utf-8"							) as writer:
                 for token_index, token in self.ids_to_tokens.items():
                      if index != token_index:
                           logger.warning(
                               F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                               " Please check that the vocabulary is not corrupted!"							)
                           _SCREAMING_SNAKE_CASE       :	Dict						    =							token_index
                      writer.write(",".join(lowerCAmelCase_							) + "\n"							)
                      index += 1
            with open(lowerCAmelCase_    ,						"w"    ,						encoding="utf-8"							) as writer:
                 json.dump(self.emoji    ,						lowerCAmelCase_							)
            return vocab_file, emoji_file





class 			lowerCAmelCase__( __a    ):






       '''simple docstring'''


       def __init__(      self    ,						__lowerCamelCase    ,						__lowerCamelCase    ,						__lowerCamelCase							)			->     str:
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							vocab  # same as swe
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							ids_to_tokens  # same as bpe
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							emoji
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							np.max([len(lowerCAmelCase_							) for w in self.vocab.keys()]							)
            _SCREAMING_SNAKE_CASE       :	Any						    =							re.compile(r"(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)"							)
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							re.compile(r"[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*"							)
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							re.compile(r"[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}"							)
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							re.compile(
                r"([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"							)
            _SCREAMING_SNAKE_CASE       :	Dict						    =							re.compile(
                r"(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"							)
            _SCREAMING_SNAKE_CASE       :	Any						    =							re.compile(
                r"((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*"							)
            _SCREAMING_SNAKE_CASE       :	str						    =							"─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"
            _SCREAMING_SNAKE_CASE       :	str						    =							"▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							str.maketrans({k: "<BLOCK>" for k in keisen + blocks}							)




       def __len__(      self							)			->     Tuple:
            return len(self.ids_to_tokens							)




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							self.content_repattera.sub("<URL>"    ,						lowerCAmelCase_							)
            _SCREAMING_SNAKE_CASE       :	int						    =							self.content_repattera.sub("<EMAIL>"    ,						lowerCAmelCase_							)
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							self.content_repattera.sub("<TEL>"    ,						lowerCAmelCase_							)
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							self.content_repattera.sub("<DATE>"    ,						lowerCAmelCase_							)
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							self.content_repattera.sub("<DATE>"    ,						lowerCAmelCase_							)
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							self.content_repattera.sub("<PRICE>"    ,						lowerCAmelCase_							)
            _SCREAMING_SNAKE_CASE       :	int						    =							content.translate(self.content_transa							)
            while "<BLOCK><BLOCK>" in content:
                 _SCREAMING_SNAKE_CASE       :	Tuple						    =							content.replace("<BLOCK><BLOCK>"    ,						"<BLOCK>"							)
            return content




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase    ,						__lowerCamelCase=False							)			->     Union[str, Any]:
            _SCREAMING_SNAKE_CASE       :	str						    =							text.replace(" "    ,						"<SP>"							)
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							text.replace("　"    ,						"<SP>"							)
            _SCREAMING_SNAKE_CASE       :	int						    =							text.replace("\r\n"    ,						"<BR>"							)
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							text.replace("\n"    ,						"<BR>"							)
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							text.replace("\r"    ,						"<BR>"							)
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							text.replace("\t"    ,						"<TAB>"							)
            _SCREAMING_SNAKE_CASE       :	str						    =							text.replace("—"    ,						"ー"							)
            _SCREAMING_SNAKE_CASE       :	str						    =							text.replace("−"    ,						"ー"							)
            for k, v in self.emoji["emoji"].items():
                 if k in text:
                      _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							text.replace(lowerCAmelCase_    ,						lowerCAmelCase_							)
            if clean:
                 _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							self.clean_text(lowerCAmelCase_							)

            def check_simbol(__lowerCamelCase							):
                 _SCREAMING_SNAKE_CASE       :	int						    =							x.encode()
                 if len(lowerCAmelCase_							) == 1 and len(lowerCAmelCase_							) == 2:
                      _SCREAMING_SNAKE_CASE       :	str						    =							(int(e[0]							) << 8) + int(e[1]							)
                      if (
                          (c >= 0xC2A1 and c <= 0xC2BF)
                          or (c >= 0xC780 and c <= 0xC783)
                          or (c >= 0xCAB9 and c <= 0xCBBF)
                          or (c >= 0xCC80 and c <= 0xCDA2)
                      ):
                           return True
                 return False

            def checkuae(__lowerCamelCase							):
                 _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							x.encode()
                 if len(lowerCAmelCase_							) == 1 and len(lowerCAmelCase_							) == 3:
                      _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							(int(e[0]							) << 1_6) + (int(e[1]							) << 8) + int(e[2]							)
                      if c >= 0xE28080 and c <= 0xE2B07F:
                           return True
                 return False

            _SCREAMING_SNAKE_CASE       :	Tuple						    =							0
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							[]
            while pos < len(lowerCAmelCase_							):
                 _SCREAMING_SNAKE_CASE       :	List[str]						    =							min(len(lowerCAmelCase_							)    ,						pos + self.maxlen + 1							) if text[pos] == "<" else pos + 3
                 _SCREAMING_SNAKE_CASE       :	Any						    =							[]  # (token_id, token, pos)
                 for e in range(lowerCAmelCase_    ,						lowerCAmelCase_    ,						-1							):
                      _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							text[pos:e]
                      if wd in self.vocab:
                           if wd[0] == "<" and len(lowerCAmelCase_							) > 2:
                                _SCREAMING_SNAKE_CASE       :	List[str]						    =							[(self.vocab[wd], wd, e)]
                                break
                           else:
                                candidates.append((self.vocab[wd], wd, e)							)
                 if len(lowerCAmelCase_							) > 0:
                      # the smallest token_id is adopted
                      _SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE       :	int						    =							sorted(lowerCAmelCase_    ,						key=lambda __lowerCamelCase							: x[0]							)[0]
                      result.append(lowerCAmelCase_							)
                      _SCREAMING_SNAKE_CASE       :	List[str]						    =							e
                 else:
                      _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							pos + 1
                      _SCREAMING_SNAKE_CASE       :	str						    =							text[pos:end]
                      if check_simbol(lowerCAmelCase_							):
                           result.append("<KIGOU>"							)
                      elif checkuae(lowerCAmelCase_							):
                           result.append("<U2000U2BFF>"							)
                      else:
                           for i in wd.encode("utf-8"							):
                                result.append("<|byte%d|>" % i							)
                      _SCREAMING_SNAKE_CASE       :	Dict						    =							end
            return result







       def     UpperCamelCase_			(      self    ,						__lowerCamelCase    ,						__lowerCamelCase="\n"							)			->     List[Any]:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							[]
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							[]
            _SCREAMING_SNAKE_CASE       :	Any						    =							self.ids_to_tokens[index][0]
            if word[:6] == "<|byte" and word[-2:] == "|>":
                 byte_tokens.append(int(word[6:-2]							)							)
            else:
                 if len(lowerCAmelCase_							) > 0:
                      words.append(bytearray(lowerCAmelCase_							).decode("utf-8"    ,						errors="replace"							)							)
                      _SCREAMING_SNAKE_CASE       :	int						    =							[]
                 if word[:7] == "<|emoji" and word[-2:] == "|>":
                      words.append(self.emoji["emoji_inv"][word]							)
                 elif word == "<SP>":
                      words.append(" "							)
                 elif word == "<BR>":
                      words.append(lowerCAmelCase_							)
                 elif word == "<TAB>":
                      words.append("\t"							)
                 elif word == "<BLOCK>":
                      words.append("▀"							)
                 elif word == "<KIGOU>":
                      words.append("ǀ"							)
                 elif word == "<U2000U2BFF>":
                      words.append("‖"							)
                 else:
                      words.append(lowerCAmelCase_							)
            if len(lowerCAmelCase_							) > 0:
                 words.append(bytearray(lowerCAmelCase_							).decode("utf-8"    ,						errors="replace"							)							)
            _SCREAMING_SNAKE_CASE       :	int						    =							"".join(lowerCAmelCase_							)
            return text






import argparse

import torch

from transformers import GPTaConfig, GPTaModel, load_tf_weights_in_gpta
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase	):
     # Construct model
     if gpta_config_file == "":
          _SCREAMING_SNAKE_CASE       :	str						    =							GPTaConfig()
     else:
          _SCREAMING_SNAKE_CASE       :	int						    =							GPTaConfig.from_json_file(__lowerCamelCase	)
     _SCREAMING_SNAKE_CASE       :	Dict						    =							GPTaModel(__lowerCamelCase	)

     # Load weights from numpy
     load_tf_weights_in_gpta(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase	)

     # Save pytorch-model
     _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							pytorch_dump_folder_path + "/" + WEIGHTS_NAME
     _SCREAMING_SNAKE_CASE       :	Tuple						    =							pytorch_dump_folder_path + "/" + CONFIG_NAME
     print(f"""Save PyTorch model to {pytorch_weights_dump_path}"""	)
     torch.save(model.state_dict(),       __lowerCamelCase	)
     print(f"""Save configuration file to {pytorch_config_dump_path}"""	)
     with open(__lowerCamelCase,       "w",       encoding="utf-8"	) as f:
          f.write(config.to_json_string()	)


if __name__ == "__main__":
       UpperCamelCase__      =argparse.ArgumentParser()
       # Required parameters
       parser.add_argument(
           '--gpt2_checkpoint_path', default=None, type=str, required=True, help='Path to the TensorFlow checkpoint path.'
       )
       parser.add_argument(
           '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
       )
       parser.add_argument(
           '--gpt2_config_file',
           default='',
           type=str,
           help=(
               'An optional config json file corresponding to the pre-trained OpenAI model. \n'
               'This specifies the model architecture.'
           ),
       )
       UpperCamelCase__      =parser.parse_args()
       convert_gpta_checkpoint_to_pytorch(args.gpta_checkpoint_path, args.gpta_config_file, args.pytorch_dump_folder_path)