




import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation


UpperCamelCase__					    =    logging.get_logger(__name__)

UpperCamelCase__					    =    {"""vocab_file""": """vocab.txt""", """emoji_file""": """emoji.json"""}

UpperCamelCase__					    =    {
    """vocab_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt""",
    },
    """emoji_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json""",
    },
}

UpperCamelCase__					    =    {
    """abeja/gpt-neox-japanese-2.7b""": 2048,
}







def 			_a   (		SCREAMING_SNAKE_CASE_							:       List[Any]   ,   SCREAMING_SNAKE_CASE_							:       Tuple  ):
					with open(SCREAMING_SNAKE_CASE_   ,   "r"   ,   encoding="utf-8"  ) as f:
										__lowerCAmelCase            = json.loads(f.read()  )

					__lowerCAmelCase            = collections.OrderedDict()
					__lowerCAmelCase            = collections.OrderedDict()
					__lowerCAmelCase            = collections.OrderedDict()
					with open(SCREAMING_SNAKE_CASE_   ,   "r"   ,   encoding="utf-8"  ) as f:
										__lowerCAmelCase            = f.readlines()
					__lowerCAmelCase            = [[t.rstrip("\n"  )] if (t == "," or "," not in t) else t.rstrip("\n"  ).split(","  ) for t in token]
					for idx, b in enumerate(SCREAMING_SNAKE_CASE_  ):
										__lowerCAmelCase            = b
										__lowerCAmelCase            = idx
										for wd in b:
															__lowerCAmelCase            = idx

					return vocab, raw_vocab, ids_to_tokens, emoji
class     a__					(      snake_case__   ):
						_a							:					Optional[Any]      =					VOCAB_FILES_NAMES
						_a							:					Optional[Any]      =					PRETRAINED_VOCAB_FILES_MAP
						_a							:					Dict      =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_a							:					str      =					["""input_ids""", """attention_mask"""]






						def __init__(     self	,    _A	,    _A	,    _A="<|endoftext|>"	,    _A="<|endoftext|>"	,    _A="<|startoftext|>"	,    _A="<|endoftext|>"	,    _A=False	,    **_A	,    ):




											"""simple docstring"""



											super().__init__(
											    unk_token=_A	,    pad_token=_A	,    bos_token=_A	,    eos_token=_A	,    do_clean_text=_A	,    **_A	,    )
											if not os.path.isfile(_A    ):
																raise ValueError(
																    f"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
																    " model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"    )
											if not os.path.isfile(_A    ):
																raise ValueError(
																    f"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
																    " pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"    )
											__lowerCAmelCase            = do_clean_text
											__lowerCAmelCase     ,							__lowerCAmelCase     ,							__lowerCAmelCase     ,							__lowerCAmelCase            = load_vocab_and_emoji(_A	,    _A    )
											__lowerCAmelCase            = SubWordJapaneseTokenizer(
											    vocab=self.vocab	,    ids_to_tokens=self.ids_to_tokens	,    emoji=self.emoji    )






						@property
						def        __SCREAMING_SNAKE_CASE(     self    ):




											"""simple docstring"""



											return len(self.raw_vocab    )






						def        __SCREAMING_SNAKE_CASE(     self    ):




											"""simple docstring"""



											return dict(self.raw_vocab	,    **self.added_tokens_encoder    )






						def        __SCREAMING_SNAKE_CASE(     self	,    _A    ):




											"""simple docstring"""



											return self.subword_tokenizer.tokenize(_A	,    clean=self.do_clean_text    )






						def        __SCREAMING_SNAKE_CASE(     self	,    _A    ):




											"""simple docstring"""



											return self.vocab.get(_A	,    self.vocab.get(self.unk_token    )    )






						def        __SCREAMING_SNAKE_CASE(     self	,    _A    ):




											"""simple docstring"""



											return self.subword_tokenizer.convert_id_to_token(_A    )






						def        __SCREAMING_SNAKE_CASE(     self	,    _A    ):




											"""simple docstring"""



											__lowerCAmelCase            = "".join(_A    ).strip()
											return out_string






						def        __SCREAMING_SNAKE_CASE(     self	,    _A    ):




											"""simple docstring"""



											__lowerCAmelCase            = []
											for is_user, text in conversation.iter_texts():
																input_ids.extend(self.encode(_A	,    add_special_tokens=_A    ) + [self.eos_token_id]    )

											if len(_A    ) > self.model_max_length:
																__lowerCAmelCase            = input_ids[-self.model_max_length :]
											return input_ids






						def        __SCREAMING_SNAKE_CASE(     self	,    _A	,    _A = None    ):




											"""simple docstring"""



											__lowerCAmelCase            = 0
											if os.path.isdir(_A    ):
																__lowerCAmelCase            = os.path.join(
																    _A	,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )
																__lowerCAmelCase            = os.path.join(
																    _A	,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["emoji_file"]    )
											else:
																__lowerCAmelCase            = (
																    (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["vocab_file"]
																)
																__lowerCAmelCase            = (
																    (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["emoji_file"]
																)
											with open(_A	,    "w"	,    encoding="utf-8"    ) as writer:
																for token_index, token in self.ids_to_tokens.items():
																					if index != token_index:
																										logger.warning(
																										    f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
																										    " Please check that the vocabulary is not corrupted!"    )
																										__lowerCAmelCase            = token_index
																					writer.write(",".join(_A    ) + "\n"    )
																					index += 1
											with open(_A	,    "w"	,    encoding="utf-8"    ) as writer:
																json.dump(self.emoji	,    _A    )
											return vocab_file, emoji_file






class     a__					(      snake_case__   ):






						def __init__(     self	,    _A	,    _A	,    _A    ):




											"""simple docstring"""



											__lowerCAmelCase            = vocab  # same as swe
											__lowerCAmelCase            = ids_to_tokens  # same as bpe
											__lowerCAmelCase            = emoji
											__lowerCAmelCase            = np.max([len(_A    ) for w in self.vocab.keys()]    )
											__lowerCAmelCase            = re.compile(R"(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)"    )
											__lowerCAmelCase            = re.compile(R"[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*"    )
											__lowerCAmelCase            = re.compile(R"[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}"    )
											__lowerCAmelCase            = re.compile(
											    R"([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"    )
											__lowerCAmelCase            = re.compile(
											    R"(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"    )
											__lowerCAmelCase            = re.compile(
											    R"((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*"    )
											__lowerCAmelCase            = "─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"
											__lowerCAmelCase            = "▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"
											__lowerCAmelCase            = str.maketrans({k: "<BLOCK>" for k in keisen + blocks}    )






						def __len__(     self    ):




											"""simple docstring"""



											return len(self.ids_to_tokens    )






						def        __SCREAMING_SNAKE_CASE(     self	,    _A    ):




											"""simple docstring"""



											__lowerCAmelCase            = self.content_repattera.sub("<URL>"	,    _A    )
											__lowerCAmelCase            = self.content_repattera.sub("<EMAIL>"	,    _A    )
											__lowerCAmelCase            = self.content_repattera.sub("<TEL>"	,    _A    )
											__lowerCAmelCase            = self.content_repattera.sub("<DATE>"	,    _A    )
											__lowerCAmelCase            = self.content_repattera.sub("<DATE>"	,    _A    )
											__lowerCAmelCase            = self.content_repattera.sub("<PRICE>"	,    _A    )
											__lowerCAmelCase            = content.translate(self.content_transa    )
											while "<BLOCK><BLOCK>" in content:
																__lowerCAmelCase            = content.replace("<BLOCK><BLOCK>"	,    "<BLOCK>"    )
											return content






						def        __SCREAMING_SNAKE_CASE(     self	,    _A	,    _A=False    ):




											"""simple docstring"""



											__lowerCAmelCase            = text.replace(" "	,    "<SP>"    )
											__lowerCAmelCase            = text.replace("　"	,    "<SP>"    )
											__lowerCAmelCase            = text.replace("\r\n"	,    "<BR>"    )
											__lowerCAmelCase            = text.replace("\n"	,    "<BR>"    )
											__lowerCAmelCase            = text.replace("\r"	,    "<BR>"    )
											__lowerCAmelCase            = text.replace("\t"	,    "<TAB>"    )
											__lowerCAmelCase            = text.replace("—"	,    "ー"    )
											__lowerCAmelCase            = text.replace("−"	,    "ー"    )
											for k, v in self.emoji["emoji"].items():
																if k in text:
																					__lowerCAmelCase            = text.replace(_A	,    _A    )
											if clean:
																__lowerCAmelCase            = self.clean_text(_A    )

											def check_simbol(_A    ):
																__lowerCAmelCase            = x.encode()
																if len(_A    ) == 1 and len(_A    ) == 2:
																					__lowerCAmelCase            = (int(e[0]    ) << 8) + int(e[1]    )
																					if (
																					    (c >= 0xc_2a1 and c <= 0xc_2bf)
																					    or (c >= 0xc_780 and c <= 0xc_783)
																					    or (c >= 0xc_ab9 and c <= 0xc_bbf)
																					    or (c >= 0xc_c80 and c <= 0xc_da2)
																					):
																										return True
																return False

											def checkuae(_A    ):
																__lowerCAmelCase            = x.encode()
																if len(_A    ) == 1 and len(_A    ) == 3:
																					__lowerCAmelCase            = (int(e[0]    ) << 1_6) + (int(e[1]    ) << 8) + int(e[2]    )
																					if c >= 0xe28_080 and c <= 0xe2b_07f:
																										return True
																return False

											__lowerCAmelCase            = 0
											__lowerCAmelCase            = []
											while pos < len(_A    ):
																__lowerCAmelCase            = min(len(_A    )	,    pos + self.maxlen + 1    ) if text[pos] == "<" else pos + 3
																__lowerCAmelCase            = []  # (token_id, token, pos)
																for e in range(_A	,    _A	,    -1    ):
																					__lowerCAmelCase            = text[pos:e]
																					if wd in self.vocab:
																										if wd[0] == "<" and len(_A    ) > 2:
																															__lowerCAmelCase            = [(self.vocab[wd], wd, e)]
																															break
																										else:
																															candidates.append((self.vocab[wd], wd, e)    )
																if len(_A    ) > 0:
																					# the smallest token_id is adopted
																					__lowerCAmelCase     ,							__lowerCAmelCase     ,							__lowerCAmelCase            = sorted(_A	,    key=lambda _A    : x[0]    )[0]
																					result.append(_A    )
																					__lowerCAmelCase            = e
																else:
																					__lowerCAmelCase            = pos + 1
																					__lowerCAmelCase            = text[pos:end]
																					if check_simbol(_A    ):
																										result.append("<KIGOU>"    )
																					elif checkuae(_A    ):
																										result.append("<U2000U2BFF>"    )
																					else:
																										for i in wd.encode("utf-8"    ):
																															result.append("<|byte%d|>" % i    )
																					__lowerCAmelCase            = end
											return result






						def        __SCREAMING_SNAKE_CASE(     self	,    _A	,    _A="\n"    ):




											"""simple docstring"""



											__lowerCAmelCase            = []
											__lowerCAmelCase            = []
											__lowerCAmelCase            = self.ids_to_tokens[index][0]
											if word[:6] == "<|byte" and word[-2:] == "|>":
																byte_tokens.append(int(word[6:-2]    )    )
											else:
																if len(_A    ) > 0:
																					words.append(bytearray(_A    ).decode("utf-8"	,    errors="replace"    )    )
																					__lowerCAmelCase            = []
																if word[:7] == "<|emoji" and word[-2:] == "|>":
																					words.append(self.emoji["emoji_inv"][word]    )
																elif word == "<SP>":
																					words.append(" "    )
																elif word == "<BR>":
																					words.append(_A    )
																elif word == "<TAB>":
																					words.append("\t"    )
																elif word == "<BLOCK>":
																					words.append("▀"    )
																elif word == "<KIGOU>":
																					words.append("ǀ"    )
																elif word == "<U2000U2BFF>":
																					words.append("‖"    )
																else:
																					words.append(_A    )
											if len(_A    ) > 0:
																words.append(bytearray(_A    ).decode("utf-8"	,    errors="replace"    )    )
											__lowerCAmelCase            = "".join(_A    )
											return text










def 			_a   (		SCREAMING_SNAKE_CASE_							:       int   ,   SCREAMING_SNAKE_CASE_							:       int  ):
					if b == 0:
										return 1
					if (b % 2) == 0:
										return actual_power(SCREAMING_SNAKE_CASE_   ,   int(b / 2  )  ) * actual_power(SCREAMING_SNAKE_CASE_   ,   int(b / 2  )  )
					else:
										return a * actual_power(SCREAMING_SNAKE_CASE_   ,   int(b / 2  )  ) * actual_power(SCREAMING_SNAKE_CASE_   ,   int(b / 2  )  )







def 			_a   (		SCREAMING_SNAKE_CASE_							:       int   ,   SCREAMING_SNAKE_CASE_							:       int  ):
					if b < 0:
										return 1 / actual_power(SCREAMING_SNAKE_CASE_   ,   SCREAMING_SNAKE_CASE_  )
					return actual_power(SCREAMING_SNAKE_CASE_   ,   SCREAMING_SNAKE_CASE_  )


if __name__ == "__main__":
					print(power(-2, -3))




