


import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation


_lowercase       		= logging.get_logger(__name__)

_lowercase       		= {'vocab_file': 'vocab.txt', 'emoji_file': 'emoji.json'}

_lowercase       		= {
    'vocab_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt',
    },
    'emoji_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json',
    },
}

_lowercase       		= {
    'abeja/gpt-neox-japanese-2.7b': 2048,
}

def   UpperCamelCase			(      snake_case__						,  snake_case__):
						with open(A__						,  "r"						,  encoding="utf-8") as f:
												lowerCAmelCase_	:     List[Any]          = json.loads(f.read())

						lowerCAmelCase_	:     List[Any]          = collections.OrderedDict()
						lowerCAmelCase_	:     Optional[int]          = collections.OrderedDict()
						lowerCAmelCase_	:     Optional[int]          = collections.OrderedDict()
						with open(A__						,  "r"						,  encoding="utf-8") as f:
												lowerCAmelCase_	:     List[str]          = f.readlines()
						lowerCAmelCase_	:     Tuple          = [[t.rstrip("\n")] if (t == "," or "," not in t) else t.rstrip("\n").split(",") for t in token]
						for idx, b in enumerate(A__):
												lowerCAmelCase_	:     List[Any]          = b
												lowerCAmelCase_	:     Optional[int]          = idx
												for wd in b:
																		lowerCAmelCase_	:     str          = idx

						return vocab, raw_vocab, ids_to_tokens, emoji

class __snake_case		( __snake_case ):





		"""simple docstring"""







		UpperCamelCase_				      =   VOCAB_FILES_NAMES
		UpperCamelCase_				      =   PRETRAINED_VOCAB_FILES_MAP
		UpperCamelCase_				      =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		UpperCamelCase_				      =   ['input_ids', 'attention_mask']



		def __init__(							self    :   Optional[int]						,lowerCAmelCase__    :   Dict						,lowerCAmelCase__    :   Dict						,lowerCAmelCase__    :   Optional[int]="<|endoftext|>"						,lowerCAmelCase__    :   int="<|endoftext|>"						,lowerCAmelCase__    :   Dict="<|startoftext|>"						,lowerCAmelCase__    :   Any="<|endoftext|>"						,lowerCAmelCase__    :   List[str]=False						,**lowerCAmelCase__    :   str						,)      ->							List[str]:




								'''simple docstring'''

								super().__init__(
								    unk_token=__lowerCamelCase						,pad_token=__lowerCamelCase						,bos_token=__lowerCamelCase						,eos_token=__lowerCamelCase						,do_clean_text=__lowerCamelCase						,**__lowerCamelCase						,)
								if not os.path.isfile(__lowerCamelCase      ):
														raise ValueError(
														    f'''Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'''
														    " model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"      )
								if not os.path.isfile(__lowerCamelCase      ):
														raise ValueError(
														    f'''Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'''
														    " pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"      )
								lowerCAmelCase_	:     List[str]          = do_clean_text
								lowerCAmelCase_    ,					lowerCAmelCase_    ,					lowerCAmelCase_    ,					lowerCAmelCase_	:     Union[str, Any]          = load_vocab_and_emoji(__lowerCamelCase						,__lowerCamelCase      )
								lowerCAmelCase_	:     Tuple          = SubWordJapaneseTokenizer(
								    vocab=self.vocab						,ids_to_tokens=self.ids_to_tokens						,emoji=self.emoji      )



		@property
		def  UpperCAmelCase_							(							self    :   Optional[Any]      )      ->							Union[str, Any]:




								'''simple docstring'''

								return len(self.raw_vocab      )



		def  UpperCAmelCase_							(							self    :   Optional[Any]      )      ->							str:




								'''simple docstring'''

								return dict(self.raw_vocab						,**self.added_tokens_encoder      )



		def  UpperCAmelCase_							(							self    :   List[Any]						,lowerCAmelCase__    :   str      )      ->							Tuple:




								'''simple docstring'''

								return self.subword_tokenizer.tokenize(__lowerCamelCase						,clean=self.do_clean_text      )



		def  UpperCAmelCase_							(							self    :   str						,lowerCAmelCase__    :   List[Any]      )      ->							Union[str, Any]:




								'''simple docstring'''

								return self.vocab.get(__lowerCamelCase						,self.vocab.get(self.unk_token      )      )



		def  UpperCAmelCase_							(							self    :   int						,lowerCAmelCase__    :   Dict      )      ->							Tuple:




								'''simple docstring'''

								return self.subword_tokenizer.convert_id_to_token(__lowerCamelCase      )



		def  UpperCAmelCase_							(							self    :   Any						,lowerCAmelCase__    :   Optional[Any]      )      ->							Tuple:




								'''simple docstring'''

								lowerCAmelCase_	:     int          = "".join(__lowerCamelCase      ).strip()
								return out_string



		def  UpperCAmelCase_							(							self    :   str						,lowerCAmelCase__    :   "Conversation"      )      ->							Tuple:




								'''simple docstring'''

								lowerCAmelCase_	:     int          = []
								for is_user, text in conversation.iter_texts():
														input_ids.extend(self.encode(__lowerCamelCase						,add_special_tokens=__lowerCamelCase      ) + [self.eos_token_id]      )

								if len(__lowerCamelCase      ) > self.model_max_length:
														lowerCAmelCase_	:     List[Any]          = input_ids[-self.model_max_length :]
								return input_ids



		def  UpperCAmelCase_							(							self    :   int						,lowerCAmelCase__    :   str						,lowerCAmelCase__    :   Optional[str] = None      )      ->							Optional[int]:




								'''simple docstring'''

								lowerCAmelCase_	:     Optional[Any]          = 0
								if os.path.isdir(__lowerCamelCase      ):
														lowerCAmelCase_	:     List[str]          = os.path.join(
														    __lowerCamelCase						,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]      )
														lowerCAmelCase_	:     List[str]          = os.path.join(
														    __lowerCamelCase						,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["emoji_file"]      )
								else:
														lowerCAmelCase_	:     Any          = (
														    (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["vocab_file"]
														)
														lowerCAmelCase_	:     int          = (
														    (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["emoji_file"]
														)
								with open(__lowerCamelCase						,"w"						,encoding="utf-8"      ) as writer:
														for token_index, token in self.ids_to_tokens.items():
																				if index != token_index:
																										logger.warning(
																										    f'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
																										    " Please check that the vocabulary is not corrupted!"      )
																										lowerCAmelCase_	:     int          = token_index
																				writer.write(",".join(__lowerCamelCase      ) + "\n"      )
																				index += 1
								with open(__lowerCamelCase						,"w"						,encoding="utf-8"      ) as writer:
														json.dump(self.emoji						,__lowerCamelCase      )
								return vocab_file, emoji_file






class __snake_case		( __snake_case ):





		"""simple docstring"""







		def __init__(							self    :   Any						,lowerCAmelCase__    :   Optional[Any]						,lowerCAmelCase__    :   str						,lowerCAmelCase__    :   List[str]      )      ->							List[str]:




								'''simple docstring'''

								lowerCAmelCase_	:     List[str]          = vocab  # same as swe
								lowerCAmelCase_	:     Tuple          = ids_to_tokens  # same as bpe
								lowerCAmelCase_	:     List[Any]          = emoji
								lowerCAmelCase_	:     Union[str, Any]          = np.max([len(__lowerCamelCase      ) for w in self.vocab.keys()]      )
								lowerCAmelCase_	:     Union[str, Any]          = re.compile(R"(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)"      )
								lowerCAmelCase_	:     Tuple          = re.compile(R"[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*"      )
								lowerCAmelCase_	:     int          = re.compile(R"[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}"      )
								lowerCAmelCase_	:     Any          = re.compile(
								    R"([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"      )
								lowerCAmelCase_	:     Optional[int]          = re.compile(
								    R"(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"      )
								lowerCAmelCase_	:     int          = re.compile(
								    R"((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*"      )
								lowerCAmelCase_	:     Optional[Any]          = "─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"
								lowerCAmelCase_	:     Optional[int]          = "▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"
								lowerCAmelCase_	:     List[str]          = str.maketrans({k: "<BLOCK>" for k in keisen + blocks}      )



		def __len__(							self    :   Any      )      ->							Dict:




								'''simple docstring'''

								return len(self.ids_to_tokens      )



		def  UpperCAmelCase_							(							self    :   int						,lowerCAmelCase__    :   Tuple      )      ->							Union[str, Any]:




								'''simple docstring'''

								lowerCAmelCase_	:     int          = self.content_repattera.sub("<URL>"						,__lowerCamelCase      )
								lowerCAmelCase_	:     List[Any]          = self.content_repattera.sub("<EMAIL>"						,__lowerCamelCase      )
								lowerCAmelCase_	:     int          = self.content_repattera.sub("<TEL>"						,__lowerCamelCase      )
								lowerCAmelCase_	:     int          = self.content_repattera.sub("<DATE>"						,__lowerCamelCase      )
								lowerCAmelCase_	:     Optional[int]          = self.content_repattera.sub("<DATE>"						,__lowerCamelCase      )
								lowerCAmelCase_	:     List[Any]          = self.content_repattera.sub("<PRICE>"						,__lowerCamelCase      )
								lowerCAmelCase_	:     List[str]          = content.translate(self.content_transa      )
								while "<BLOCK><BLOCK>" in content:
														lowerCAmelCase_	:     Any          = content.replace("<BLOCK><BLOCK>"						,"<BLOCK>"      )
								return content



		def  UpperCAmelCase_							(							self    :   Dict						,lowerCAmelCase__    :   Optional[Any]						,lowerCAmelCase__    :   Optional[Any]=False      )      ->							List[Any]:




								'''simple docstring'''

								lowerCAmelCase_	:     str          = text.replace(" "						,"<SP>"      )
								lowerCAmelCase_	:     Any          = text.replace("　"						,"<SP>"      )
								lowerCAmelCase_	:     Dict          = text.replace("\r\n"						,"<BR>"      )
								lowerCAmelCase_	:     Optional[Any]          = text.replace("\n"						,"<BR>"      )
								lowerCAmelCase_	:     Optional[int]          = text.replace("\r"						,"<BR>"      )
								lowerCAmelCase_	:     str          = text.replace("\t"						,"<TAB>"      )
								lowerCAmelCase_	:     List[str]          = text.replace("—"						,"ー"      )
								lowerCAmelCase_	:     Dict          = text.replace("−"						,"ー"      )
								for k, v in self.emoji["emoji"].items():
														if k in text:
																				lowerCAmelCase_	:     Dict          = text.replace(__lowerCamelCase						,__lowerCamelCase      )
								if clean:
														lowerCAmelCase_	:     List[Any]          = self.clean_text(__lowerCamelCase      )

								def check_simbol(lowerCAmelCase__    :   Any      ):
														lowerCAmelCase_	:     List[Any]          = x.encode()
														if len(__lowerCamelCase      ) == 1 and len(__lowerCamelCase      ) == 2:
																				lowerCAmelCase_	:     Tuple          = (int(e[0]      ) << 8) + int(e[1]      )
																				if (
																				    (c >= 0Xc2a1 and c <= 0Xc2bf)
																				    or (c >= 0Xc780 and c <= 0Xc783)
																				    or (c >= 0Xcab9 and c <= 0Xcbbf)
																				    or (c >= 0Xcc80 and c <= 0Xcda2)
																				):
																										return True
														return False

								def checkuae(lowerCAmelCase__    :   Dict      ):
														lowerCAmelCase_	:     List[str]          = x.encode()
														if len(__lowerCamelCase      ) == 1 and len(__lowerCamelCase      ) == 3:
																				lowerCAmelCase_	:     Tuple          = (int(e[0]      ) << 16) + (int(e[1]      ) << 8) + int(e[2]      )
																				if c >= 0Xe28080 and c <= 0Xe2b07f:
																										return True
														return False

								lowerCAmelCase_	:     Dict          = 0
								lowerCAmelCase_	:     Any          = []
								while pos < len(__lowerCamelCase      ):
														lowerCAmelCase_	:     Any          = min(len(__lowerCamelCase      )						,pos + self.maxlen + 1      ) if text[pos] == "<" else pos + 3
														lowerCAmelCase_	:     Tuple          = []  # (token_id, token, pos)
														for e in range(__lowerCamelCase						,__lowerCamelCase						,-1      ):
																				lowerCAmelCase_	:     Union[str, Any]          = text[pos:e]
																				if wd in self.vocab:
																										if wd[0] == "<" and len(__lowerCamelCase      ) > 2:
																																lowerCAmelCase_	:     List[Any]          = [(self.vocab[wd], wd, e)]
																																break
																										else:
																																candidates.append((self.vocab[wd], wd, e)      )
														if len(__lowerCamelCase      ) > 0:
																				# the smallest token_id is adopted
																				lowerCAmelCase_    ,					lowerCAmelCase_    ,					lowerCAmelCase_	:     List[Any]          = sorted(__lowerCamelCase						,key=lambda lowerCAmelCase__      : x[0]      )[0]
																				result.append(__lowerCamelCase      )
																				lowerCAmelCase_	:     Optional[Any]          = e
														else:
																				lowerCAmelCase_	:     Union[str, Any]          = pos + 1
																				lowerCAmelCase_	:     List[str]          = text[pos:end]
																				if check_simbol(__lowerCamelCase      ):
																										result.append("<KIGOU>"      )
																				elif checkuae(__lowerCamelCase      ):
																										result.append("<U2000U2BFF>"      )
																				else:
																										for i in wd.encode("utf-8"      ):
																																result.append("<|byte%d|>" % i      )
																				lowerCAmelCase_	:     Dict          = end
								return result



		def  UpperCAmelCase_							(							self    :   Optional[int]						,lowerCAmelCase__    :   Optional[int]						,lowerCAmelCase__    :   Optional[Any]="\n"      )      ->							Union[str, Any]:




								'''simple docstring'''

								lowerCAmelCase_	:     List[str]          = []
								lowerCAmelCase_	:     Optional[Any]          = []
								lowerCAmelCase_	:     int          = self.ids_to_tokens[index][0]
								if word[:6] == "<|byte" and word[-2:] == "|>":
														byte_tokens.append(int(word[6:-2]      )      )
								else:
														if len(__lowerCamelCase      ) > 0:
																				words.append(bytearray(__lowerCamelCase      ).decode("utf-8"						,errors="replace"      )      )
																				lowerCAmelCase_	:     str          = []
														if word[:7] == "<|emoji" and word[-2:] == "|>":
																				words.append(self.emoji["emoji_inv"][word]      )
														elif word == "<SP>":
																				words.append(" "      )
														elif word == "<BR>":
																				words.append(__lowerCamelCase      )
														elif word == "<TAB>":
																				words.append("\t"      )
														elif word == "<BLOCK>":
																				words.append("▀"      )
														elif word == "<KIGOU>":
																				words.append("ǀ"      )
														elif word == "<U2000U2BFF>":
																				words.append("‖"      )
														else:
																				words.append(__lowerCamelCase      )
								if len(__lowerCamelCase      ) > 0:
														words.append(bytearray(__lowerCamelCase      ).decode("utf-8"						,errors="replace"      )      )
								lowerCAmelCase_	:     Optional[int]          = "".join(__lowerCamelCase      )
								return text





def 	__a     (      A__			:  int       ):
    if not isinstance(A__			,  A__       ):
        raise ValueError("Input must be an integer"       )
    if input_num <= 0:
        raise ValueError("Input must be positive"       )
    return sum(
        divisor for divisor in range(1			,  input_num // 2 + 1       ) if input_num % divisor == 0       )


if __name__ == "__main__":
    import doctest

    doctest.testmod()