





import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation


_lowerCamelCase 					=      logging.get_logger(__name__)

_lowerCamelCase 					=      {'vocab_file': 'vocab.txt', 'emoji_file': 'emoji.json'}

_lowerCamelCase 					=      {
    'vocab_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt',
    },
    'emoji_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json',
    },
}

_lowerCamelCase 					=      {
    'abeja/gpt-neox-japanese-2.7b': 2048,
}






def 					SCREAMING_SNAKE_CASE__				(							UpperCamelCase__:				Union[str, Any]       , UpperCamelCase__:				Dict     ):



		with open(__a       , """r"""       , encoding="""utf-8"""     ) as f:
				SCREAMING_SNAKE_CASE__								=   json.loads(f.read()     )

		SCREAMING_SNAKE_CASE__								=   collections.OrderedDict()
		SCREAMING_SNAKE_CASE__								=   collections.OrderedDict()
		SCREAMING_SNAKE_CASE__								=   collections.OrderedDict()
		with open(__a       , """r"""       , encoding="""utf-8"""     ) as f:
				SCREAMING_SNAKE_CASE__								=   f.readlines()
		SCREAMING_SNAKE_CASE__								=   [[t.rstrip("""\n"""     )] if (t == ',' or ',' not in t) else t.rstrip("""\n"""     ).split(""","""     ) for t in token]
		for idx, b in enumerate(__a     ):
				SCREAMING_SNAKE_CASE__								=   b
				SCREAMING_SNAKE_CASE__								=   idx
				for wd in b:
						SCREAMING_SNAKE_CASE__								=   idx

		return vocab, raw_vocab, ids_to_tokens, emoji




class 	UpperCamelCase_							(  SCREAMING_SNAKE_CASE_     ):
							lowerCamelCase_							     =       VOCAB_FILES_NAMES
							lowerCamelCase_							     =       PRETRAINED_VOCAB_FILES_MAP
							lowerCamelCase_							     =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCamelCase_							     =       ["""input_ids""", """attention_mask"""]


							def __init__(				self     :int				,					__A     :Tuple				,					__A     :List[Any]				,					__A     :int="<|endoftext|>"				,					__A     :str="<|endoftext|>"				,					__A     :int="<|startoftext|>"				,					__A     :Dict="<|endoftext|>"				,					__A     :int=False				,					**__A     :Tuple				,					) -> int:



									"""simple docstring"""



									super().__init__(
									    unk_token=__A				,					pad_token=__A				,					bos_token=__A				,					eos_token=__A				,					do_clean_text=__A				,					**__A				,					)
									if not os.path.isfile(__A		):
											raise ValueError(
											    f'''Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'''
											    """ model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"""		)
									if not os.path.isfile(__A		):
											raise ValueError(
											    f'''Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'''
											    """ pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"""		)
									SCREAMING_SNAKE_CASE__								=   do_clean_text
									SCREAMING_SNAKE_CASE__								=   load_vocab_and_emoji(__A				,					__A		)
									SCREAMING_SNAKE_CASE__								=   SubWordJapaneseTokenizer(
									    vocab=self.vocab				,					ids_to_tokens=self.ids_to_tokens				,					emoji=self.emoji		)


							@property
							def   _snake_case	(				self     :str		) -> int:



									"""simple docstring"""



									return len(self.raw_vocab		)


							def   _snake_case	(				self     :Union[str, Any]		) -> List[str]:



									"""simple docstring"""



									return dict(self.raw_vocab				,					**self.added_tokens_encoder		)


							def   _snake_case	(				self     :str				,					__A     :List[Any]		) -> Optional[int]:



									"""simple docstring"""



									return self.subword_tokenizer.tokenize(__A				,					clean=self.do_clean_text		)


							def   _snake_case	(				self     :Union[str, Any]				,					__A     :Dict		) -> Optional[Any]:



									"""simple docstring"""



									return self.vocab.get(__A				,					self.vocab.get(self.unk_token		)		)


							def   _snake_case	(				self     :Tuple				,					__A     :Optional[Any]		) -> List[str]:



									"""simple docstring"""



									return self.subword_tokenizer.convert_id_to_token(__A		)


							def   _snake_case	(				self     :Any				,					__A     :Dict		) -> List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   ''.join(__A		).strip()
									return out_string


							def   _snake_case	(				self     :Optional[Any]				,					__A     :Dict		) -> Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   []
									for is_user, text in conversation.iter_texts():
											input_ids.extend(self.encode(__A				,					add_special_tokens=__A		) + [self.eos_token_id]		)

									if len(__A		) > self.model_max_length:
											SCREAMING_SNAKE_CASE__								=   input_ids[-self.model_max_length :]
									return input_ids


							def   _snake_case	(				self     :int				,					__A     :List[Any]				,					__A     :Optional[Any] = None		) -> Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   0
									if os.path.isdir(__A		):
											SCREAMING_SNAKE_CASE__								=   os.path.join(
											    __A				,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)
											SCREAMING_SNAKE_CASE__								=   os.path.join(
											    __A				,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""emoji_file"""]		)
									else:
											SCREAMING_SNAKE_CASE__								=   (
											    (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['vocab_file']
											)
											SCREAMING_SNAKE_CASE__								=   (
											    (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['emoji_file']
											)
									with open(__A				,					"""w"""				,					encoding="""utf-8"""		) as writer:
											for token_index, token in self.ids_to_tokens.items():
													if index != token_index:
															logger.warning(
															    f'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
															    """ Please check that the vocabulary is not corrupted!"""		)
															SCREAMING_SNAKE_CASE__								=   token_index
													writer.write(""",""".join(__A		) + """\n"""		)
													index += 1
									with open(__A				,					"""w"""				,					encoding="""utf-8"""		) as writer:
											json.dump(self.emoji				,					__A		)
									return vocab_file, emoji_file




class 	UpperCamelCase_							(  SCREAMING_SNAKE_CASE_     ):


							def __init__(				self     :str				,					__A     :Optional[Any]				,					__A     :Dict				,					__A     :List[Any]		) -> Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   vocab  # same as swe
									SCREAMING_SNAKE_CASE__								=   ids_to_tokens  # same as bpe
									SCREAMING_SNAKE_CASE__								=   emoji
									SCREAMING_SNAKE_CASE__								=   np.max([len(__A		) for w in self.vocab.keys()]		)
									SCREAMING_SNAKE_CASE__								=   re.compile(r"""(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)"""		)
									SCREAMING_SNAKE_CASE__								=   re.compile(r"""[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*"""		)
									SCREAMING_SNAKE_CASE__								=   re.compile(r"""[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}"""		)
									SCREAMING_SNAKE_CASE__								=   re.compile(
									    r"""([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"""		)
									SCREAMING_SNAKE_CASE__								=   re.compile(
									    r"""(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"""		)
									SCREAMING_SNAKE_CASE__								=   re.compile(
									    r"""((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*"""		)
									SCREAMING_SNAKE_CASE__								=   '─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'
									SCREAMING_SNAKE_CASE__								=   '▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'
									SCREAMING_SNAKE_CASE__								=   str.maketrans({k: """<BLOCK>""" for k in keisen + blocks}		)


							def __len__(				self     :str		) -> Optional[Any]:



									"""simple docstring"""



									return len(self.ids_to_tokens		)


							def   _snake_case	(				self     :List[Any]				,					__A     :Optional[Any]		) -> List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   self.content_repattera.sub("""<URL>"""				,					__A		)
									SCREAMING_SNAKE_CASE__								=   self.content_repattera.sub("""<EMAIL>"""				,					__A		)
									SCREAMING_SNAKE_CASE__								=   self.content_repattera.sub("""<TEL>"""				,					__A		)
									SCREAMING_SNAKE_CASE__								=   self.content_repattera.sub("""<DATE>"""				,					__A		)
									SCREAMING_SNAKE_CASE__								=   self.content_repattera.sub("""<DATE>"""				,					__A		)
									SCREAMING_SNAKE_CASE__								=   self.content_repattera.sub("""<PRICE>"""				,					__A		)
									SCREAMING_SNAKE_CASE__								=   content.translate(self.content_transa		)
									while "<BLOCK><BLOCK>" in content:
											SCREAMING_SNAKE_CASE__								=   content.replace("""<BLOCK><BLOCK>"""				,					"""<BLOCK>"""		)
									return content


							def   _snake_case	(				self     :Dict				,					__A     :Dict				,					__A     :Tuple=False		) -> Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   text.replace(""" """				,					"""<SP>"""		)
									SCREAMING_SNAKE_CASE__								=   text.replace("""　"""				,					"""<SP>"""		)
									SCREAMING_SNAKE_CASE__								=   text.replace("""\r\n"""				,					"""<BR>"""		)
									SCREAMING_SNAKE_CASE__								=   text.replace("""\n"""				,					"""<BR>"""		)
									SCREAMING_SNAKE_CASE__								=   text.replace("""\r"""				,					"""<BR>"""		)
									SCREAMING_SNAKE_CASE__								=   text.replace("""\t"""				,					"""<TAB>"""		)
									SCREAMING_SNAKE_CASE__								=   text.replace("""—"""				,					"""ー"""		)
									SCREAMING_SNAKE_CASE__								=   text.replace("""−"""				,					"""ー"""		)
									for k, v in self.emoji["emoji"].items():
											if k in text:
													SCREAMING_SNAKE_CASE__								=   text.replace(__A				,					__A		)
									if clean:
											SCREAMING_SNAKE_CASE__								=   self.clean_text(__A		)

									def check_simbol(__A     :str		):
											SCREAMING_SNAKE_CASE__								=   x.encode()
											if len(__A		) == 1 and len(__A		) == 2:
													SCREAMING_SNAKE_CASE__								=   (int(e[0]		) << 8) + int(e[1]		)
													if (
													    (c >= 0XC2_A1 and c <= 0XC2_BF)
													    or (c >= 0XC7_80 and c <= 0XC7_83)
													    or (c >= 0XCA_B9 and c <= 0XCB_BF)
													    or (c >= 0XCC_80 and c <= 0XCD_A2)
													):
															return True
											return False

									def checkuae(__A     :str		):
											SCREAMING_SNAKE_CASE__								=   x.encode()
											if len(__A		) == 1 and len(__A		) == 3:
													SCREAMING_SNAKE_CASE__								=   (int(e[0]		) << 16) + (int(e[1]		) << 8) + int(e[2]		)
													if c >= 0XE2_80_80 and c <= 0XE2_B0_7F:
															return True
											return False

									SCREAMING_SNAKE_CASE__								=   0
									SCREAMING_SNAKE_CASE__								=   []
									while pos < len(__A		):
											SCREAMING_SNAKE_CASE__								=   min(len(__A		)				,					pos + self.maxlen + 1		) if text[pos] == '<' else pos + 3
											SCREAMING_SNAKE_CASE__								=   []  # (token_id, token, pos)
											for e in range(__A				,					__A				,					-1		):
													SCREAMING_SNAKE_CASE__								=   text[pos:e]
													if wd in self.vocab:
															if wd[0] == "<" and len(__A		) > 2:
																	SCREAMING_SNAKE_CASE__								=   [(self.vocab[wd], wd, e)]
																	break
															else:
																	candidates.append((self.vocab[wd], wd, e)		)
											if len(__A		) > 0:
													# the smallest token_id is adopted
													SCREAMING_SNAKE_CASE__								=   sorted(__A				,					key=lambda __A		: x[0]		)[0]
													result.append(__A		)
													SCREAMING_SNAKE_CASE__								=   e
											else:
													SCREAMING_SNAKE_CASE__								=   pos + 1
													SCREAMING_SNAKE_CASE__								=   text[pos:end]
													if check_simbol(__A		):
															result.append("""<KIGOU>"""		)
													elif checkuae(__A		):
															result.append("""<U2000U2BFF>"""		)
													else:
															for i in wd.encode("""utf-8"""		):
																	result.append("""<|byte%d|>""" % i		)
													SCREAMING_SNAKE_CASE__								=   end
									return result


							def   _snake_case	(				self     :List[str]				,					__A     :int				,					__A     :Any="\n"		) -> int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   []
									SCREAMING_SNAKE_CASE__								=   []
									SCREAMING_SNAKE_CASE__								=   self.ids_to_tokens[index][0]
									if word[:6] == "<|byte" and word[-2:] == "|>":
											byte_tokens.append(int(word[6:-2]		)		)
									else:
											if len(__A		) > 0:
													words.append(bytearray(__A		).decode("""utf-8"""				,					errors="""replace"""		)		)
													SCREAMING_SNAKE_CASE__								=   []
											if word[:7] == "<|emoji" and word[-2:] == "|>":
													words.append(self.emoji["""emoji_inv"""][word]		)
											elif word == "<SP>":
													words.append(""" """		)
											elif word == "<BR>":
													words.append(__A		)
											elif word == "<TAB>":
													words.append("""\t"""		)
											elif word == "<BLOCK>":
													words.append("""▀"""		)
											elif word == "<KIGOU>":
													words.append("""ǀ"""		)
											elif word == "<U2000U2BFF>":
													words.append("""‖"""		)
											else:
													words.append(__A		)
									if len(__A		) > 0:
											words.append(bytearray(__A		).decode("""utf-8"""				,					errors="""replace"""		)		)
									SCREAMING_SNAKE_CASE__								=   ''.join(__A		)
									return text




"""simple docstring"""







def     UpperCAmelCase_					(				__a   :			float      ,							__a   :			float						):




			'''simple docstring'''



			if density <= 0:
						raise ValueError('Impossible fluid density'						)
			if bulk_modulus <= 0:
						raise ValueError('Impossible bulk modulus'						)

			return (bulk_modulus / density) ** 0.5


if __name__ == "__main__":
	import doctest

	doctest.testmod()





