





import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation


__A     =    logging.get_logger(__name__)

__A     =    {"vocab_file": "vocab.txt", "emoji_file": "emoji.json"}

__A     =    {
    "vocab_file": {
        "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt",
    },
    "emoji_file": {
        "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json",
    },
}

__A     =    {
    "abeja/gpt-neox-japanese-2.7b": 2048,
}





def 					lowerCAmelCase_			(	__a     ,   __a		)					->		Union[str, Any]:







  """simple docstring"""





  with open(__a     ,   "r"     ,   encoding="utf-8"		) as f:
    lowerCamelCase__:				str        =json.loads(f.read()		)

  lowerCamelCase__:				Any        =collections.OrderedDict()
  lowerCamelCase__:				Optional[int]        =collections.OrderedDict()
  lowerCamelCase__:				Dict        =collections.OrderedDict()
  with open(__a     ,   "r"     ,   encoding="utf-8"		) as f:
    lowerCamelCase__:				Tuple        =f.readlines()
  lowerCamelCase__:				List[str]        =[[t.rstrip("\n"		)] if (t == "," or "," not in t) else t.rstrip("\n"		).split(","		) for t in token]
  for idx, b in enumerate(__a		):
    lowerCamelCase__:				Optional[int]        =b
    lowerCamelCase__:				List[str]        =idx
    for wd in b:
      lowerCamelCase__:				str        =idx

  return vocab, raw_vocab, ids_to_tokens, emoji



class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''
 lowercase_		=      VOCAB_FILES_NAMES
 lowercase_		=      PRETRAINED_VOCAB_FILES_MAP
 lowercase_		=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowercase_		=      ["input_ids", "attention_mask"]

 def __init__(self    :       str	,			UpperCAmelCase_    :       int	,			UpperCAmelCase_    :       Dict	,			UpperCAmelCase_    :       Any="<|endoftext|>"	,			UpperCAmelCase_    :       Optional[Any]="<|endoftext|>"	,			UpperCAmelCase_    :       Optional[Any]="<|startoftext|>"	,			UpperCAmelCase_    :       Any="<|endoftext|>"	,			UpperCAmelCase_    :       Optional[int]=False	,			**UpperCAmelCase_    :       Tuple	,			)	->List[str]:



   '''simple docstring'''



   super().__init__(
       unk_token=UpperCAmelCase_	,			pad_token=UpperCAmelCase_	,			bos_token=UpperCAmelCase_	,			eos_token=UpperCAmelCase_	,			do_clean_text=UpperCAmelCase_	,			**UpperCAmelCase_	,			)
   if not os.path.isfile(UpperCAmelCase_):
     raise ValueError(
         F"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
         " model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`")
   if not os.path.isfile(UpperCAmelCase_):
     raise ValueError(
         F"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
         " pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`")
   lowerCamelCase__:				Dict        =do_clean_text
   lowerCamelCase__  , lowerCamelCase__  , lowerCamelCase__  , lowerCamelCase__:				int        =load_vocab_and_emoji(UpperCAmelCase_	,			UpperCAmelCase_)
   lowerCamelCase__:				Any        =SubWordJapaneseTokenizer(
       vocab=self.vocab	,			ids_to_tokens=self.ids_to_tokens	,			emoji=self.emoji)

 @property
 def 							SCREAMING_SNAKE_CASE_ (self    :       Tuple)	->str:



   '''simple docstring'''



   return len(self.raw_vocab)

 def 							SCREAMING_SNAKE_CASE_ (self    :       Optional[Any])	->Tuple:



   '''simple docstring'''



   return dict(self.raw_vocab	,			**self.added_tokens_encoder)

 def 							SCREAMING_SNAKE_CASE_ (self    :       Any	,			UpperCAmelCase_    :       Dict)	->Union[str, Any]:



   '''simple docstring'''



   return self.subword_tokenizer.tokenize(UpperCAmelCase_	,			clean=self.do_clean_text)

 def 							SCREAMING_SNAKE_CASE_ (self    :       Optional[int]	,			UpperCAmelCase_    :       Union[str, Any])	->List[Any]:



   '''simple docstring'''



   return self.vocab.get(UpperCAmelCase_	,			self.vocab.get(self.unk_token))

 def 							SCREAMING_SNAKE_CASE_ (self    :       List[Any]	,			UpperCAmelCase_    :       Any)	->Any:



   '''simple docstring'''



   return self.subword_tokenizer.convert_id_to_token(UpperCAmelCase_)

 def 							SCREAMING_SNAKE_CASE_ (self    :       Optional[Any]	,			UpperCAmelCase_    :       List[Any])	->Tuple:



   '''simple docstring'''



   lowerCamelCase__:				List[Any]        ="".join(UpperCAmelCase_).strip()
   return out_string

 def 							SCREAMING_SNAKE_CASE_ (self    :       str	,			UpperCAmelCase_    :       "Conversation")	->List[int]:



   '''simple docstring'''



   lowerCamelCase__:				List[Any]        =[]
   for is_user, text in conversation.iter_texts():
     input_ids.extend(self.encode(UpperCAmelCase_	,			add_special_tokens=UpperCAmelCase_) + [self.eos_token_id])

   if len(UpperCAmelCase_) > self.model_max_length:
     lowerCamelCase__:				str        =input_ids[-self.model_max_length :]
   return input_ids







 def 							SCREAMING_SNAKE_CASE_ (self    :       Optional[Any]	,			UpperCAmelCase_    :       str	,			UpperCAmelCase_    :       Optional[str] = None)	->Tuple[str]:



   '''simple docstring'''



   lowerCamelCase__:				Tuple        =0
   if os.path.isdir(UpperCAmelCase_):
     lowerCamelCase__:				List[Any]        =os.path.join(
         UpperCAmelCase_	,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])
     lowerCamelCase__:				str        =os.path.join(
         UpperCAmelCase_	,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["emoji_file"])
   else:
     lowerCamelCase__:				Tuple        =(
         (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["vocab_file"]
     )
     lowerCamelCase__:				List[Any]        =(
         (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["emoji_file"]
     )
   with open(UpperCAmelCase_	,			"w"	,			encoding="utf-8") as writer:
     for token_index, token in self.ids_to_tokens.items():
       if index != token_index:
         logger.warning(
             F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
             " Please check that the vocabulary is not corrupted!")
         lowerCamelCase__:				List[Any]        =token_index
       writer.write(",".join(UpperCAmelCase_) + "\n")
       index += 1
   with open(UpperCAmelCase_	,			"w"	,			encoding="utf-8") as writer:
     json.dump(self.emoji	,			UpperCAmelCase_)
   return vocab_file, emoji_file



class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''

 def __init__(self    :       List[str]	,			UpperCAmelCase_    :       Optional[int]	,			UpperCAmelCase_    :       List[str]	,			UpperCAmelCase_    :       Optional[Any])	->Any:



   '''simple docstring'''



   lowerCamelCase__:				Dict        =vocab  # same as swe
   lowerCamelCase__:				Any        =ids_to_tokens  # same as bpe
   lowerCamelCase__:				List[Any]        =emoji
   lowerCamelCase__:				str        =np.max([len(UpperCAmelCase_) for w in self.vocab.keys()])
   lowerCamelCase__:				Union[str, Any]        =re.compile(R"(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)")
   lowerCamelCase__:				List[Any]        =re.compile(R"[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*")
   lowerCamelCase__:				str        =re.compile(R"[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}")
   lowerCamelCase__:				Optional[int]        =re.compile(
       R"([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*")
   lowerCamelCase__:				Any        =re.compile(
       R"(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*")
   lowerCamelCase__:				Any        =re.compile(
       R"((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*")
   lowerCamelCase__:				Optional[int]        ="─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"
   lowerCamelCase__:				Optional[int]        ="▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"
   lowerCamelCase__:				int        =str.maketrans({k: "<BLOCK>" for k in keisen + blocks})

 def __len__(self    :       Union[str, Any])	->Optional[Any]:



   '''simple docstring'''



   return len(self.ids_to_tokens)

 def 							SCREAMING_SNAKE_CASE_ (self    :       List[Any]	,			UpperCAmelCase_    :       Dict)	->Any:



   '''simple docstring'''



   lowerCamelCase__:				List[Any]        =self.content_repattera.sub("<URL>"	,			UpperCAmelCase_)
   lowerCamelCase__:				Tuple        =self.content_repattera.sub("<EMAIL>"	,			UpperCAmelCase_)
   lowerCamelCase__:				List[str]        =self.content_repattera.sub("<TEL>"	,			UpperCAmelCase_)
   lowerCamelCase__:				Tuple        =self.content_repattera.sub("<DATE>"	,			UpperCAmelCase_)
   lowerCamelCase__:				List[Any]        =self.content_repattera.sub("<DATE>"	,			UpperCAmelCase_)
   lowerCamelCase__:				Union[str, Any]        =self.content_repattera.sub("<PRICE>"	,			UpperCAmelCase_)
   lowerCamelCase__:				Any        =content.translate(self.content_transa)
   while "<BLOCK><BLOCK>" in content:
     lowerCamelCase__:				Dict        =content.replace("<BLOCK><BLOCK>"	,			"<BLOCK>")
   return content

 def 							SCREAMING_SNAKE_CASE_ (self    :       List[Any]	,			UpperCAmelCase_    :       Dict	,			UpperCAmelCase_    :       List[str]=False)	->Optional[int]:



   '''simple docstring'''



   lowerCamelCase__:				str        =text.replace(" "	,			"<SP>")
   lowerCamelCase__:				List[str]        =text.replace("　"	,			"<SP>")
   lowerCamelCase__:				Optional[int]        =text.replace("\r\n"	,			"<BR>")
   lowerCamelCase__:				Any        =text.replace("\n"	,			"<BR>")
   lowerCamelCase__:				List[Any]        =text.replace("\r"	,			"<BR>")
   lowerCamelCase__:				Dict        =text.replace("\t"	,			"<TAB>")
   lowerCamelCase__:				Tuple        =text.replace("—"	,			"ー")
   lowerCamelCase__:				Optional[int]        =text.replace("−"	,			"ー")
   for k, v in self.emoji["emoji"].items():
     if k in text:
       lowerCamelCase__:				List[str]        =text.replace(UpperCAmelCase_	,			UpperCAmelCase_)
   if clean:
     lowerCamelCase__:				Union[str, Any]        =self.clean_text(UpperCAmelCase_)

   def check_simbol(UpperCAmelCase_    :       Optional[Any]):
     lowerCamelCase__:				Optional[Any]        =x.encode()
     if len(UpperCAmelCase_) == 1 and len(UpperCAmelCase_) == 2:
       lowerCamelCase__:				Tuple        =(int(e[0]) << 8) + int(e[1])
       if (
           (c >= 0xC2A1 and c <= 0xC2BF)
           or (c >= 0xC780 and c <= 0xC783)
           or (c >= 0xCAB9 and c <= 0xCBBF)
           or (c >= 0xCC80 and c <= 0xCDA2)
       ):
         return True
     return False

   def checkuae(UpperCAmelCase_    :       Optional[int]):
     lowerCamelCase__:				Optional[Any]        =x.encode()
     if len(UpperCAmelCase_) == 1 and len(UpperCAmelCase_) == 3:
       lowerCamelCase__:				Optional[Any]        =(int(e[0]) << 16) + (int(e[1]) << 8) + int(e[2])
       if c >= 0xE2_8080 and c <= 0xE2_B07F:
         return True
     return False

   lowerCamelCase__:				Any        =0
   lowerCamelCase__:				str        =[]
   while pos < len(UpperCAmelCase_):
     lowerCamelCase__:				Tuple        =min(len(UpperCAmelCase_)	,			pos + self.maxlen + 1) if text[pos] == "<" else pos + 3
     lowerCamelCase__:				Optional[int]        =[]  # (token_id, token, pos)
     for e in range(UpperCAmelCase_	,			UpperCAmelCase_	,			-1):
       lowerCamelCase__:				List[Any]        =text[pos:e]
       if wd in self.vocab:
         if wd[0] == "<" and len(UpperCAmelCase_) > 2:
           lowerCamelCase__:				Dict        =[(self.vocab[wd], wd, e)]
           break
         else:
           candidates.append((self.vocab[wd], wd, e))
     if len(UpperCAmelCase_) > 0:
       # the smallest token_id is adopted
       lowerCamelCase__  , lowerCamelCase__  , lowerCamelCase__:				List[Any]        =sorted(UpperCAmelCase_	,			key=lambda UpperCAmelCase_: x[0])[0]
       result.append(UpperCAmelCase_)
       lowerCamelCase__:				Tuple        =e
     else:
       lowerCamelCase__:				Union[str, Any]        =pos + 1
       lowerCamelCase__:				str        =text[pos:end]
       if check_simbol(UpperCAmelCase_):
         result.append("<KIGOU>")
       elif checkuae(UpperCAmelCase_):
         result.append("<U2000U2BFF>")
       else:
         for i in wd.encode("utf-8"):
           result.append("<|byte%d|>" % i)
       lowerCamelCase__:				str        =end
   return result







 def 							SCREAMING_SNAKE_CASE_ (self    :       str	,			UpperCAmelCase_    :       List[Any]	,			UpperCAmelCase_    :       str="\n")	->Optional[int]:



   '''simple docstring'''



   lowerCamelCase__:				str        =[]
   lowerCamelCase__:				List[str]        =[]
   lowerCamelCase__:				Any        =self.ids_to_tokens[index][0]
   if word[:6] == "<|byte" and word[-2:] == "|>":
     byte_tokens.append(int(word[6:-2]))
   else:
     if len(UpperCAmelCase_) > 0:
       words.append(bytearray(UpperCAmelCase_).decode("utf-8"	,			errors="replace"))
       lowerCamelCase__:				Dict        =[]
     if word[:7] == "<|emoji" and word[-2:] == "|>":
       words.append(self.emoji["emoji_inv"][word])
     elif word == "<SP>":
       words.append(" ")
     elif word == "<BR>":
       words.append(UpperCAmelCase_)
     elif word == "<TAB>":
       words.append("\t")
     elif word == "<BLOCK>":
       words.append("▀")
     elif word == "<KIGOU>":
       words.append("ǀ")
     elif word == "<U2000U2BFF>":
       words.append("‖")
     else:
       words.append(UpperCAmelCase_)
   if len(UpperCAmelCase_) > 0:
     words.append(bytearray(UpperCAmelCase_).decode("utf-8"	,			errors="replace"))
   lowerCamelCase__:				Union[str, Any]        ="".join(UpperCAmelCase_)
   return text








from __future__ import annotations

from typing import Any



class        _SCREAMING_SNAKE_CASE						:







 '''simple docstring'''

 def __init__(self    :       Any	,			UpperCAmelCase_    :       int)	->None:



   '''simple docstring'''



   lowerCamelCase__:				int        =num_of_nodes
   lowerCamelCase__:				list[list[int]]        =[]
   lowerCamelCase__:				dict[int, int]        ={}

 def 							SCREAMING_SNAKE_CASE_ (self    :       Tuple	,			UpperCAmelCase_    :       int	,			UpperCAmelCase_    :       int	,			UpperCAmelCase_    :       int)	->None:



   '''simple docstring'''



   self.m_edges.append([u_node, v_node, weight])

 def 							SCREAMING_SNAKE_CASE_ (self    :       List[str]	,			UpperCAmelCase_    :       int)	->int:



   '''simple docstring'''



   if self.m_component[u_node] == u_node:
     return u_node
   return self.find_component(self.m_component[u_node])

 def 							SCREAMING_SNAKE_CASE_ (self    :       Union[str, Any]	,			UpperCAmelCase_    :       int)	->None:



   '''simple docstring'''



   if self.m_component[u_node] != u_node:
     for k in self.m_component:
       lowerCamelCase__:				Dict        =self.find_component(UpperCAmelCase_)

 def 							SCREAMING_SNAKE_CASE_ (self    :       Union[str, Any]	,			UpperCAmelCase_    :       list[int]	,			UpperCAmelCase_    :       int	,			UpperCAmelCase_    :       int)	->None:



   '''simple docstring'''



   if component_size[u_node] <= component_size[v_node]:
     lowerCamelCase__:				Optional[int]        =v_node
     component_size[v_node] += component_size[u_node]
     self.set_component(UpperCAmelCase_)

   elif component_size[u_node] >= component_size[v_node]:
     lowerCamelCase__:				Tuple        =self.find_component(UpperCAmelCase_)
     component_size[u_node] += component_size[v_node]
     self.set_component(UpperCAmelCase_)







 def 							SCREAMING_SNAKE_CASE_ (self    :       List[str])	->None:



   '''simple docstring'''



   lowerCamelCase__:				List[Any]        =[]
   lowerCamelCase__:				List[str]        =0

   lowerCamelCase__:				list[Any]        =[-1] * self.m_num_of_nodes

   # A list of components (initialized to all of the nodes)
   for node in range(self.m_num_of_nodes):
     self.m_component.update({node: node})
     component_size.append(1)

   lowerCamelCase__:				List[str]        =self.m_num_of_nodes

   while num_of_components > 1:
     for edge in self.m_edges:
       lowerCamelCase__  , lowerCamelCase__  , lowerCamelCase__:				Tuple        =edge

       lowerCamelCase__:				List[Any]        =self.m_component[u]
       lowerCamelCase__:				str        =self.m_component[v]

       if u_component != v_component:

         for component in (u_component, v_component):
           if (
               minimum_weight_edge[component] == -1
               or minimum_weight_edge[component][2] > w
           ):
             lowerCamelCase__:				Union[str, Any]        =[u, v, w]

     for edge in minimum_weight_edge:
       if isinstance(UpperCAmelCase_	,			UpperCAmelCase_):
         lowerCamelCase__  , lowerCamelCase__  , lowerCamelCase__:				Tuple        =edge

         lowerCamelCase__:				str        =self.m_component[u]
         lowerCamelCase__:				Any        =self.m_component[v]

         if u_component != v_component:
           mst_weight += w
           self.union(UpperCAmelCase_	,			UpperCAmelCase_	,			UpperCAmelCase_)
           print(F"""Added edge [{u} - {v}]\nAdded weight: {w}\n""")
           num_of_components -= 1

     lowerCamelCase__:				Tuple        =[-1] * self.m_num_of_nodes
   print(F"""The total weight of the minimal spanning tree is: {mst_weight}""")





def 					lowerCAmelCase_			(	)					->		None:







  """simple docstring"""







if __name__ == "__main__":
     import doctest

     doctest.testmod()

