


'''simple docstring'''






import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


__A   :    Optional[int]											= logging.get_logger(__name__)

__A   :    Optional[int]											= {"vocab_file": "vocab.txt", "emoji_file": "emoji.json"}

__A   :    int											= {
    "vocab_file": {
        "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt",
    },
    "emoji_file": {
        "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json",
    },
}

__A   :    int											= {
    "abeja/gpt-neox-japanese-2.7b": 2048,
}







def 							UpperCamelCase_     (       A__   :  List[Any]				,       A__   :  List[Any]   ):
  '''simple docstring'''





  with open(A__				,       """r"""				,       encoding="""utf-8"""   ) as f:
    lowerCAmelCase_     : List[str]     =  json.loads(f.read()   )

  lowerCAmelCase_     : Optional[Any]     =  collections.OrderedDict()
  lowerCAmelCase_     : Any     =  collections.OrderedDict()
  lowerCAmelCase_     : Union[str, Any]     =  collections.OrderedDict()
  with open(A__				,       """r"""				,       encoding="""utf-8"""   ) as f:
    lowerCAmelCase_     : Tuple     =  f.readlines()
  lowerCAmelCase_     : List[str]     =  [[t.rstrip("""\n"""   )] if (t == """,""" or """,""" not in t) else t.rstrip("""\n"""   ).split(""","""   ) for t in token]
  for idx, b in enumerate(A__   ):
    lowerCAmelCase_     : str     =  b
    lowerCAmelCase_     : List[str]     =  idx
    for wd in b:
      lowerCAmelCase_     : List[Any]     =  idx

  return vocab, raw_vocab, ids_to_tokens, emoji






class   __snake_case			(  _SCREAMING_SNAKE_CASE):







    """simple docstring"""



    lowercase  					=   VOCAB_FILES_NAMES
    lowercase  					=   PRETRAINED_VOCAB_FILES_MAP
    lowercase  					=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase  					=   ['input_ids', 'attention_mask']

    def __init__(  self       :    List[str]       ,		lowerCamelCase       :    Any       ,		lowerCamelCase       :    Optional[Any]       ,		lowerCamelCase       :    Optional[Any]="<|endoftext|>"       ,		lowerCamelCase       :    List[Any]="<|endoftext|>"       ,		lowerCamelCase       :    int="<|startoftext|>"       ,		lowerCamelCase       :    int="<|endoftext|>"       ,		lowerCamelCase       :    int=False       ,		**lowerCamelCase       :    Union[str, Any]       ,		)  ->    Optional[int]:
      super().__init__(
          unk_token=lowerCamelCase       ,		pad_token=lowerCamelCase       ,		bos_token=lowerCamelCase       ,		eos_token=lowerCamelCase       ,		do_clean_text=lowerCamelCase       ,		**lowerCamelCase       ,		)
      if not os.path.isfile(lowerCamelCase						):
        raise ValueError(
            F'Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'
            """ model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"""						)
      if not os.path.isfile(lowerCamelCase						):
        raise ValueError(
            F'Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'
            """ pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"""						)
      lowerCAmelCase_     : str     =  do_clean_text
      lowerCAmelCase_,				lowerCAmelCase_,				lowerCAmelCase_,				lowerCAmelCase_     : Union[str, Any]     =  load_vocab_and_emoji(lowerCamelCase       ,		lowerCamelCase						)
      lowerCAmelCase_     : Union[str, Any]     =  SubWordJapaneseTokenizer(
          vocab=self.vocab       ,		ids_to_tokens=self.ids_to_tokens       ,		emoji=self.emoji						)

    @property
    def __lowercase  (  self       :    Optional[Any]						)  ->    Union[str, Any]:
      # self.vocab contains support for character fluctuation unique to Japanese, and has a large number of vocab
      return len(self.raw_vocab						)

    def __lowercase  (  self       :    str						)  ->    Any:
      return dict(self.raw_vocab       ,		**self.added_tokens_encoder						)

    def __lowercase  (  self       :    Optional[Any]       ,		lowerCamelCase       :    List[Any]						)  ->    List[Any]:
      return self.subword_tokenizer.tokenize(lowerCamelCase       ,		clean=self.do_clean_text						)

    def __lowercase  (  self       :    str       ,		lowerCamelCase       :    str						)  ->    Dict:
      return self.vocab.get(lowerCamelCase       ,		self.vocab.get(self.unk_token						)						)

    def __lowercase  (  self       :    int       ,		lowerCamelCase       :    Optional[int]						)  ->    Optional[Any]:
      return self.subword_tokenizer.convert_id_to_token(lowerCamelCase						)

    def __lowercase  (  self       :    Tuple       ,		lowerCamelCase       :    Any						)  ->    Dict:
      lowerCAmelCase_     : Any     =  """""".join(lowerCamelCase						).strip()
      return out_string

    def __lowercase  (  self       :    str       ,		lowerCamelCase       :    "Conversation"						)  ->    List[int]:
      lowerCAmelCase_     : Dict     =  []
      for is_user, text in conversation.iter_texts():
        input_ids.extend(self.encode(lowerCamelCase       ,		add_special_tokens=lowerCamelCase						) + [self.eos_token_id]						)

      if len(lowerCamelCase						) > self.model_max_length:
        lowerCAmelCase_     : Any     =  input_ids[-self.model_max_length :]
      return input_ids





    def __lowercase  (  self       :    List[Any]       ,		lowerCamelCase       :    str       ,		lowerCamelCase       :    Optional[str] = None						)  ->    Tuple[str]:
      lowerCAmelCase_     : Optional[int]     =  0
      if os.path.isdir(lowerCamelCase						):
        lowerCAmelCase_     : Dict     =  os.path.join(
            lowerCamelCase       ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)
        lowerCAmelCase_     : Tuple     =  os.path.join(
            lowerCamelCase       ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""emoji_file"""]						)
      else:
        lowerCAmelCase_     : Optional[Any]     =  (
            (filename_prefix + """-""" if filename_prefix else """""") + save_directory + VOCAB_FILES_NAMES["""vocab_file"""]
        )
        lowerCAmelCase_     : Tuple     =  (
            (filename_prefix + """-""" if filename_prefix else """""") + save_directory + VOCAB_FILES_NAMES["""emoji_file"""]
        )
      with open(lowerCamelCase       ,		"""w"""       ,		encoding="""utf-8"""						) as writer:
        for token_index, token in self.ids_to_tokens.items():
          if index != token_index:
            logger.warning(
                F'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
                """ Please check that the vocabulary is not corrupted!"""						)
            lowerCAmelCase_     : str     =  token_index
          writer.write(""",""".join(lowerCamelCase						) + """\n"""						)
          index += 1
      with open(lowerCamelCase       ,		"""w"""       ,		encoding="""utf-8"""						) as writer:
        json.dump(self.emoji       ,		lowerCamelCase						)
      return vocab_file, emoji_file







class   __snake_case			(  _SCREAMING_SNAKE_CASE):







    """simple docstring"""



    def __init__(  self       :    str       ,		lowerCamelCase       :    List[Any]       ,		lowerCamelCase       :    Dict       ,		lowerCamelCase       :    List[str]						)  ->    Dict:
      lowerCAmelCase_     : Dict     =  vocab  # same as swe
      lowerCAmelCase_     : Union[str, Any]     =  ids_to_tokens  # same as bpe
      lowerCAmelCase_     : str     =  emoji
      lowerCAmelCase_     : List[str]     =  np.max([len(lowerCamelCase						) for w in self.vocab.keys()]						)
      lowerCAmelCase_     : Union[str, Any]     =  re.compile(R"""(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)"""						)
      lowerCAmelCase_     : Dict     =  re.compile(R"""[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*"""						)
      lowerCAmelCase_     : List[str]     =  re.compile(R"""[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}"""						)
      lowerCAmelCase_     : Any     =  re.compile(
          R"""([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"""						)
      lowerCAmelCase_     : str     =  re.compile(
          R"""(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"""						)
      lowerCAmelCase_     : List[Any]     =  re.compile(
          R"""((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*"""						)
      lowerCAmelCase_     : Tuple     =  """─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"""
      lowerCAmelCase_     : int     =  """▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"""
      lowerCAmelCase_     : Tuple     =  str.maketrans({k: """<BLOCK>""" for k in keisen + blocks}						)

    def __len__(  self       :    Optional[int]						)  ->    str:
      return len(self.ids_to_tokens						)

    def __lowercase  (  self       :    Tuple       ,		lowerCamelCase       :    Optional[int]						)  ->    Dict:
      lowerCAmelCase_     : Union[str, Any]     =  self.content_repattera.sub("""<URL>"""       ,		lowerCamelCase						)
      lowerCAmelCase_     : Optional[int]     =  self.content_repattera.sub("""<EMAIL>"""       ,		lowerCamelCase						)
      lowerCAmelCase_     : Tuple     =  self.content_repattera.sub("""<TEL>"""       ,		lowerCamelCase						)
      lowerCAmelCase_     : Optional[Any]     =  self.content_repattera.sub("""<DATE>"""       ,		lowerCamelCase						)
      lowerCAmelCase_     : List[str]     =  self.content_repattera.sub("""<DATE>"""       ,		lowerCamelCase						)
      lowerCAmelCase_     : Union[str, Any]     =  self.content_repattera.sub("""<PRICE>"""       ,		lowerCamelCase						)
      lowerCAmelCase_     : Dict     =  content.translate(self.content_transa						)
      while "<BLOCK><BLOCK>" in content:
        lowerCAmelCase_     : Optional[int]     =  content.replace("""<BLOCK><BLOCK>"""       ,		"""<BLOCK>"""						)
      return content

    def __lowercase  (  self       :    Optional[int]       ,		lowerCamelCase       :    Optional[int]       ,		lowerCamelCase       :    Union[str, Any]=False						)  ->    Dict:
      lowerCAmelCase_     : str     =  text.replace(""" """       ,		"""<SP>"""						)
      lowerCAmelCase_     : Optional[int]     =  text.replace("""　"""       ,		"""<SP>"""						)
      lowerCAmelCase_     : Optional[Any]     =  text.replace("""\r\n"""       ,		"""<BR>"""						)
      lowerCAmelCase_     : List[str]     =  text.replace("""\n"""       ,		"""<BR>"""						)
      lowerCAmelCase_     : Optional[Any]     =  text.replace("""\r"""       ,		"""<BR>"""						)
      lowerCAmelCase_     : Optional[Any]     =  text.replace("""\t"""       ,		"""<TAB>"""						)
      lowerCAmelCase_     : Optional[int]     =  text.replace("""—"""       ,		"""ー"""						)
      lowerCAmelCase_     : Optional[int]     =  text.replace("""−"""       ,		"""ー"""						)
      for k, v in self.emoji["emoji"].items():
        if k in text:
          lowerCAmelCase_     : Optional[int]     =  text.replace(lowerCamelCase       ,		lowerCamelCase						)
      if clean:
        lowerCAmelCase_     : Dict     =  self.clean_text(lowerCamelCase						)

      def check_simbol(lowerCamelCase       :    int						):
        lowerCAmelCase_     : Dict     =  x.encode()
        if len(lowerCamelCase						) == 1 and len(lowerCamelCase						) == 2:
          lowerCAmelCase_     : int     =  (int(e[0]						) << 8) + int(e[1]						)
          if (
              (c >= 0xC2A1 and c <= 0xC2BF)
              or (c >= 0xC780 and c <= 0xC783)
              or (c >= 0xCAB9 and c <= 0xCBBF)
              or (c >= 0xCC80 and c <= 0xCDA2)
          ):
            return True
        return False

      def checkuae(lowerCamelCase       :    Dict						):
        lowerCAmelCase_     : Tuple     =  x.encode()
        if len(lowerCamelCase						) == 1 and len(lowerCamelCase						) == 3:
          lowerCAmelCase_     : int     =  (int(e[0]						) << 16) + (int(e[1]						) << 8) + int(e[2]						)
          if c >= 0xE2_8080 and c <= 0xE2_B07F:
            return True
        return False

      lowerCAmelCase_     : int     =  0
      lowerCAmelCase_     : List[str]     =  []
      while pos < len(lowerCamelCase						):
        lowerCAmelCase_     : Optional[int]     =  min(len(lowerCamelCase						)       ,		pos + self.maxlen + 1						) if text[pos] == """<""" else pos + 3
        lowerCAmelCase_     : List[Any]     =  []  # (token_id, token, pos)
        for e in range(lowerCamelCase       ,		lowerCamelCase       ,		-1						):
          lowerCAmelCase_     : int     =  text[pos:e]
          if wd in self.vocab:
            if wd[0] == "<" and len(lowerCamelCase						) > 2:
              lowerCAmelCase_     : Dict     =  [(self.vocab[wd], wd, e)]
              break
            else:
              candidates.append((self.vocab[wd], wd, e)						)
        if len(lowerCamelCase						) > 0:
          # the smallest token_id is adopted
          lowerCAmelCase_,				lowerCAmelCase_,				lowerCAmelCase_     : Tuple     =  sorted(lowerCamelCase       ,		key=lambda lowerCamelCase						: x[0]						)[0]
          result.append(lowerCamelCase						)
          lowerCAmelCase_     : Optional[Any]     =  e
        else:
          lowerCAmelCase_     : str     =  pos + 1
          lowerCAmelCase_     : Union[str, Any]     =  text[pos:end]
          if check_simbol(lowerCamelCase						):
            result.append("""<KIGOU>"""						)
          elif checkuae(lowerCamelCase						):
            result.append("""<U2000U2BFF>"""						)
          else:
            for i in wd.encode("""utf-8"""						):
              result.append("""<|byte%d|>""" % i						)
          lowerCAmelCase_     : Tuple     =  end
      return result





    def __lowercase  (  self       :    List[str]       ,		lowerCamelCase       :    Optional[Any]       ,		lowerCamelCase       :    Dict="\n"						)  ->    Union[str, Any]:
      lowerCAmelCase_     : Optional[Any]     =  []
      lowerCAmelCase_     : Union[str, Any]     =  []
      lowerCAmelCase_     : Union[str, Any]     =  self.ids_to_tokens[index][0]
      if word[:6] == "<|byte" and word[-2:] == "|>":
        byte_tokens.append(int(word[6:-2]						)						)
      else:
        if len(lowerCamelCase						) > 0:
          words.append(bytearray(lowerCamelCase						).decode("""utf-8"""       ,		errors="""replace"""						)						)
          lowerCAmelCase_     : str     =  []
        if word[:7] == "<|emoji" and word[-2:] == "|>":
          words.append(self.emoji["""emoji_inv"""][word]						)
        elif word == "<SP>":
          words.append(""" """						)
        elif word == "<BR>":
          words.append(lowerCamelCase						)
        elif word == "<TAB>":
          words.append("""\t"""						)
        elif word == "<BLOCK>":
          words.append("""▀"""						)
        elif word == "<KIGOU>":
          words.append("""ǀ"""						)
        elif word == "<U2000U2BFF>":
          words.append("""‖"""						)
        else:
          words.append(lowerCamelCase						)
      if len(lowerCamelCase						) > 0:
        words.append(bytearray(lowerCamelCase						).decode("""utf-8"""       ,		errors="""replace"""						)						)
      lowerCAmelCase_     : Optional[int]     =  """""".join(lowerCamelCase						)
      return text







'''simple docstring'''






from collections.abc import Iterator, MutableMapping
from dataclasses import dataclass
from typing import Generic, TypeVar

__A   :    Union[str, Any]											= TypeVar("KEY")
__A   :    Union[str, Any]											= TypeVar("VAL")






@dataclass(frozen=_SCREAMING_SNAKE_CASE      ,slots=_SCREAMING_SNAKE_CASE)
class   __snake_case			(  Generic[KEY, VAL]):







    """simple docstring"""



    lowercase  					=   42
    lowercase  					=   42







class   __snake_case			(  _Item):







    """simple docstring"""



    def __init__(  self       :    int						)  ->    None:
      super().__init__(lowerCamelCase       ,		lowerCamelCase						)





    def __bool__(  self       :    Tuple						)  ->    bool:
      return False







__A   :    Any											= _DeletedItem()






class   __snake_case			(  MutableMapping[KEY, VAL]):







    """simple docstring"""



    def __init__(  self       :    Optional[int]       ,		lowerCamelCase       :    int = 8       ,		lowerCamelCase       :    float = 0.75						)  ->    None:
      lowerCAmelCase_     : str     =  initial_block_size
      lowerCAmelCase_     : list[_Item | None]     =  [None] * initial_block_size
      assert 0.0 < capacity_factor < 1.0
      lowerCAmelCase_     : Optional[Any]     =  capacity_factor
      lowerCAmelCase_     : List[Any]     =  0

    def __lowercase  (  self       :    List[Any]       ,		lowerCamelCase       :    KEY						)  ->    int:
      return hash(lowerCamelCase						) % len(self._buckets						)

    def __lowercase  (  self       :    Optional[Any]       ,		lowerCamelCase       :    int						)  ->    int:
      return (ind + 1) % len(self._buckets						)

    def __lowercase  (  self       :    Union[str, Any]       ,		lowerCamelCase       :    int       ,		lowerCamelCase       :    KEY       ,		lowerCamelCase       :    VAL						)  ->    bool:
      lowerCAmelCase_     : Union[str, Any]     =  self._buckets[ind]
      if not stored:
        lowerCAmelCase_     : List[Any]     =  _Item(lowerCamelCase       ,		lowerCamelCase						)
        self._len += 1
        return True
      elif stored.key == key:
        lowerCAmelCase_     : List[str]     =  _Item(lowerCamelCase       ,		lowerCamelCase						)
        return True
      else:
        return False

    def __lowercase  (  self       :    Optional[Any]						)  ->    bool:
      lowerCAmelCase_     : str     =  len(self._buckets						) * self._capacity_factor
      return len(self						) >= int(lowerCamelCase						)

    def __lowercase  (  self       :    Dict						)  ->    bool:
      if len(self._buckets						) <= self._initial_block_size:
        return False
      lowerCAmelCase_     : Optional[int]     =  len(self._buckets						) * self._capacity_factor / 2
      return len(self						) < limit

    def __lowercase  (  self       :    Tuple       ,		lowerCamelCase       :    int						)  ->    None:
      lowerCAmelCase_     : Union[str, Any]     =  self._buckets
      lowerCAmelCase_     : str     =  [None] * new_size
      lowerCAmelCase_     : str     =  0
      for item in old_buckets:
        if item:
          self._add_item(item.key       ,		item.val						)

    def __lowercase  (  self       :    Tuple						)  ->    None:
      self._resize(len(self._buckets						) * 2						)

    def __lowercase  (  self       :    Union[str, Any]						)  ->    None:
      self._resize(len(self._buckets						) // 2						)

    def __lowercase  (  self       :    Optional[int]       ,		lowerCamelCase       :    KEY						)  ->    Iterator[int]:
      lowerCAmelCase_     : int     =  self._get_bucket_index(lowerCamelCase						)
      for _ in range(len(self._buckets						)						):
        yield ind
        lowerCAmelCase_     : Dict     =  self._get_next_ind(lowerCamelCase						)

    def __lowercase  (  self       :    Tuple       ,		lowerCamelCase       :    KEY       ,		lowerCamelCase       :    VAL						)  ->    None:
      for ind in self._iterate_buckets(lowerCamelCase						):
        if self._try_set(lowerCamelCase       ,		lowerCamelCase       ,		lowerCamelCase						):
          break

    def __setitem__(  self       :    Tuple       ,		lowerCamelCase       :    KEY       ,		lowerCamelCase       :    VAL						)  ->    None:
      if self._is_full():
        self._size_up()

      self._add_item(lowerCamelCase       ,		lowerCamelCase						)

    def __delitem__(  self       :    Optional[int]       ,		lowerCamelCase       :    KEY						)  ->    None:
      for ind in self._iterate_buckets(lowerCamelCase						):
        lowerCAmelCase_     : int     =  self._buckets[ind]
        if item is None:
          raise KeyError(lowerCamelCase						)
        if item is _deleted:
          continue
        if item.key == key:
          lowerCAmelCase_     : List[Any]     =  _deleted
          self._len -= 1
          break
      if self._is_sparse():
        self._size_down()

    def __getitem__(  self       :    Any       ,		lowerCamelCase       :    KEY						)  ->    VAL:
      for ind in self._iterate_buckets(lowerCamelCase						):
        lowerCAmelCase_     : Dict     =  self._buckets[ind]
        if item is None:
          break
        if item is _deleted:
          continue
        if item.key == key:
          return item.val
      raise KeyError(lowerCamelCase						)

    def __len__(  self       :    Any						)  ->    int:
      return self._len

    def __iter__(  self       :    Optional[Any]						)  ->    Iterator[KEY]:
      yield from (item.key for item in self._buckets if item)





    def __repr__(  self       :    List[Any]						)  ->    str:
      lowerCAmelCase_     : int     =  """ ,""".join(
          F'{item.key}: {item.val}' for item in self._buckets if item						)
      return F'HashMap({val_string})'



