





import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation


_snake_case    :       Optional[Any]		       =    logging.get_logger(__name__)

_snake_case    :       Optional[int]		       =    {'vocab_file': 'vocab.txt', 'emoji_file': 'emoji.json'}

_snake_case    :       Dict		       =    {
    'vocab_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt',
    },
    'emoji_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json',
    },
}

_snake_case    :       str		       =    {
    'abeja/gpt-neox-japanese-2.7b': 2048,
}

def     a_       (					lowerCAmelCase_ :     Optional[Any],      lowerCAmelCase_ :     List[str]      ):
   with open(lowerCAmelCase_,      'r',      encoding='utf-8'      ) as f:
      __lowerCAmelCase				      =      json.loads(f.read()      )

   __lowerCAmelCase				      =      collections.OrderedDict()
   __lowerCAmelCase				      =      collections.OrderedDict()
   __lowerCAmelCase				      =      collections.OrderedDict()
   with open(lowerCAmelCase_,      'r',      encoding='utf-8'      ) as f:
      __lowerCAmelCase				      =      f.readlines()
   __lowerCAmelCase				      =      [[t.rstrip('\n'      )] if (t == ',' or ',' not in t) else t.rstrip('\n'      ).split(','      ) for t in token]
   for idx, b in enumerate(lowerCAmelCase_      ):
      __lowerCAmelCase				      =      b
      __lowerCAmelCase				      =      idx
      for wd in b:
         __lowerCAmelCase				      =      idx

   return vocab, raw_vocab, ids_to_tokens, emoji



class 							_UpperCAmelCase       (       _UpperCamelCase       ):

  """simple docstring"""


  a_              =					VOCAB_FILES_NAMES
  a_              =					PRETRAINED_VOCAB_FILES_MAP
  a_              =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a_              =					["""input_ids""", """attention_mask"""]






  def __init__(							self					:						Tuple   ,    lowerCAmelCase_					:						Optional[Any]   ,    lowerCAmelCase_					:						Optional[Any]   ,    lowerCAmelCase_					:						Tuple="<|endoftext|>"   ,    lowerCAmelCase_					:						Optional[int]="<|endoftext|>"   ,    lowerCAmelCase_					:						Optional[Any]="<|startoftext|>"   ,    lowerCAmelCase_					:						Optional[Any]="<|endoftext|>"   ,    lowerCAmelCase_					:						Union[str, Any]=False   ,    **lowerCAmelCase_					:						List[str]   ,    )				->		Any:
     super().__init__(
         unk_token=lowerCAmelCase_   ,    pad_token=lowerCAmelCase_   ,    bos_token=lowerCAmelCase_   ,    eos_token=lowerCAmelCase_   ,    do_clean_text=lowerCAmelCase_   ,    **lowerCAmelCase_   ,    )
     if not os.path.isfile(lowerCAmelCase_	):
        raise ValueError(
            f"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
            ' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'	)
     if not os.path.isfile(lowerCAmelCase_	):
        raise ValueError(
            f"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
            ' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'	)
     __lowerCAmelCase				      =      do_clean_text
     __lowerCAmelCase      ,      __lowerCAmelCase      ,      __lowerCAmelCase      ,      __lowerCAmelCase				      =      load_vocab_and_emoji(lowerCAmelCase_   ,    lowerCAmelCase_	)
     __lowerCAmelCase				      =      SubWordJapaneseTokenizer(
         vocab=self.vocab   ,    ids_to_tokens=self.ids_to_tokens   ,    emoji=self.emoji	)






  @property
  def 							lowercase						(							self					:						List[Any]	)				->		List[Any]:
     # self.vocab contains support for character fluctuation unique to Japanese, and has a large number of vocab
     return len(self.raw_vocab	)






  def 							lowercase						(							self					:						Any	)				->		Tuple:
     return dict(self.raw_vocab   ,    **self.added_tokens_encoder	)






  def 							lowercase						(							self					:						Any   ,    lowerCAmelCase_					:						Optional[Any]	)				->		Tuple:
     return self.subword_tokenizer.tokenize(lowerCAmelCase_   ,    clean=self.do_clean_text	)






  def 							lowercase						(							self					:						Optional[int]   ,    lowerCAmelCase_					:						Dict	)				->		Optional[int]:
     return self.vocab.get(lowerCAmelCase_   ,    self.vocab.get(self.unk_token	)	)






  def 							lowercase						(							self					:						List[str]   ,    lowerCAmelCase_					:						int	)				->		int:
     return self.subword_tokenizer.convert_id_to_token(lowerCAmelCase_	)






  def 							lowercase						(							self					:						Optional[Any]   ,    lowerCAmelCase_					:						Dict	)				->		Optional[int]:
     __lowerCAmelCase				      =      ''.join(lowerCAmelCase_	).strip()
     return out_string






  def 							lowercase						(							self					:						List[str]   ,    lowerCAmelCase_					:						"Conversation"	)				->		List[int]:
     __lowerCAmelCase				      =      []
     for is_user, text in conversation.iter_texts():
        input_ids.extend(self.encode(lowerCAmelCase_   ,    add_special_tokens=lowerCAmelCase_	) + [self.eos_token_id]	)

     if len(lowerCAmelCase_	) > self.model_max_length:
        __lowerCAmelCase				      =      input_ids[-self.model_max_length :]
     return input_ids






  def 							lowercase						(							self					:						List[Any]   ,    lowerCAmelCase_					:						str   ,    lowerCAmelCase_					:						Optional[str] = None	)				->		Tuple[str]:
     __lowerCAmelCase				      =      0
     if os.path.isdir(lowerCAmelCase_	):
        __lowerCAmelCase				      =      os.path.join(
            lowerCAmelCase_   ,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)
        __lowerCAmelCase				      =      os.path.join(
            lowerCAmelCase_   ,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['emoji_file']	)
     else:
        __lowerCAmelCase				      =      (
            (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['vocab_file']
        )
        __lowerCAmelCase				      =      (
            (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['emoji_file']
        )
     with open(lowerCAmelCase_   ,    'w'   ,    encoding='utf-8'	) as writer:
        for token_index, token in self.ids_to_tokens.items():
           if index != token_index:
              logger.warning(
                  f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                  ' Please check that the vocabulary is not corrupted!'	)
              __lowerCAmelCase				      =      token_index
           writer.write(','.join(lowerCAmelCase_	) + '\n'	)
           index += 1
     with open(lowerCAmelCase_   ,    'w'   ,    encoding='utf-8'	) as writer:
        json.dump(self.emoji   ,    lowerCAmelCase_	)
     return vocab_file, emoji_file





class 							_UpperCAmelCase       (       _UpperCamelCase       ):

  """simple docstring"""


  def __init__(							self					:						Any   ,    lowerCAmelCase_					:						Any   ,    lowerCAmelCase_					:						Tuple   ,    lowerCAmelCase_					:						List[str]	)				->		str:
     __lowerCAmelCase				      =      vocab  # same as swe
     __lowerCAmelCase				      =      ids_to_tokens  # same as bpe
     __lowerCAmelCase				      =      emoji
     __lowerCAmelCase				      =      np.max([len(lowerCAmelCase_	) for w in self.vocab.keys()]	)
     __lowerCAmelCase				      =      re.compile(R'(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'	)
     __lowerCAmelCase				      =      re.compile(R'[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'	)
     __lowerCAmelCase				      =      re.compile(R'[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'	)
     __lowerCAmelCase				      =      re.compile(
         R'([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'	)
     __lowerCAmelCase				      =      re.compile(
         R'(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'	)
     __lowerCAmelCase				      =      re.compile(
         R'((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'	)
     __lowerCAmelCase				      =      '─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'
     __lowerCAmelCase				      =      '▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'
     __lowerCAmelCase				      =      str.maketrans({k: '<BLOCK>' for k in keisen + blocks}	)






  def __len__(							self					:						Union[str, Any]	)				->		int:
     return len(self.ids_to_tokens	)






  def 							lowercase						(							self					:						Optional[int]   ,    lowerCAmelCase_					:						Union[str, Any]	)				->		Optional[int]:
     __lowerCAmelCase				      =      self.content_repattera.sub('<URL>'   ,    lowerCAmelCase_	)
     __lowerCAmelCase				      =      self.content_repattera.sub('<EMAIL>'   ,    lowerCAmelCase_	)
     __lowerCAmelCase				      =      self.content_repattera.sub('<TEL>'   ,    lowerCAmelCase_	)
     __lowerCAmelCase				      =      self.content_repattera.sub('<DATE>'   ,    lowerCAmelCase_	)
     __lowerCAmelCase				      =      self.content_repattera.sub('<DATE>'   ,    lowerCAmelCase_	)
     __lowerCAmelCase				      =      self.content_repattera.sub('<PRICE>'   ,    lowerCAmelCase_	)
     __lowerCAmelCase				      =      content.translate(self.content_transa	)
     while "<BLOCK><BLOCK>" in content:
        __lowerCAmelCase				      =      content.replace('<BLOCK><BLOCK>'   ,    '<BLOCK>'	)
     return content






  def 							lowercase						(							self					:						List[str]   ,    lowerCAmelCase_					:						Optional[int]   ,    lowerCAmelCase_					:						List[Any]=False	)				->		Any:
     __lowerCAmelCase				      =      text.replace(' '   ,    '<SP>'	)
     __lowerCAmelCase				      =      text.replace('　'   ,    '<SP>'	)
     __lowerCAmelCase				      =      text.replace('\r\n'   ,    '<BR>'	)
     __lowerCAmelCase				      =      text.replace('\n'   ,    '<BR>'	)
     __lowerCAmelCase				      =      text.replace('\r'   ,    '<BR>'	)
     __lowerCAmelCase				      =      text.replace('\t'   ,    '<TAB>'	)
     __lowerCAmelCase				      =      text.replace('—'   ,    'ー'	)
     __lowerCAmelCase				      =      text.replace('−'   ,    'ー'	)
     for k, v in self.emoji["emoji"].items():
        if k in text:
           __lowerCAmelCase				      =      text.replace(lowerCAmelCase_   ,    lowerCAmelCase_	)
     if clean:
        __lowerCAmelCase				      =      self.clean_text(lowerCAmelCase_	)

     def check_simbol(lowerCAmelCase_					:						List[str]	):
        __lowerCAmelCase				      =      x.encode()
        if len(lowerCAmelCase_	) == 1 and len(lowerCAmelCase_	) == 2:
           __lowerCAmelCase				      =      (int(e[0]	) << 8) + int(e[1]	)
           if (
               (c >= 0Xc2_a1 and c <= 0Xc2_bf)
               or (c >= 0Xc7_80 and c <= 0Xc7_83)
               or (c >= 0Xca_b9 and c <= 0Xcb_bf)
               or (c >= 0Xcc_80 and c <= 0Xcd_a2)
           ):
              return True
        return False

     def checkuae(lowerCAmelCase_					:						List[str]	):
        __lowerCAmelCase				      =      x.encode()
        if len(lowerCAmelCase_	) == 1 and len(lowerCAmelCase_	) == 3:
           __lowerCAmelCase				      =      (int(e[0]	) << 1_6) + (int(e[1]	) << 8) + int(e[2]	)
           if c >= 0Xe2_80_80 and c <= 0Xe2_b0_7f:
              return True
        return False

     __lowerCAmelCase				      =      0
     __lowerCAmelCase				      =      []
     while pos < len(lowerCAmelCase_	):
        __lowerCAmelCase				      =      min(len(lowerCAmelCase_	)   ,    pos + self.maxlen + 1	) if text[pos] == '<' else pos + 3
        __lowerCAmelCase				      =      []  # (token_id, token, pos)
        for e in range(lowerCAmelCase_   ,    lowerCAmelCase_   ,    -1	):
           __lowerCAmelCase				      =      text[pos:e]
           if wd in self.vocab:
              if wd[0] == "<" and len(lowerCAmelCase_	) > 2:
                 __lowerCAmelCase				      =      [(self.vocab[wd], wd, e)]
                 break
              else:
                 candidates.append((self.vocab[wd], wd, e)	)
        if len(lowerCAmelCase_	) > 0:
           # the smallest token_id is adopted
           __lowerCAmelCase      ,      __lowerCAmelCase      ,      __lowerCAmelCase				      =      sorted(lowerCAmelCase_   ,    key=lambda lowerCAmelCase_	: x[0]	)[0]
           result.append(lowerCAmelCase_	)
           __lowerCAmelCase				      =      e
        else:
           __lowerCAmelCase				      =      pos + 1
           __lowerCAmelCase				      =      text[pos:end]
           if check_simbol(lowerCAmelCase_	):
              result.append('<KIGOU>'	)
           elif checkuae(lowerCAmelCase_	):
              result.append('<U2000U2BFF>'	)
           else:
              for i in wd.encode('utf-8'	):
                 result.append('<|byte%d|>' % i	)
           __lowerCAmelCase				      =      end
     return result






  def 							lowercase						(							self					:						Dict   ,    lowerCAmelCase_					:						Any   ,    lowerCAmelCase_					:						List[Any]="\n"	)				->		Tuple:
     __lowerCAmelCase				      =      []
     __lowerCAmelCase				      =      []
     __lowerCAmelCase				      =      self.ids_to_tokens[index][0]
     if word[:6] == "<|byte" and word[-2:] == "|>":
        byte_tokens.append(int(word[6:-2]	)	)
     else:
        if len(lowerCAmelCase_	) > 0:
           words.append(bytearray(lowerCAmelCase_	).decode('utf-8'   ,    errors='replace'	)	)
           __lowerCAmelCase				      =      []
        if word[:7] == "<|emoji" and word[-2:] == "|>":
           words.append(self.emoji['emoji_inv'][word]	)
        elif word == "<SP>":
           words.append(' '	)
        elif word == "<BR>":
           words.append(lowerCAmelCase_	)
        elif word == "<TAB>":
           words.append('\t'	)
        elif word == "<BLOCK>":
           words.append('▀'	)
        elif word == "<KIGOU>":
           words.append('ǀ'	)
        elif word == "<U2000U2BFF>":
           words.append('‖'	)
        else:
           words.append(lowerCAmelCase_	)
     if len(lowerCAmelCase_	) > 0:
        words.append(bytearray(lowerCAmelCase_	).decode('utf-8'   ,    errors='replace'	)	)
     __lowerCAmelCase				      =      ''.join(lowerCAmelCase_	)
     return text







'''simple docstring'''


import os
from typing import Dict, List, Union

import tensorflow as tf
from keras_nlp.tokenizers import BytePairTokenizer
from tensorflow_text import pad_model_inputs

from .tokenization_gpta import GPTaTokenizer







class      SCREAMING_SNAKE_CASE    (						tf.keras.layers.Layer				):






      def __init__( self   :  List[Any]						,						lowercase__   :  Dict[str, int]						,						lowercase__   :  List[str]						,						lowercase__   :  int = None						,						lowercase__   :  int = None       ):


           '''simple docstring'''

           super().__init__()
           a_     :							Optional[Any]     						=  pad_token_id
           a_     :							str     						=  max_length
           a_     :							List[str]     						=  vocab
           a_     :							Union[str, Any]     						=  merges
           a_     :							Union[str, Any]     						=  BytePairTokenizer(lowercase__						,						lowercase__						,						sequence_length=lowercase__       )






      @classmethod
      def 							lowercase_						( cls   :  List[Any]						,						lowercase__   :  GPTaTokenizer						,						*lowercase__   :  Optional[Any]						,						**lowercase__   :  int       ):


           '''simple docstring'''

           a_     :							str     						=  [""" """.join(lowercase__       ) for m in tokenizer.bpe_ranks.keys()]
           a_     :							Any     						=  tokenizer.get_vocab()
           return cls(lowercase__						,						lowercase__						,						*lowercase__						,						**lowercase__       )






      @classmethod
      def 							lowercase_						( cls   :  Dict						,						lowercase__   :  Union[str, os.PathLike]						,						*lowercase__   :  str						,						**lowercase__   :  Optional[int]       ):


           '''simple docstring'''

           a_     :							Optional[int]     						=  GPTaTokenizer.from_pretrained(lowercase__						,						*lowercase__						,						**lowercase__       )
           return cls.from_tokenizer(lowercase__						,						*lowercase__						,						**lowercase__       )






      @classmethod
      def 							lowercase_						( cls   :  Dict						,						lowercase__   :  Optional[Any]       ):


           '''simple docstring'''

           return cls(**lowercase__       )






      def 							lowercase_						( self   :  Optional[Any]       ):


           '''simple docstring'''

           return {
               "vocab": self.vocab,
               "merges": self.merges,
               "max_length": self.max_length,
               "pad_token_id": self.pad_token_id,
           }






      def 							lowercase_						( self   :  Dict						,						lowercase__   :  Union[str, Any]						,						lowercase__   :  int = None       ):


           '''simple docstring'''

           a_     :							List[str]     						=  self.tf_tokenizer(lowercase__       )
           a_     :							int     						=  tf.ones_like(lowercase__       )

           if self.pad_token_id is not None:
                # pad the tokens up to max length
                a_     :							int     						=  max_length if max_length is not None else self.max_length

                if max_length is not None:
                     a_						,     a_     :							int     						=  pad_model_inputs(
                         lowercase__						,						max_seq_length=lowercase__						,						pad_value=self.pad_token_id       )

           return {"attention_mask": attention_mask, "input_ids": input_ids}
