


'''simple docstring'''

import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
       from transformers.pipelines.conversational import Conversation


lowercase   :						Optional[Any]								=			logging.get_logger(__name__)

lowercase   :						int								=			{'vocab_file': 'vocab.txt', 'emoji_file': 'emoji.json'}

lowercase   :						Dict								=			{
    'vocab_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt',
    },
    'emoji_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json',
    },
}

lowercase   :						Tuple								=			{
    'abeja/gpt-neox-japanese-2.7b': 2_0_4_8,
}
def      __a	(							A__						,       A__       )						->			Union[str, Any]:





  with open(__snake_case						,       "r"						,       encoding="utf-8"       ) as f:
    lowerCAmelCase											=	json.loads(f.read()       )

  lowerCAmelCase											=	collections.OrderedDict()
  lowerCAmelCase											=	collections.OrderedDict()
  lowerCAmelCase											=	collections.OrderedDict()
  with open(__snake_case						,       "r"						,       encoding="utf-8"       ) as f:
    lowerCAmelCase											=	f.readlines()
  lowerCAmelCase											=	[[t.rstrip("\n"       )] if (t == "," or "," not in t) else t.rstrip("\n"       ).split(","       ) for t in token]
  for idx, b in enumerate(__snake_case       ):
    lowerCAmelCase											=	b
    lowerCAmelCase											=	idx
    for wd in b:
      lowerCAmelCase											=	idx

  return vocab, raw_vocab, ids_to_tokens, emoji
class      _lowerCAmelCase				(			UpperCamelCase_			):





  """simple docstring"""



  lowerCAmelCase					   =			VOCAB_FILES_NAMES
  lowerCAmelCase					   =			PRETRAINED_VOCAB_FILES_MAP
  lowerCAmelCase					   =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowerCAmelCase					   =			['input_ids', 'attention_mask']
  def __init__(      self      :     List[str]      , SCREAMING_SNAKE_CASE      :     str      , SCREAMING_SNAKE_CASE      :     str      , SCREAMING_SNAKE_CASE      :     Optional[Any]="<|endoftext|>"      , SCREAMING_SNAKE_CASE      :     str="<|endoftext|>"      , SCREAMING_SNAKE_CASE      :     Any="<|startoftext|>"      , SCREAMING_SNAKE_CASE      :     Dict="<|endoftext|>"      , SCREAMING_SNAKE_CASE      :     Any=False      , **SCREAMING_SNAKE_CASE      :     Dict      , )	->						Optional[Any]:
    """simple docstring"""

    super().__init__(
        unk_token=__a      , pad_token=__a      , bos_token=__a      , eos_token=__a      , do_clean_text=__a      , **__a      , )
    if not os.path.isfile(__a				):
      raise ValueError(
          f"Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained"
          " model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"				)
    if not os.path.isfile(__a				):
      raise ValueError(
          f"Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google"
          " pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"				)
    lowerCAmelCase											=	do_clean_text
    lowerCAmelCase				,       lowerCAmelCase				,       lowerCAmelCase				,       lowerCAmelCase											=	load_vocab_and_emoji(__a      , __a				)
    lowerCAmelCase											=	SubWordJapaneseTokenizer(
        vocab=self.vocab      , ids_to_tokens=self.ids_to_tokens      , emoji=self.emoji				)
  @property
  def __A				(      self      :     List[str]				)	->						Optional[Any]:
    """simple docstring"""

    return len(self.raw_vocab				)
  def __A				(      self      :     Dict				)	->						Optional[int]:
    """simple docstring"""

    return dict(self.raw_vocab      , **self.added_tokens_encoder				)
  def __A				(      self      :     Optional[int]      , SCREAMING_SNAKE_CASE      :     Optional[int]				)	->						Optional[Any]:
    """simple docstring"""

    return self.subword_tokenizer.tokenize(__a      , clean=self.do_clean_text				)
  def __A				(      self      :     Tuple      , SCREAMING_SNAKE_CASE      :     Dict				)	->						Optional[Any]:
    """simple docstring"""

    return self.vocab.get(__a      , self.vocab.get(self.unk_token				)				)
  def __A				(      self      :     str      , SCREAMING_SNAKE_CASE      :     List[str]				)	->						str:
    """simple docstring"""

    return self.subword_tokenizer.convert_id_to_token(__a				)
  def __A				(      self      :     Optional[int]      , SCREAMING_SNAKE_CASE      :     List[Any]				)	->						Union[str, Any]:
    """simple docstring"""

    lowerCAmelCase											=	"".join(__a				).strip()
    return out_string
  def __A				(      self      :     Optional[int]      , SCREAMING_SNAKE_CASE      :     Tuple				)	->						List[int]:
    """simple docstring"""

    lowerCAmelCase											=	[]
    for is_user, text in conversation.iter_texts():
      input_ids.extend(self.encode(__a      , add_special_tokens=__a				) + [self.eos_token_id]				)

    if len(__a				) > self.model_max_length:
      lowerCAmelCase											=	input_ids[-self.model_max_length :]
    return input_ids


  def __A				(      self      :     List[Any]      , SCREAMING_SNAKE_CASE      :     Dict      , SCREAMING_SNAKE_CASE      :     Any = None				)	->						Tuple[str]:
    """simple docstring"""

    lowerCAmelCase											=	0
    if os.path.isdir(__a				):
      lowerCAmelCase											=	os.path.join(
          __a      , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]				)
      lowerCAmelCase											=	os.path.join(
          __a      , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["emoji_file"]				)
    else:
      lowerCAmelCase											=	(
          (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["vocab_file"]
      )
      lowerCAmelCase											=	(
          (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["emoji_file"]
      )
    with open(__a      , "w"      , encoding="utf-8"				) as writer:
      for token_index, token in self.ids_to_tokens.items():
        if index != token_index:
          logger.warning(
              f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
              " Please check that the vocabulary is not corrupted!"				)
          lowerCAmelCase											=	token_index
        writer.write(",".join(__a				) + "\n"				)
        index += 1
    with open(__a      , "w"      , encoding="utf-8"				) as writer:
      json.dump(self.emoji      , __a				)
    return vocab_file, emoji_file






class      _lowerCAmelCase				(			UpperCamelCase_			):





  """simple docstring"""



  def __init__(      self      :     List[Any]      , SCREAMING_SNAKE_CASE      :     Optional[int]      , SCREAMING_SNAKE_CASE      :     Dict      , SCREAMING_SNAKE_CASE      :     Union[str, Any]				)	->						int:
    """simple docstring"""

    lowerCAmelCase											=	vocab  # same as swe
    lowerCAmelCase											=	ids_to_tokens  # same as bpe
    lowerCAmelCase											=	emoji
    lowerCAmelCase											=	np.max([len(__a				) for w in self.vocab.keys()]				)
    lowerCAmelCase											=	re.compile(R"(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)"				)
    lowerCAmelCase											=	re.compile(R"[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*"				)
    lowerCAmelCase											=	re.compile(R"[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}"				)
    lowerCAmelCase											=	re.compile(
        R"([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"				)
    lowerCAmelCase											=	re.compile(
        R"(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"				)
    lowerCAmelCase											=	re.compile(
        R"((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*"				)
    lowerCAmelCase											=	"─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"
    lowerCAmelCase											=	"▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"
    lowerCAmelCase											=	str.maketrans({k: "<BLOCK>" for k in keisen + blocks}				)
  def __len__(      self      :     Any				)	->						Optional[Any]:
    """simple docstring"""

    return len(self.ids_to_tokens				)
  def __A				(      self      :     Any      , SCREAMING_SNAKE_CASE      :     List[str]				)	->						Dict:
    """simple docstring"""

    lowerCAmelCase											=	self.content_repattera.sub("<URL>"      , __a				)
    lowerCAmelCase											=	self.content_repattera.sub("<EMAIL>"      , __a				)
    lowerCAmelCase											=	self.content_repattera.sub("<TEL>"      , __a				)
    lowerCAmelCase											=	self.content_repattera.sub("<DATE>"      , __a				)
    lowerCAmelCase											=	self.content_repattera.sub("<DATE>"      , __a				)
    lowerCAmelCase											=	self.content_repattera.sub("<PRICE>"      , __a				)
    lowerCAmelCase											=	content.translate(self.content_transa				)
    while "<BLOCK><BLOCK>" in content:
      lowerCAmelCase											=	content.replace("<BLOCK><BLOCK>"      , "<BLOCK>"				)
    return content
  def __A				(      self      :     Tuple      , SCREAMING_SNAKE_CASE      :     Optional[int]      , SCREAMING_SNAKE_CASE      :     List[Any]=False				)	->						Any:
    """simple docstring"""

    lowerCAmelCase											=	text.replace(" "      , "<SP>"				)
    lowerCAmelCase											=	text.replace("　"      , "<SP>"				)
    lowerCAmelCase											=	text.replace("\r\n"      , "<BR>"				)
    lowerCAmelCase											=	text.replace("\n"      , "<BR>"				)
    lowerCAmelCase											=	text.replace("\r"      , "<BR>"				)
    lowerCAmelCase											=	text.replace("\t"      , "<TAB>"				)
    lowerCAmelCase											=	text.replace("—"      , "ー"				)
    lowerCAmelCase											=	text.replace("−"      , "ー"				)
    for k, v in self.emoji["emoji"].items():
      if k in text:
        lowerCAmelCase											=	text.replace(__a      , __a				)
    if clean:
      lowerCAmelCase											=	self.clean_text(__a				)

    def check_simbol(SCREAMING_SNAKE_CASE      :     Union[str, Any]				):
      lowerCAmelCase											=	x.encode()
      if len(__a				) == 1 and len(__a				) == 2:
        lowerCAmelCase											=	(int(e[0]				) << 8) + int(e[1]				)
        if (
            (c >= 0xC2A1 and c <= 0xC2BF)
            or (c >= 0xC780 and c <= 0xC783)
            or (c >= 0xCAB9 and c <= 0xCBBF)
            or (c >= 0xCC80 and c <= 0xCDA2)
        ):
          return True
      return False

    def checkuae(SCREAMING_SNAKE_CASE      :     List[Any]				):
      lowerCAmelCase											=	x.encode()
      if len(__a				) == 1 and len(__a				) == 3:
        lowerCAmelCase											=	(int(e[0]				) << 1_6) + (int(e[1]				) << 8) + int(e[2]				)
        if c >= 0xE2_8080 and c <= 0xE2_B07F:
          return True
      return False

    lowerCAmelCase											=	0
    lowerCAmelCase											=	[]
    while pos < len(__a				):
      lowerCAmelCase											=	min(len(__a				)      , pos + self.maxlen + 1				) if text[pos] == "<" else pos + 3
      lowerCAmelCase											=	[]  # (token_id, token, pos)
      for e in range(__a      , __a      , -1				):
        lowerCAmelCase											=	text[pos:e]
        if wd in self.vocab:
          if wd[0] == "<" and len(__a				) > 2:
            lowerCAmelCase											=	[(self.vocab[wd], wd, e)]
            break
          else:
            candidates.append((self.vocab[wd], wd, e)				)
      if len(__a				) > 0:
        # the smallest token_id is adopted
        lowerCAmelCase				,       lowerCAmelCase				,       lowerCAmelCase											=	sorted(__a      , key=lambda SCREAMING_SNAKE_CASE				: x[0]				)[0]
        result.append(__a				)
        lowerCAmelCase											=	e
      else:
        lowerCAmelCase											=	pos + 1
        lowerCAmelCase											=	text[pos:end]
        if check_simbol(__a				):
          result.append("<KIGOU>"				)
        elif checkuae(__a				):
          result.append("<U2000U2BFF>"				)
        else:
          for i in wd.encode("utf-8"				):
            result.append("<|byte%d|>" % i				)
        lowerCAmelCase											=	end
    return result


  def __A				(      self      :     List[Any]      , SCREAMING_SNAKE_CASE      :     Union[str, Any]      , SCREAMING_SNAKE_CASE      :     List[Any]="\n"				)	->						int:
    """simple docstring"""

    lowerCAmelCase											=	[]
    lowerCAmelCase											=	[]
    lowerCAmelCase											=	self.ids_to_tokens[index][0]
    if word[:6] == "<|byte" and word[-2:] == "|>":
      byte_tokens.append(int(word[6:-2]				)				)
    else:
      if len(__a				) > 0:
        words.append(bytearray(__a				).decode("utf-8"      , errors="replace"				)				)
        lowerCAmelCase											=	[]
      if word[:7] == "<|emoji" and word[-2:] == "|>":
        words.append(self.emoji["emoji_inv"][word]				)
      elif word == "<SP>":
        words.append(" "				)
      elif word == "<BR>":
        words.append(__a				)
      elif word == "<TAB>":
        words.append("\t"				)
      elif word == "<BLOCK>":
        words.append("▀"				)
      elif word == "<KIGOU>":
        words.append("ǀ"				)
      elif word == "<U2000U2BFF>":
        words.append("‖"				)
      else:
        words.append(__a				)
    if len(__a				) > 0:
      words.append(bytearray(__a				).decode("utf-8"      , errors="replace"				)				)
    lowerCAmelCase											=	"".join(__a				)
    return text




"""simple docstring"""





import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding






class        _UpperCAmelCase(       lowerCamelCase      ):
		lowercase__         = ['image_processor', 'tokenizer']
		lowercase__         = 'ViTImageProcessor'
		lowercase__         = ('CLIPTokenizer', 'CLIPTokenizerFast')



		def __init__(   self							,						__a=None							,						__a=None							,						**__a)			->			Union[str, Any]:






							'''simple docstring'''
							_UpperCamelCase									=    None
							if "feature_extractor" in kwargs:
												warnings.warn(
												    '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
												    ''' instead.'''							,						__a							,						)
												_UpperCamelCase									=    kwargs.pop('''feature_extractor''')

							_UpperCamelCase									=    image_processor if image_processor is not None else feature_extractor
							if image_processor is None:
												raise ValueError('''You need to specify an `image_processor`.''')
							if tokenizer is None:
												raise ValueError('''You need to specify a `tokenizer`.''')

							super().__init__(__a							,						__a)



		def __call__(   self							,						__a=None							,						__a=None							,						__a=None							,						__a=None							,						**__a)			->			Tuple:






							'''simple docstring'''
							if text is None and visual_prompt is None and images is None:
												raise ValueError('''You have to specify either text, visual prompt or images.''')

							if text is not None and visual_prompt is not None:
												raise ValueError('''You have to specify exactly one type of prompt. Either text or visual prompt.''')

							if text is not None:
												_UpperCamelCase									=    self.tokenizer(__a							,						return_tensors=__a							,						**__a)

							if visual_prompt is not None:
												_UpperCamelCase									=    self.image_processor(__a							,						return_tensors=__a							,						**__a)

							if images is not None:
												_UpperCamelCase									=    self.image_processor(__a							,						return_tensors=__a							,						**__a)

							if visual_prompt is not None and images is not None:
												_UpperCamelCase									=    {
												    '''pixel_values''': image_features.pixel_values,
												    '''conditional_pixel_values''': prompt_features.pixel_values,
												}
												return encoding
							elif text is not None and images is not None:
												_UpperCamelCase									=    image_features.pixel_values
												return encoding
							elif text is not None:
												return encoding
							elif visual_prompt is not None:
												_UpperCamelCase									=    {
												    '''conditional_pixel_values''': prompt_features.pixel_values,
												}
												return encoding
							else:
												return BatchEncoding(data=dict(**__a)							,						tensor_type=__a)



		def 		UpperCAmelCase				(   self							,						*__a							,						**__a)			->			Any:






							'''simple docstring'''
							return self.tokenizer.batch_decode(*__a							,						**__a)



		def 		UpperCAmelCase				(   self							,						*__a							,						**__a)			->			List[str]:






							'''simple docstring'''
							return self.tokenizer.decode(*__a							,						**__a)



		@property
		def 		UpperCAmelCase				(   self)			->			List[str]:






							'''simple docstring'''
							warnings.warn(
							    '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'''							,						__a							,						)
							return self.image_processor_class




		@property
		def 		UpperCAmelCase				(   self)			->			List[str]:






							'''simple docstring'''
							warnings.warn(
							    '''`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'''							,						__a							,						)
							return self.image_processor






