





'''simple docstring'''

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


a   	=    {
    "configuration_deberta": ["DEBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP", "DebertaConfig", "DebertaOnnxConfig"],
    "tokenization_deberta": ["DebertaTokenizer"],
}

try:
    if not is_tokenizers_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    a   	=    ["DebertaTokenizerFast"]

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    a   	=    [
        "DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST",
        "DebertaForMaskedLM",
        "DebertaForQuestionAnswering",
        "DebertaForSequenceClassification",
        "DebertaForTokenClassification",
        "DebertaModel",
        "DebertaPreTrainedModel",
    ]

try:
    if not is_tf_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    a   	=    [
        "TF_DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST",
        "TFDebertaForMaskedLM",
        "TFDebertaForQuestionAnswering",
        "TFDebertaForSequenceClassification",
        "TFDebertaForTokenClassification",
        "TFDebertaModel",
        "TFDebertaPreTrainedModel",
    ]


if TYPE_CHECKING:
    from .configuration_deberta import DEBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, DebertaConfig, DebertaOnnxConfig
    from .tokenization_deberta import DebertaTokenizer

    try:
        if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_deberta_fast import DebertaTokenizerFast

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_deberta import (
            DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
            DebertaForMaskedLM,
            DebertaForQuestionAnswering,
            DebertaForSequenceClassification,
            DebertaForTokenClassification,
            DebertaModel,
            DebertaPreTrainedModel,
        )

    try:
        if not is_tf_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_tf_deberta import (
            TF_DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
            TFDebertaForMaskedLM,
            TFDebertaForQuestionAnswering,
            TFDebertaForSequenceClassification,
            TFDebertaForTokenClassification,
            TFDebertaModel,
            TFDebertaPreTrainedModel,
        )


else:
    import sys

    a   	=    _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)












'''simple docstring'''

import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation


a   	=    logging.get_logger(__name__)

a   	=    {"vocab_file": "vocab.txt", "emoji_file": "emoji.json"}

a   	=    {
    "vocab_file": {
        "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt",
    },
    "emoji_file": {
        "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json",
    },
}

a   	=    {
    "abeja/gpt-neox-japanese-2.7b": 2048,
}

def     __magic_name__							(				__UpperCAmelCase				,						__UpperCAmelCase   )    ->       Any:

   '''simple docstring'''







   with open(__UpperCAmelCase				,						"""r"""				,						encoding="""utf-8"""   ) as f:
      __SCREAMING_SNAKE_CASE   				=							json.loads(f.read()   )

   __SCREAMING_SNAKE_CASE   				=							collections.OrderedDict()
   __SCREAMING_SNAKE_CASE   				=							collections.OrderedDict()
   __SCREAMING_SNAKE_CASE   				=							collections.OrderedDict()
   with open(__UpperCAmelCase				,						"""r"""				,						encoding="""utf-8"""   ) as f:
      __SCREAMING_SNAKE_CASE   				=							f.readlines()
   __SCREAMING_SNAKE_CASE   				=							[[t.rstrip("""\n"""   )] if (t == """,""" or """,""" not in t) else t.rstrip("""\n"""   ).split(""","""   ) for t in token]
   for idx, b in enumerate(__UpperCAmelCase   ):
      __SCREAMING_SNAKE_CASE   				=							b
      __SCREAMING_SNAKE_CASE   				=							idx
      for wd in b:
         __SCREAMING_SNAKE_CASE   				=							idx

   return vocab, raw_vocab, ids_to_tokens, emoji







class __a			( _snake_case      ):
   __UpperCamelCase :			Tuple						   =						VOCAB_FILES_NAMES
   __UpperCamelCase :			Tuple						   =						PRETRAINED_VOCAB_FILES_MAP
   __UpperCamelCase :			Any						   =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCamelCase :			Union[str, Any]						   =						['input_ids', 'attention_mask']





   def __init__(       self					:      Dict					,lowerCamelCase					:      Union[str, Any]					,lowerCamelCase					:      int					,lowerCamelCase					:      str="<|endoftext|>"					,lowerCamelCase					:      List[Any]="<|endoftext|>"					,lowerCamelCase					:      str="<|startoftext|>"					,lowerCamelCase					:      Dict="<|endoftext|>"					,lowerCamelCase					:      int=False					,**lowerCamelCase					:      str					,):

      '''simple docstring'''

      super().__init__(
          unk_token=lowerCamelCase					,pad_token=lowerCamelCase					,bos_token=lowerCamelCase					,eos_token=lowerCamelCase					,do_clean_text=lowerCamelCase					,**lowerCamelCase					,)
      if not os.path.isfile(lowerCamelCase     ):
         raise ValueError(
             f"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
             """ model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"""     )
      if not os.path.isfile(lowerCamelCase     ):
         raise ValueError(
             f"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
             """ pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"""     )
      __SCREAMING_SNAKE_CASE   				=							do_clean_text
      __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE   				=							load_vocab_and_emoji(lowerCamelCase					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							SubWordJapaneseTokenizer(
          vocab=self.vocab					,ids_to_tokens=self.ids_to_tokens					,emoji=self.emoji     )





   @property
   def   UpperCAmelCase__  (       self					:      Optional[Any]     ):

      '''simple docstring'''

      return len(self.raw_vocab     )





   def   UpperCAmelCase__  (       self					:      List[str]     ):

      '''simple docstring'''

      return dict(self.raw_vocab					,**self.added_tokens_encoder     )





   def   UpperCAmelCase__  (       self					:      Dict					,lowerCamelCase					:      str     ):

      '''simple docstring'''

      return self.subword_tokenizer.tokenize(lowerCamelCase					,clean=self.do_clean_text     )





   def   UpperCAmelCase__  (       self					:      Union[str, Any]					,lowerCamelCase					:      Union[str, Any]     ):

      '''simple docstring'''

      return self.vocab.get(lowerCamelCase					,self.vocab.get(self.unk_token     )     )





   def   UpperCAmelCase__  (       self					:      List[Any]					,lowerCamelCase					:      Optional[Any]     ):

      '''simple docstring'''

      return self.subword_tokenizer.convert_id_to_token(lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      List[str]					,lowerCamelCase					:      str     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							"""""".join(lowerCamelCase     ).strip()
      return out_string





   def   UpperCAmelCase__  (       self					:      List[str]					,lowerCamelCase					:      "Conversation"     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							[]
      for is_user, text in conversation.iter_texts():
         input_ids.extend(self.encode(lowerCamelCase					,add_special_tokens=lowerCamelCase     ) + [self.eos_token_id]     )

      if len(lowerCamelCase     ) > self.model_max_length:
         __SCREAMING_SNAKE_CASE   				=							input_ids[-self.model_max_length :]
      return input_ids





   def   UpperCAmelCase__  (       self					:      Dict					,lowerCamelCase					:      str					,lowerCamelCase					:      Optional[str] = None     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							0
      if os.path.isdir(lowerCamelCase     ):
         __SCREAMING_SNAKE_CASE   				=							os.path.join(
             lowerCamelCase					,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]     )
         __SCREAMING_SNAKE_CASE   				=							os.path.join(
             lowerCamelCase					,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""emoji_file"""]     )
      else:
         __SCREAMING_SNAKE_CASE   				=							(
             (filename_prefix + """-""" if filename_prefix else """""") + save_directory + VOCAB_FILES_NAMES["""vocab_file"""]
         )
         __SCREAMING_SNAKE_CASE   				=							(
             (filename_prefix + """-""" if filename_prefix else """""") + save_directory + VOCAB_FILES_NAMES["""emoji_file"""]
         )
      with open(lowerCamelCase					,"""w"""					,encoding="""utf-8"""     ) as writer:
         for token_index, token in self.ids_to_tokens.items():
            if index != token_index:
               logger.warning(
                   f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                   """ Please check that the vocabulary is not corrupted!"""     )
               __SCREAMING_SNAKE_CASE   				=							token_index
            writer.write(""",""".join(lowerCamelCase     ) + """\n"""     )
            index += 1
      with open(lowerCamelCase					,"""w"""					,encoding="""utf-8"""     ) as writer:
         json.dump(self.emoji					,lowerCamelCase     )
      return vocab_file, emoji_file







class __a			( _snake_case      ):





   def __init__(       self					:      Tuple					,lowerCamelCase					:      List[str]					,lowerCamelCase					:      Union[str, Any]					,lowerCamelCase					:      Optional[int]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							vocab  # same as swe
      __SCREAMING_SNAKE_CASE   				=							ids_to_tokens  # same as bpe
      __SCREAMING_SNAKE_CASE   				=							emoji
      __SCREAMING_SNAKE_CASE   				=							np.max([len(lowerCamelCase     ) for w in self.vocab.keys()]     )
      __SCREAMING_SNAKE_CASE   				=							re.compile(r"""(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)"""     )
      __SCREAMING_SNAKE_CASE   				=							re.compile(r"""[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*"""     )
      __SCREAMING_SNAKE_CASE   				=							re.compile(r"""[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}"""     )
      __SCREAMING_SNAKE_CASE   				=							re.compile(
          r"""([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"""     )
      __SCREAMING_SNAKE_CASE   				=							re.compile(
          r"""(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"""     )
      __SCREAMING_SNAKE_CASE   				=							re.compile(
          r"""((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*"""     )
      __SCREAMING_SNAKE_CASE   				=							"""─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"""
      __SCREAMING_SNAKE_CASE   				=							"""▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"""
      __SCREAMING_SNAKE_CASE   				=							str.maketrans({k: """<BLOCK>""" for k in keisen + blocks}     )





   def __len__(       self					:      List[str]     ):

      '''simple docstring'''

      return len(self.ids_to_tokens     )





   def   UpperCAmelCase__  (       self					:      Optional[int]					,lowerCamelCase					:      Dict     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.content_repattera.sub("""<URL>"""					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							self.content_repattera.sub("""<EMAIL>"""					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							self.content_repattera.sub("""<TEL>"""					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							self.content_repattera.sub("""<DATE>"""					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							self.content_repattera.sub("""<DATE>"""					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							self.content_repattera.sub("""<PRICE>"""					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							content.translate(self.content_transa     )
      while "<BLOCK><BLOCK>" in content:
         __SCREAMING_SNAKE_CASE   				=							content.replace("""<BLOCK><BLOCK>"""					,"""<BLOCK>"""     )
      return content





   def   UpperCAmelCase__  (       self					:      Dict					,lowerCamelCase					:      Optional[Any]					,lowerCamelCase					:      List[str]=False     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							text.replace(""" """					,"""<SP>"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""　"""					,"""<SP>"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""\r\n"""					,"""<BR>"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""\n"""					,"""<BR>"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""\r"""					,"""<BR>"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""\t"""					,"""<TAB>"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""—"""					,"""ー"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""−"""					,"""ー"""     )
      for k, v in self.emoji["emoji"].items():
         if k in text:
            __SCREAMING_SNAKE_CASE   				=							text.replace(lowerCamelCase					,lowerCamelCase     )
      if clean:
         __SCREAMING_SNAKE_CASE   				=							self.clean_text(lowerCamelCase     )

      def check_simbol(lowerCamelCase					:      List[str]     ):
         __SCREAMING_SNAKE_CASE   				=							x.encode()
         if len(lowerCamelCase     ) == 1 and len(lowerCamelCase     ) == 2:
            __SCREAMING_SNAKE_CASE   				=							(int(e[0]     ) << 8) + int(e[1]     )
            if (
                (c >= 0xC_2A1 and c <= 0xC_2BF)
                or (c >= 0xC_780 and c <= 0xC_783)
                or (c >= 0xC_AB9 and c <= 0xC_BBF)
                or (c >= 0xC_C80 and c <= 0xC_DA2)
            ):
               return True
         return False

      def checkuae(lowerCamelCase					:      Union[str, Any]     ):
         __SCREAMING_SNAKE_CASE   				=							x.encode()
         if len(lowerCamelCase     ) == 1 and len(lowerCamelCase     ) == 3:
            __SCREAMING_SNAKE_CASE   				=							(int(e[0]     ) << 16) + (int(e[1]     ) << 8) + int(e[2]     )
            if c >= 0xE28_080 and c <= 0xE2B_07F:
               return True
         return False

      __SCREAMING_SNAKE_CASE   				=							0
      __SCREAMING_SNAKE_CASE   				=							[]
      while pos < len(lowerCamelCase     ):
         __SCREAMING_SNAKE_CASE   				=							min(len(lowerCamelCase     )					,pos + self.maxlen + 1     ) if text[pos] == """<""" else pos + 3
         __SCREAMING_SNAKE_CASE   				=							[]  # (token_id, token, pos)
         for e in range(lowerCamelCase					,lowerCamelCase					,-1     ):
            __SCREAMING_SNAKE_CASE   				=							text[pos:e]
            if wd in self.vocab:
               if wd[0] == "<" and len(lowerCamelCase     ) > 2:
                  __SCREAMING_SNAKE_CASE   				=							[(self.vocab[wd], wd, e)]
                  break
               else:
                  candidates.append((self.vocab[wd], wd, e)     )
         if len(lowerCamelCase     ) > 0:
            # the smallest token_id is adopted
            __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE   				=							sorted(lowerCamelCase					,key=lambda lowerCamelCase     : x[0]     )[0]
            result.append(lowerCamelCase     )
            __SCREAMING_SNAKE_CASE   				=							e
         else:
            __SCREAMING_SNAKE_CASE   				=							pos + 1
            __SCREAMING_SNAKE_CASE   				=							text[pos:end]
            if check_simbol(lowerCamelCase     ):
               result.append("""<KIGOU>"""     )
            elif checkuae(lowerCamelCase     ):
               result.append("""<U2000U2BFF>"""     )
            else:
               for i in wd.encode("""utf-8"""     ):
                  result.append("""<|byte%d|>""" % i     )
            __SCREAMING_SNAKE_CASE   				=							end
      return result





   def   UpperCAmelCase__  (       self					:      Optional[Any]					,lowerCamelCase					:      Dict					,lowerCamelCase					:      Optional[Any]="\n"     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							[]
      __SCREAMING_SNAKE_CASE   				=							[]
      __SCREAMING_SNAKE_CASE   				=							self.ids_to_tokens[index][0]
      if word[:6] == "<|byte" and word[-2:] == "|>":
         byte_tokens.append(int(word[6:-2]     )     )
      else:
         if len(lowerCamelCase     ) > 0:
            words.append(bytearray(lowerCamelCase     ).decode("""utf-8"""					,errors="""replace"""     )     )
            __SCREAMING_SNAKE_CASE   				=							[]
         if word[:7] == "<|emoji" and word[-2:] == "|>":
            words.append(self.emoji["""emoji_inv"""][word]     )
         elif word == "<SP>":
            words.append(""" """     )
         elif word == "<BR>":
            words.append(lowerCamelCase     )
         elif word == "<TAB>":
            words.append("""\t"""     )
         elif word == "<BLOCK>":
            words.append("""▀"""     )
         elif word == "<KIGOU>":
            words.append("""ǀ"""     )
         elif word == "<U2000U2BFF>":
            words.append("""‖"""     )
         else:
            words.append(lowerCamelCase     )
      if len(lowerCamelCase     ) > 0:
         words.append(bytearray(lowerCamelCase     ).decode("""utf-8"""					,errors="""replace"""     )     )
      __SCREAMING_SNAKE_CASE   				=							"""""".join(lowerCamelCase     )
      return text





