
import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowerCamelCase__     :		Any        = re.compile(R"""\b(a|an|the)\b""", re.UNICODE)

lowerCamelCase__     :		Optional[int]        = None

def 		UpperCamelCase						(   )		->						Optional[int]:


  '''simple docstring'''






  lowercase__	: Union[str, Any]     		=   argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0.""" )
  parser.add_argument("""data_file"""			,			metavar="""data.json"""			,			help="""Input data JSON file.""" )
  parser.add_argument("""pred_file"""			,			metavar="""pred.json"""			,			help="""Model predictions.""" )
  parser.add_argument(
      """--out-file"""			,			"""-o"""			,			metavar="""eval.json"""			,			help="""Write accuracy metrics to file (default is stdout).""" )
  parser.add_argument(
      """--na-prob-file"""			,			"""-n"""			,			metavar="""na_prob.json"""			,			help="""Model estimates of probability of no answer.""" )
  parser.add_argument(
      """--na-prob-thresh"""			,			"""-t"""			,			type=lowercase_			,			default=1.0			,			help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""			,			)
  parser.add_argument(
      """--out-image-dir"""			,			"""-p"""			,			metavar="""out_images"""			,			default=lowercase_			,			help="""Save precision-recall curves to directory.""" )
  parser.add_argument("""--verbose"""			,			"""-v"""			,			action="""store_true""" )
  if len(sys.argv ) == 1:
    parser.print_help()
    sys.exit(1 )
  return parser.parse_args()

def 		UpperCamelCase						(   lowercase_ )		->						Any:


  '''simple docstring'''






  lowercase__	: Optional[int]     		=   {}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        lowercase__	: Optional[Any]     		=   bool(qa["""answers"""]["""text"""] )
  return qid_to_has_ans

def 		UpperCamelCase						(   lowercase_ )		->						Union[str, Any]:


  '''simple docstring'''






  def remove_articles(lowercase_ ):
    return ARTICLES_REGEX.sub(""" """			,			lowercase_ )

  def white_space_fix(lowercase_ ):
    return " ".join(text.split() )

  def remove_punc(lowercase_ ):
    lowercase__	: Union[str, Any]     		=   set(string.punctuation )
    return "".join(ch for ch in text if ch not in exclude )

  def lower(lowercase_ ):
    return text.lower()

  return white_space_fix(remove_articles(remove_punc(lower(lowercase_ ) ) ) )

def 		UpperCamelCase						(   lowercase_ )		->						Tuple:


  '''simple docstring'''






  if not s:
    return []
  return normalize_answer(lowercase_ ).split()

def 		UpperCamelCase						(   lowercase_			,			lowercase_ )		->						List[Any]:


  '''simple docstring'''






  return int(normalize_answer(lowercase_ ) == normalize_answer(lowercase_ ) )

def 		UpperCamelCase						(   lowercase_			,			lowercase_ )		->						int:


  '''simple docstring'''






  lowercase__	: List[Any]     		=   get_tokens(lowercase_ )
  lowercase__	: Optional[int]     		=   get_tokens(lowercase_ )
  lowercase__	: List[Any]     		=   collections.Counter(lowercase_ ) & collections.Counter(lowercase_ )
  lowercase__	: Any     		=   sum(common.values() )
  if len(lowercase_ ) == 0 or len(lowercase_ ) == 0:
    # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
    return int(gold_toks == pred_toks )
  if num_same == 0:
    return 0
  lowercase__	: List[Any]     		=   1.0 * num_same / len(lowercase_ )
  lowercase__	: Any     		=   1.0 * num_same / len(lowercase_ )
  lowercase__	: List[str]     		=   (2 * precision * recall) / (precision + recall)
  return fa

def 		UpperCamelCase						(   lowercase_			,			lowercase_ )		->						Dict:


  '''simple docstring'''






  lowercase__	: Tuple     		=   {}
  lowercase__	: Optional[Any]     		=   {}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        lowercase__	: List[Any]     		=   qa["""id"""]
        lowercase__	: Union[str, Any]     		=   [t for t in qa["""answers"""]["""text"""] if normalize_answer(lowercase_ )]
        if not gold_answers:
          # For unanswerable questions, only correct answer is empty string
          lowercase__	: List[str]     		=   [""""""]
        if qid not in preds:
          print(F'Missing prediction for {qid}' )
          continue
        lowercase__	: Union[str, Any]     		=   preds[qid]
        # Take max over all gold answers
        lowercase__	: int     		=   max(compute_exact(lowercase_			,			lowercase_ ) for a in gold_answers )
        lowercase__	: Tuple     		=   max(compute_fa(lowercase_			,			lowercase_ ) for a in gold_answers )
  return exact_scores, fa_scores

def 		UpperCamelCase						(   lowercase_			,			lowercase_			,			lowercase_			,			lowercase_ )		->						int:


  '''simple docstring'''






  lowercase__	: Any     		=   {}
  for qid, s in scores.items():
    lowercase__	: Union[str, Any]     		=   na_probs[qid] > na_prob_thresh
    if pred_na:
      lowercase__	: Tuple     		=   float(not qid_to_has_ans[qid] )
    else:
      lowercase__	: Union[str, Any]     		=   s
  return new_scores

def 		UpperCamelCase						(   lowercase_			,			lowercase_			,			lowercase_=None )		->						int:


  '''simple docstring'''






  if not qid_list:
    lowercase__	: Optional[int]     		=   len(lowercase_ )
    return collections.OrderedDict(
        [
            ("""exact""", 100.0 * sum(exact_scores.values() ) / total),
            ("""f1""", 100.0 * sum(fa_scores.values() ) / total),
            ("""total""", total),
        ] )
  else:
    lowercase__	: Optional[int]     		=   len(lowercase_ )
    return collections.OrderedDict(
        [
            ("""exact""", 100.0 * sum(exact_scores[k] for k in qid_list ) / total),
            ("""f1""", 100.0 * sum(fa_scores[k] for k in qid_list ) / total),
            ("""total""", total),
        ] )

def 		UpperCamelCase						(   lowercase_			,			lowercase_			,			lowercase_ )		->						Union[str, Any]:


  '''simple docstring'''






  for k in new_eval:
    lowercase__	: int     		=   new_eval[k]

def 		UpperCamelCase						(   lowercase_			,			lowercase_			,			lowercase_			,			lowercase_ )		->						Any:


  '''simple docstring'''






  plt.step(lowercase_			,			lowercase_			,			color="""b"""			,			alpha=0.2			,			where="""post""" )
  plt.fill_between(lowercase_			,			lowercase_			,			step="""post"""			,			alpha=0.2			,			color="""b""" )
  plt.xlabel("""Recall""" )
  plt.ylabel("""Precision""" )
  plt.xlim([0.0, 1.05] )
  plt.ylim([0.0, 1.05] )
  plt.title(lowercase_ )
  plt.savefig(lowercase_ )
  plt.clf()

def 		UpperCamelCase						(   lowercase_			,			lowercase_			,			lowercase_			,			lowercase_			,			lowercase_=None			,			lowercase_=None )		->						Optional[int]:


  '''simple docstring'''






  lowercase__	: List[Any]     		=   sorted(lowercase_			,			key=lambda lowercase_ : na_probs[k] )
  lowercase__	: Any     		=   0.0
  lowercase__	: str     		=   1.0
  lowercase__	: Any     		=   0.0
  lowercase__	: Optional[int]     		=   [1.0]
  lowercase__	: Tuple     		=   [0.0]
  lowercase__	: Dict     		=   0.0
  for i, qid in enumerate(lowercase_ ):
    if qid_to_has_ans[qid]:
      true_pos += scores[qid]
    lowercase__	: List[Any]     		=   true_pos / float(i + 1 )
    lowercase__	: Tuple     		=   true_pos / float(lowercase_ )
    if i == len(lowercase_ ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
      # i.e., if we can put a threshold after this point
      avg_prec += cur_p * (cur_r - recalls[-1])
      precisions.append(lowercase_ )
      recalls.append(lowercase_ )
  if out_image:
    plot_pr_curve(lowercase_			,			lowercase_			,			lowercase_			,			lowercase_ )
  return {"ap": 100.0 * avg_prec}

def 		UpperCamelCase						(   lowercase_			,			lowercase_			,			lowercase_			,			lowercase_			,			lowercase_			,			lowercase_ )		->						str:


  '''simple docstring'''






  if out_image_dir and not os.path.exists(lowercase_ ):
    os.makedirs(lowercase_ )
  lowercase__	: Dict     		=   sum(1 for v in qid_to_has_ans.values() if v )
  if num_true_pos == 0:
    return
  lowercase__	: Optional[int]     		=   make_precision_recall_eval(
      lowercase_			,			lowercase_			,			lowercase_			,			lowercase_			,			out_image=os.path.join(lowercase_			,			"""pr_exact.png""" )			,			title="""Precision-Recall curve for Exact Match score"""			,			)
  lowercase__	: Union[str, Any]     		=   make_precision_recall_eval(
      lowercase_			,			lowercase_			,			lowercase_			,			lowercase_			,			out_image=os.path.join(lowercase_			,			"""pr_f1.png""" )			,			title="""Precision-Recall curve for F1 score"""			,			)
  lowercase__	: Union[str, Any]     		=   {k: float(lowercase_ ) for k, v in qid_to_has_ans.items()}
  lowercase__	: Dict     		=   make_precision_recall_eval(
      lowercase_			,			lowercase_			,			lowercase_			,			lowercase_			,			out_image=os.path.join(lowercase_			,			"""pr_oracle.png""" )			,			title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""			,			)
  merge_eval(lowercase_			,			lowercase_			,			"""pr_exact""" )
  merge_eval(lowercase_			,			lowercase_			,			"""pr_f1""" )
  merge_eval(lowercase_			,			lowercase_			,			"""pr_oracle""" )

def 		UpperCamelCase						(   lowercase_			,			lowercase_			,			lowercase_			,			lowercase_ )		->						Dict:


  '''simple docstring'''






  if not qid_list:
    return
  lowercase__	: Union[str, Any]     		=   [na_probs[k] for k in qid_list]
  lowercase__	: int     		=   np.ones_like(lowercase_ ) / float(len(lowercase_ ) )
  plt.hist(lowercase_			,			weights=lowercase_			,			bins=20			,			range=(0.0, 1.0) )
  plt.xlabel("""Model probability of no-answer""" )
  plt.ylabel("""Proportion of dataset""" )
  plt.title(F'Histogram of no-answer probability: {name}' )
  plt.savefig(os.path.join(lowercase_			,			F'na_prob_hist_{name}.png' ) )
  plt.clf()

def 		UpperCamelCase						(   lowercase_			,			lowercase_			,			lowercase_			,			lowercase_ )		->						Dict:


  '''simple docstring'''






  lowercase__	: Union[str, Any]     		=   sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k] )
  lowercase__	: Tuple     		=   num_no_ans
  lowercase__	: Dict     		=   cur_score
  lowercase__	: Tuple     		=   0.0
  lowercase__	: str     		=   sorted(lowercase_			,			key=lambda lowercase_ : na_probs[k] )
  for i, qid in enumerate(lowercase_ ):
    if qid not in scores:
      continue
    if qid_to_has_ans[qid]:
      lowercase__	: List[str]     		=   scores[qid]
    else:
      if preds[qid]:
        lowercase__	: Tuple     		=   -1
      else:
        lowercase__	: Optional[int]     		=   0
    cur_score += diff
    if cur_score > best_score:
      lowercase__	: Tuple     		=   cur_score
      lowercase__	: Optional[Any]     		=   na_probs[qid]
  return 100.0 * best_score / len(lowercase_ ), best_thresh

def 		UpperCamelCase						(   lowercase_			,			lowercase_			,			lowercase_			,			lowercase_			,			lowercase_			,			lowercase_ )		->						Optional[Any]:


  '''simple docstring'''






  lowercase__		,     lowercase__	: List[Any]     		=   find_best_thresh(lowercase_			,			lowercase_			,			lowercase_			,			lowercase_ )
  lowercase__		,     lowercase__	: Optional[Any]     		=   find_best_thresh(lowercase_			,			lowercase_			,			lowercase_			,			lowercase_ )
  lowercase__	: str     		=   best_exact
  lowercase__	: Tuple     		=   exact_thresh
  lowercase__	: str     		=   best_fa
  lowercase__	: Any     		=   fa_thresh

def 		UpperCamelCase						(   )		->						List[Any]:


  '''simple docstring'''






  with open(OPTS.data_file ) as f:
    lowercase__	: str     		=   json.load(lowercase_ )
    lowercase__	: int     		=   dataset_json["""data"""]
  with open(OPTS.pred_file ) as f:
    lowercase__	: List[str]     		=   json.load(lowercase_ )
  if OPTS.na_prob_file:
    with open(OPTS.na_prob_file ) as f:
      lowercase__	: int     		=   json.load(lowercase_ )
  else:
    lowercase__	: List[Any]     		=   {k: 0.0 for k in preds}
  lowercase__	: Optional[Any]     		=   make_qid_to_has_ans(lowercase_ )  # maps qid to True/False
  lowercase__	: int     		=   [k for k, v in qid_to_has_ans.items() if v]
  lowercase__	: Union[str, Any]     		=   [k for k, v in qid_to_has_ans.items() if not v]
  lowercase__		,     lowercase__	: List[Any]     		=   get_raw_scores(lowercase_			,			lowercase_ )
  lowercase__	: str     		=   apply_no_ans_threshold(lowercase_			,			lowercase_			,			lowercase_			,			OPTS.na_prob_thresh )
  lowercase__	: Optional[Any]     		=   apply_no_ans_threshold(lowercase_			,			lowercase_			,			lowercase_			,			OPTS.na_prob_thresh )
  lowercase__	: List[str]     		=   make_eval_dict(lowercase_			,			lowercase_ )
  if has_ans_qids:
    lowercase__	: Any     		=   make_eval_dict(lowercase_			,			lowercase_			,			qid_list=lowercase_ )
    merge_eval(lowercase_			,			lowercase_			,			"""HasAns""" )
  if no_ans_qids:
    lowercase__	: Optional[Any]     		=   make_eval_dict(lowercase_			,			lowercase_			,			qid_list=lowercase_ )
    merge_eval(lowercase_			,			lowercase_			,			"""NoAns""" )
  if OPTS.na_prob_file:
    find_all_best_thresh(lowercase_			,			lowercase_			,			lowercase_			,			lowercase_			,			lowercase_			,			lowercase_ )
  if OPTS.na_prob_file and OPTS.out_image_dir:
    run_precision_recall_analysis(lowercase_			,			lowercase_			,			lowercase_			,			lowercase_			,			lowercase_			,			OPTS.out_image_dir )
    histogram_na_prob(lowercase_			,			lowercase_			,			OPTS.out_image_dir			,			"""hasAns""" )
    histogram_na_prob(lowercase_			,			lowercase_			,			OPTS.out_image_dir			,			"""noAns""" )
  if OPTS.out_file:
    with open(OPTS.out_file			,			"""w""" ) as f:
      json.dump(lowercase_			,			lowercase_ )
  else:
    print(json.dumps(lowercase_			,			indent=2 ) )


if __name__ == "__main__":
     lowerCamelCase__     :		List[str]        = parse_args()
     if OPTS.out_image_dir:
          import matplotlib

          matplotlib.use("""Agg""")
          import matplotlib.pyplot as plt
     main()













'''simple docstring'''

import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation


a   	=    logging.get_logger(__name__)

a   	=    {"vocab_file": "vocab.txt", "emoji_file": "emoji.json"}

a   	=    {
    "vocab_file": {
        "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt",
    },
    "emoji_file": {
        "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json",
    },
}

a   	=    {
    "abeja/gpt-neox-japanese-2.7b": 2048,
}

def     __magic_name__							(				__UpperCAmelCase				,						__UpperCAmelCase   )    ->       Any:

   '''simple docstring'''







   with open(__UpperCAmelCase				,						"""r"""				,						encoding="""utf-8"""   ) as f:
      __SCREAMING_SNAKE_CASE   				=							json.loads(f.read()   )

   __SCREAMING_SNAKE_CASE   				=							collections.OrderedDict()
   __SCREAMING_SNAKE_CASE   				=							collections.OrderedDict()
   __SCREAMING_SNAKE_CASE   				=							collections.OrderedDict()
   with open(__UpperCAmelCase				,						"""r"""				,						encoding="""utf-8"""   ) as f:
      __SCREAMING_SNAKE_CASE   				=							f.readlines()
   __SCREAMING_SNAKE_CASE   				=							[[t.rstrip("""\n"""   )] if (t == """,""" or """,""" not in t) else t.rstrip("""\n"""   ).split(""","""   ) for t in token]
   for idx, b in enumerate(__UpperCAmelCase   ):
      __SCREAMING_SNAKE_CASE   				=							b
      __SCREAMING_SNAKE_CASE   				=							idx
      for wd in b:
         __SCREAMING_SNAKE_CASE   				=							idx

   return vocab, raw_vocab, ids_to_tokens, emoji







class __a			( _snake_case      ):
   __UpperCamelCase :			Tuple						   =						VOCAB_FILES_NAMES
   __UpperCamelCase :			Tuple						   =						PRETRAINED_VOCAB_FILES_MAP
   __UpperCamelCase :			Any						   =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCamelCase :			Union[str, Any]						   =						['input_ids', 'attention_mask']





   def __init__(       self					:      Dict					,lowerCamelCase					:      Union[str, Any]					,lowerCamelCase					:      int					,lowerCamelCase					:      str="<|endoftext|>"					,lowerCamelCase					:      List[Any]="<|endoftext|>"					,lowerCamelCase					:      str="<|startoftext|>"					,lowerCamelCase					:      Dict="<|endoftext|>"					,lowerCamelCase					:      int=False					,**lowerCamelCase					:      str					,):

      '''simple docstring'''

      super().__init__(
          unk_token=lowerCamelCase					,pad_token=lowerCamelCase					,bos_token=lowerCamelCase					,eos_token=lowerCamelCase					,do_clean_text=lowerCamelCase					,**lowerCamelCase					,)
      if not os.path.isfile(lowerCamelCase     ):
         raise ValueError(
             f"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
             """ model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"""     )
      if not os.path.isfile(lowerCamelCase     ):
         raise ValueError(
             f"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
             """ pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"""     )
      __SCREAMING_SNAKE_CASE   				=							do_clean_text
      __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE   				=							load_vocab_and_emoji(lowerCamelCase					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							SubWordJapaneseTokenizer(
          vocab=self.vocab					,ids_to_tokens=self.ids_to_tokens					,emoji=self.emoji     )





   @property
   def   UpperCAmelCase__  (       self					:      Optional[Any]     ):

      '''simple docstring'''

      return len(self.raw_vocab     )





   def   UpperCAmelCase__  (       self					:      List[str]     ):

      '''simple docstring'''

      return dict(self.raw_vocab					,**self.added_tokens_encoder     )





   def   UpperCAmelCase__  (       self					:      Dict					,lowerCamelCase					:      str     ):

      '''simple docstring'''

      return self.subword_tokenizer.tokenize(lowerCamelCase					,clean=self.do_clean_text     )





   def   UpperCAmelCase__  (       self					:      Union[str, Any]					,lowerCamelCase					:      Union[str, Any]     ):

      '''simple docstring'''

      return self.vocab.get(lowerCamelCase					,self.vocab.get(self.unk_token     )     )





   def   UpperCAmelCase__  (       self					:      List[Any]					,lowerCamelCase					:      Optional[Any]     ):

      '''simple docstring'''

      return self.subword_tokenizer.convert_id_to_token(lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      List[str]					,lowerCamelCase					:      str     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							"""""".join(lowerCamelCase     ).strip()
      return out_string





   def   UpperCAmelCase__  (       self					:      List[str]					,lowerCamelCase					:      "Conversation"     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							[]
      for is_user, text in conversation.iter_texts():
         input_ids.extend(self.encode(lowerCamelCase					,add_special_tokens=lowerCamelCase     ) + [self.eos_token_id]     )

      if len(lowerCamelCase     ) > self.model_max_length:
         __SCREAMING_SNAKE_CASE   				=							input_ids[-self.model_max_length :]
      return input_ids





   def   UpperCAmelCase__  (       self					:      Dict					,lowerCamelCase					:      str					,lowerCamelCase					:      Optional[str] = None     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							0
      if os.path.isdir(lowerCamelCase     ):
         __SCREAMING_SNAKE_CASE   				=							os.path.join(
             lowerCamelCase					,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]     )
         __SCREAMING_SNAKE_CASE   				=							os.path.join(
             lowerCamelCase					,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""emoji_file"""]     )
      else:
         __SCREAMING_SNAKE_CASE   				=							(
             (filename_prefix + """-""" if filename_prefix else """""") + save_directory + VOCAB_FILES_NAMES["""vocab_file"""]
         )
         __SCREAMING_SNAKE_CASE   				=							(
             (filename_prefix + """-""" if filename_prefix else """""") + save_directory + VOCAB_FILES_NAMES["""emoji_file"""]
         )
      with open(lowerCamelCase					,"""w"""					,encoding="""utf-8"""     ) as writer:
         for token_index, token in self.ids_to_tokens.items():
            if index != token_index:
               logger.warning(
                   f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                   """ Please check that the vocabulary is not corrupted!"""     )
               __SCREAMING_SNAKE_CASE   				=							token_index
            writer.write(""",""".join(lowerCamelCase     ) + """\n"""     )
            index += 1
      with open(lowerCamelCase					,"""w"""					,encoding="""utf-8"""     ) as writer:
         json.dump(self.emoji					,lowerCamelCase     )
      return vocab_file, emoji_file







class __a			( _snake_case      ):





   def __init__(       self					:      Tuple					,lowerCamelCase					:      List[str]					,lowerCamelCase					:      Union[str, Any]					,lowerCamelCase					:      Optional[int]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							vocab  # same as swe
      __SCREAMING_SNAKE_CASE   				=							ids_to_tokens  # same as bpe
      __SCREAMING_SNAKE_CASE   				=							emoji
      __SCREAMING_SNAKE_CASE   				=							np.max([len(lowerCamelCase     ) for w in self.vocab.keys()]     )
      __SCREAMING_SNAKE_CASE   				=							re.compile(r"""(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)"""     )
      __SCREAMING_SNAKE_CASE   				=							re.compile(r"""[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*"""     )
      __SCREAMING_SNAKE_CASE   				=							re.compile(r"""[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}"""     )
      __SCREAMING_SNAKE_CASE   				=							re.compile(
          r"""([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"""     )
      __SCREAMING_SNAKE_CASE   				=							re.compile(
          r"""(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"""     )
      __SCREAMING_SNAKE_CASE   				=							re.compile(
          r"""((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*"""     )
      __SCREAMING_SNAKE_CASE   				=							"""─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"""
      __SCREAMING_SNAKE_CASE   				=							"""▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"""
      __SCREAMING_SNAKE_CASE   				=							str.maketrans({k: """<BLOCK>""" for k in keisen + blocks}     )





   def __len__(       self					:      List[str]     ):

      '''simple docstring'''

      return len(self.ids_to_tokens     )





   def   UpperCAmelCase__  (       self					:      Optional[int]					,lowerCamelCase					:      Dict     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.content_repattera.sub("""<URL>"""					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							self.content_repattera.sub("""<EMAIL>"""					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							self.content_repattera.sub("""<TEL>"""					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							self.content_repattera.sub("""<DATE>"""					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							self.content_repattera.sub("""<DATE>"""					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							self.content_repattera.sub("""<PRICE>"""					,lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							content.translate(self.content_transa     )
      while "<BLOCK><BLOCK>" in content:
         __SCREAMING_SNAKE_CASE   				=							content.replace("""<BLOCK><BLOCK>"""					,"""<BLOCK>"""     )
      return content





   def   UpperCAmelCase__  (       self					:      Dict					,lowerCamelCase					:      Optional[Any]					,lowerCamelCase					:      List[str]=False     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							text.replace(""" """					,"""<SP>"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""　"""					,"""<SP>"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""\r\n"""					,"""<BR>"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""\n"""					,"""<BR>"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""\r"""					,"""<BR>"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""\t"""					,"""<TAB>"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""—"""					,"""ー"""     )
      __SCREAMING_SNAKE_CASE   				=							text.replace("""−"""					,"""ー"""     )
      for k, v in self.emoji["emoji"].items():
         if k in text:
            __SCREAMING_SNAKE_CASE   				=							text.replace(lowerCamelCase					,lowerCamelCase     )
      if clean:
         __SCREAMING_SNAKE_CASE   				=							self.clean_text(lowerCamelCase     )

      def check_simbol(lowerCamelCase					:      List[str]     ):
         __SCREAMING_SNAKE_CASE   				=							x.encode()
         if len(lowerCamelCase     ) == 1 and len(lowerCamelCase     ) == 2:
            __SCREAMING_SNAKE_CASE   				=							(int(e[0]     ) << 8) + int(e[1]     )
            if (
                (c >= 0xC_2A1 and c <= 0xC_2BF)
                or (c >= 0xC_780 and c <= 0xC_783)
                or (c >= 0xC_AB9 and c <= 0xC_BBF)
                or (c >= 0xC_C80 and c <= 0xC_DA2)
            ):
               return True
         return False

      def checkuae(lowerCamelCase					:      Union[str, Any]     ):
         __SCREAMING_SNAKE_CASE   				=							x.encode()
         if len(lowerCamelCase     ) == 1 and len(lowerCamelCase     ) == 3:
            __SCREAMING_SNAKE_CASE   				=							(int(e[0]     ) << 16) + (int(e[1]     ) << 8) + int(e[2]     )
            if c >= 0xE28_080 and c <= 0xE2B_07F:
               return True
         return False

      __SCREAMING_SNAKE_CASE   				=							0
      __SCREAMING_SNAKE_CASE   				=							[]
      while pos < len(lowerCamelCase     ):
         __SCREAMING_SNAKE_CASE   				=							min(len(lowerCamelCase     )					,pos + self.maxlen + 1     ) if text[pos] == """<""" else pos + 3
         __SCREAMING_SNAKE_CASE   				=							[]  # (token_id, token, pos)
         for e in range(lowerCamelCase					,lowerCamelCase					,-1     ):
            __SCREAMING_SNAKE_CASE   				=							text[pos:e]
            if wd in self.vocab:
               if wd[0] == "<" and len(lowerCamelCase     ) > 2:
                  __SCREAMING_SNAKE_CASE   				=							[(self.vocab[wd], wd, e)]
                  break
               else:
                  candidates.append((self.vocab[wd], wd, e)     )
         if len(lowerCamelCase     ) > 0:
            # the smallest token_id is adopted
            __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE				,  __SCREAMING_SNAKE_CASE   				=							sorted(lowerCamelCase					,key=lambda lowerCamelCase     : x[0]     )[0]
            result.append(lowerCamelCase     )
            __SCREAMING_SNAKE_CASE   				=							e
         else:
            __SCREAMING_SNAKE_CASE   				=							pos + 1
            __SCREAMING_SNAKE_CASE   				=							text[pos:end]
            if check_simbol(lowerCamelCase     ):
               result.append("""<KIGOU>"""     )
            elif checkuae(lowerCamelCase     ):
               result.append("""<U2000U2BFF>"""     )
            else:
               for i in wd.encode("""utf-8"""     ):
                  result.append("""<|byte%d|>""" % i     )
            __SCREAMING_SNAKE_CASE   				=							end
      return result





   def   UpperCAmelCase__  (       self					:      Optional[Any]					,lowerCamelCase					:      Dict					,lowerCamelCase					:      Optional[Any]="\n"     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							[]
      __SCREAMING_SNAKE_CASE   				=							[]
      __SCREAMING_SNAKE_CASE   				=							self.ids_to_tokens[index][0]
      if word[:6] == "<|byte" and word[-2:] == "|>":
         byte_tokens.append(int(word[6:-2]     )     )
      else:
         if len(lowerCamelCase     ) > 0:
            words.append(bytearray(lowerCamelCase     ).decode("""utf-8"""					,errors="""replace"""     )     )
            __SCREAMING_SNAKE_CASE   				=							[]
         if word[:7] == "<|emoji" and word[-2:] == "|>":
            words.append(self.emoji["""emoji_inv"""][word]     )
         elif word == "<SP>":
            words.append(""" """     )
         elif word == "<BR>":
            words.append(lowerCamelCase     )
         elif word == "<TAB>":
            words.append("""\t"""     )
         elif word == "<BLOCK>":
            words.append("""▀"""     )
         elif word == "<KIGOU>":
            words.append("""ǀ"""     )
         elif word == "<U2000U2BFF>":
            words.append("""‖"""     )
         else:
            words.append(lowerCamelCase     )
      if len(lowerCamelCase     ) > 0:
         words.append(bytearray(lowerCamelCase     ).decode("""utf-8"""					,errors="""replace"""     )     )
      __SCREAMING_SNAKE_CASE   				=							"""""".join(lowerCamelCase     )
      return text





