


import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation


UpperCAmelCase    =					logging.get_logger(__name__)

UpperCAmelCase    =					{"""vocab_file""": """vocab.txt""", """emoji_file""": """emoji.json"""}

UpperCAmelCase    =					{
    """vocab_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt""",
    },
    """emoji_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json""",
    },
}

UpperCAmelCase    =					{
    """abeja/gpt-neox-japanese-2.7b""": 2048,
}






def  __lowerCAmelCase					(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      )->						Optional[int]:




      """simple docstring"""

      with open(_lowerCAmelCase      ,      '''r'''      ,      encoding='''utf-8'''      ) as f:
            snake_case_ =   json.loads(f.read()      )

      snake_case_ =   collections.OrderedDict()
      snake_case_ =   collections.OrderedDict()
      snake_case_ =   collections.OrderedDict()
      with open(_lowerCAmelCase      ,      '''r'''      ,      encoding='''utf-8'''      ) as f:
            snake_case_ =   f.readlines()
      snake_case_ =   [[t.rstrip('''\n'''      )] if (t == ''',''' or ''',''' not in t) else t.rstrip('''\n'''      ).split(''','''      ) for t in token]
      for idx, b in enumerate(_lowerCAmelCase      ):
            snake_case_ =   b
            snake_case_ =   idx
            for wd in b:
                  snake_case_ =   idx

      return vocab, raw_vocab, ids_to_tokens, emoji







class      lowerCAmelCase_		(				__UpperCAmelCase ):

 '''simple docstring'''

 __snake_case				=						VOCAB_FILES_NAMES
 __snake_case				=						PRETRAINED_VOCAB_FILES_MAP
 __snake_case				=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __snake_case				=						["input_ids", "attention_mask"]



 def __init__(			self	,   _UpperCAmelCase	,   _UpperCAmelCase	,   _UpperCAmelCase="<|endoftext|>"	,   _UpperCAmelCase="<|endoftext|>"	,   _UpperCAmelCase="<|startoftext|>"	,   _UpperCAmelCase="<|endoftext|>"	,   _UpperCAmelCase=False	,   **_UpperCAmelCase	,   ):
       super().__init__(
           unk_token=_lowerCamelCase	,   pad_token=_lowerCamelCase	,   bos_token=_lowerCamelCase	,   eos_token=_lowerCamelCase	,   do_clean_text=_lowerCamelCase	,   **_lowerCamelCase	,   )
       if not os.path.isfile(_lowerCamelCase      ):
             raise ValueError(
                 F'''Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'''
                 ''' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'''      )
       if not os.path.isfile(_lowerCamelCase      ):
             raise ValueError(
                 F'''Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'''
                 ''' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'''      )
       snake_case_ =   do_clean_text
       snake_case_ ,					snake_case_ ,					snake_case_ ,					snake_case_ =   load_vocab_and_emoji(_lowerCamelCase	,   _lowerCamelCase      )
       snake_case_ =   SubWordJapaneseTokenizer(
           vocab=self.vocab	,   ids_to_tokens=self.ids_to_tokens	,   emoji=self.emoji      )



 @property
 def 		UpperCamelCase__  (			self      ):
       # self.vocab contains support for character fluctuation unique to Japanese, and has a large number of vocab
       return len(self.raw_vocab      )



 def 		UpperCamelCase__  (			self      ):
       return dict(self.raw_vocab	,   **self.added_tokens_encoder      )



 def 		UpperCamelCase__  (			self	,   _UpperCAmelCase      ):
       return self.subword_tokenizer.tokenize(_lowerCamelCase	,   clean=self.do_clean_text      )



 def 		UpperCamelCase__  (			self	,   _UpperCAmelCase      ):
       return self.vocab.get(_lowerCamelCase	,   self.vocab.get(self.unk_token      )      )



 def 		UpperCamelCase__  (			self	,   _UpperCAmelCase      ):
       return self.subword_tokenizer.convert_id_to_token(_lowerCamelCase      )



 def 		UpperCamelCase__  (			self	,   _UpperCAmelCase      ):
       snake_case_ =   ''''''.join(_lowerCamelCase      ).strip()
       return out_string



 def 		UpperCamelCase__  (			self	,   _UpperCAmelCase      ):
       snake_case_ =   []
       for is_user, text in conversation.iter_texts():
             input_ids.extend(self.encode(_lowerCamelCase	,   add_special_tokens=_lowerCamelCase      ) + [self.eos_token_id]      )

       if len(_lowerCamelCase      ) > self.model_max_length:
             snake_case_ =   input_ids[-self.model_max_length :]
       return input_ids





 def 		UpperCamelCase__  (			self	,   _UpperCAmelCase	,   _UpperCAmelCase = None      ):
       snake_case_ =   0
       if os.path.isdir(_lowerCamelCase      ):
             snake_case_ =   os.path.join(
                 _lowerCamelCase	,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )
             snake_case_ =   os.path.join(
                 _lowerCamelCase	,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''emoji_file''']      )
       else:
             snake_case_ =   (
                 (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''vocab_file''']
             )
             snake_case_ =   (
                 (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''emoji_file''']
             )
       with open(_lowerCamelCase	,   '''w'''	,   encoding='''utf-8'''      ) as writer:
             for token_index, token in self.ids_to_tokens.items():
                   if index != token_index:
                         logger.warning(
                             F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
                             ''' Please check that the vocabulary is not corrupted!'''      )
                         snake_case_ =   token_index
                   writer.write(''','''.join(_lowerCamelCase      ) + '''\n'''      )
                   index += 1
       with open(_lowerCamelCase	,   '''w'''	,   encoding='''utf-8'''      ) as writer:
             json.dump(self.emoji	,   _lowerCamelCase      )
       return vocab_file, emoji_file







class      lowerCAmelCase_		(				__UpperCAmelCase ):

 '''simple docstring'''

 def __init__(			self	,   _UpperCAmelCase	,   _UpperCAmelCase	,   _UpperCAmelCase      ):
       snake_case_ =   vocab  # same as swe
       snake_case_ =   ids_to_tokens  # same as bpe
       snake_case_ =   emoji
       snake_case_ =   np.max([len(_lowerCamelCase      ) for w in self.vocab.keys()]      )
       snake_case_ =   re.compile(R'''(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'''      )
       snake_case_ =   re.compile(R'''[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'''      )
       snake_case_ =   re.compile(R'''[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'''      )
       snake_case_ =   re.compile(
           R'''([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'''      )
       snake_case_ =   re.compile(
           R'''(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'''      )
       snake_case_ =   re.compile(
           R'''((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'''      )
       snake_case_ =   '''─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'''
       snake_case_ =   '''▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'''
       snake_case_ =   str.maketrans({k: '''<BLOCK>''' for k in keisen + blocks}      )



 def __len__(			self      ):
       return len(self.ids_to_tokens      )



 def 		UpperCamelCase__  (			self	,   _UpperCAmelCase      ):
       snake_case_ =   self.content_repattera.sub('''<URL>'''	,   _lowerCamelCase      )
       snake_case_ =   self.content_repattera.sub('''<EMAIL>'''	,   _lowerCamelCase      )
       snake_case_ =   self.content_repattera.sub('''<TEL>'''	,   _lowerCamelCase      )
       snake_case_ =   self.content_repattera.sub('''<DATE>'''	,   _lowerCamelCase      )
       snake_case_ =   self.content_repattera.sub('''<DATE>'''	,   _lowerCamelCase      )
       snake_case_ =   self.content_repattera.sub('''<PRICE>'''	,   _lowerCamelCase      )
       snake_case_ =   content.translate(self.content_transa      )
       while "<BLOCK><BLOCK>" in content:
             snake_case_ =   content.replace('''<BLOCK><BLOCK>'''	,   '''<BLOCK>'''      )
       return content



 def 		UpperCamelCase__  (			self	,   _UpperCAmelCase	,   _UpperCAmelCase=False      ):
       snake_case_ =   text.replace(''' '''	,   '''<SP>'''      )
       snake_case_ =   text.replace('''　'''	,   '''<SP>'''      )
       snake_case_ =   text.replace('''\r\n'''	,   '''<BR>'''      )
       snake_case_ =   text.replace('''\n'''	,   '''<BR>'''      )
       snake_case_ =   text.replace('''\r'''	,   '''<BR>'''      )
       snake_case_ =   text.replace('''\t'''	,   '''<TAB>'''      )
       snake_case_ =   text.replace('''—'''	,   '''ー'''      )
       snake_case_ =   text.replace('''−'''	,   '''ー'''      )
       for k, v in self.emoji["emoji"].items():
             if k in text:
                   snake_case_ =   text.replace(_lowerCamelCase	,   _lowerCamelCase      )
       if clean:
             snake_case_ =   self.clean_text(_lowerCamelCase      )

       def check_simbol(_UpperCAmelCase      ):
             snake_case_ =   x.encode()
             if len(_lowerCamelCase      ) == 1 and len(_lowerCamelCase      ) == 2:
                   snake_case_ =   (int(e[0]      ) << 8) + int(e[1]      )
                   if (
                       (c >= 0xC2A1 and c <= 0xC2BF)
                       or (c >= 0xC780 and c <= 0xC783)
                       or (c >= 0xCAB9 and c <= 0xCBBF)
                       or (c >= 0xCC80 and c <= 0xCDA2)
                   ):
                         return True
             return False

       def checkuae(_UpperCAmelCase      ):
             snake_case_ =   x.encode()
             if len(_lowerCamelCase      ) == 1 and len(_lowerCamelCase      ) == 3:
                   snake_case_ =   (int(e[0]      ) << 16) + (int(e[1]      ) << 8) + int(e[2]      )
                   if c >= 0xE28080 and c <= 0xE2B07F:
                         return True
             return False

       snake_case_ =   0
       snake_case_ =   []
       while pos < len(_lowerCamelCase      ):
             snake_case_ =   min(len(_lowerCamelCase      )	,   pos + self.maxlen + 1      ) if text[pos] == '''<''' else pos + 3
             snake_case_ =   []  # (token_id, token, pos)
             for e in range(_lowerCamelCase	,   _lowerCamelCase	,   -1      ):
                   snake_case_ =   text[pos:e]
                   if wd in self.vocab:
                         if wd[0] == "<" and len(_lowerCamelCase      ) > 2:
                               snake_case_ =   [(self.vocab[wd], wd, e)]
                               break
                         else:
                               candidates.append((self.vocab[wd], wd, e)      )
             if len(_lowerCamelCase      ) > 0:
                   # the smallest token_id is adopted
                   snake_case_ ,					snake_case_ ,					snake_case_ =   sorted(_lowerCamelCase	,   key=lambda _UpperCAmelCase      : x[0]      )[0]
                   result.append(_lowerCamelCase      )
                   snake_case_ =   e
             else:
                   snake_case_ =   pos + 1
                   snake_case_ =   text[pos:end]
                   if check_simbol(_lowerCamelCase      ):
                         result.append('''<KIGOU>'''      )
                   elif checkuae(_lowerCamelCase      ):
                         result.append('''<U2000U2BFF>'''      )
                   else:
                         for i in wd.encode('''utf-8'''      ):
                               result.append('''<|byte%d|>''' % i      )
                   snake_case_ =   end
       return result





 def 		UpperCamelCase__  (			self	,   _UpperCAmelCase	,   _UpperCAmelCase="\n"      ):
       snake_case_ =   []
       snake_case_ =   []
       snake_case_ =   self.ids_to_tokens[index][0]
       if word[:6] == "<|byte" and word[-2:] == "|>":
             byte_tokens.append(int(word[6:-2]      )      )
       else:
             if len(_lowerCamelCase      ) > 0:
                   words.append(bytearray(_lowerCamelCase      ).decode('''utf-8'''	,   errors='''replace'''      )      )
                   snake_case_ =   []
             if word[:7] == "<|emoji" and word[-2:] == "|>":
                   words.append(self.emoji['''emoji_inv'''][word]      )
             elif word == "<SP>":
                   words.append(''' '''      )
             elif word == "<BR>":
                   words.append(_lowerCamelCase      )
             elif word == "<TAB>":
                   words.append('''\t'''      )
             elif word == "<BLOCK>":
                   words.append('''▀'''      )
             elif word == "<KIGOU>":
                   words.append('''ǀ'''      )
             elif word == "<U2000U2BFF>":
                   words.append('''‖'''      )
             else:
                   words.append(_lowerCamelCase      )
       if len(_lowerCamelCase      ) > 0:
             words.append(bytearray(_lowerCamelCase      ).decode('''utf-8'''	,   errors='''replace'''      )      )
       snake_case_ =   ''''''.join(_lowerCamelCase      )
       return text



import numpy as np

import datasets


UpperCAmelCase    =					"""
Compute the Mahalanobis Distance

Mahalonobis distance is the distance between a point and a distribution.
And not between two distinct points. It is effectively a multivariate equivalent of the Euclidean distance.
It was introduced by Prof. P. C. Mahalanobis in 1936
and has been used in various statistical applications ever since
[source: https://www.machinelearningplus.com/statistics/mahalanobis-distance/]
"""

UpperCAmelCase    =					"""\
@article{de2000mahalanobis,
  title={The mahalanobis distance},
  author={De Maesschalck, Roy and Jouan-Rimbaud, Delphine and Massart, D{\'e}sir{\'e} L},
  journal={Chemometrics and intelligent laboratory systems},
  volume={50},
  number={1},
  pages={1--18},
  year={2000},
  publisher={Elsevier}
}
"""

UpperCAmelCase    =					"""
Args:
    X: List of datapoints to be compared with the `reference_distribution`.
    reference_distribution: List of datapoints from the reference distribution we want to compare to.
Returns:
    mahalanobis: The Mahalonobis distance for each datapoint in `X`.
Examples:

    >>> mahalanobis_metric = datasets.load_metric(\"mahalanobis\")
    >>> results = mahalanobis_metric.compute(reference_distribution=[[0, 1], [1, 0]], X=[[0, 1]])
    >>> print(results)
    {'mahalanobis': array([0.5])}
"""







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION      ,		_KWARGS_DESCRIPTION )
class lowerCAmelCase_		(				datasets.Metric ):

 '''simple docstring'''

 def 		UpperCamelCase__  (			self      ):
       return datasets.MetricInfo(
           description=_DESCRIPTION	,   citation=_CITATION	,   inputs_description=_KWARGS_DESCRIPTION	,   features=datasets.Features(
               {
                   '''X''': datasets.Sequence(datasets.Value('''float'''	,   id='''sequence'''      )	,   id='''X'''      ),
               }      )	,   )





 def 		UpperCamelCase__  (			self	,   _UpperCAmelCase	,   _UpperCAmelCase      ):
       # convert to numpy arrays
       snake_case_ =   np.array(_UpperCAmelCase      )
       snake_case_ =   np.array(_UpperCAmelCase      )

       # Assert that arrays are 2D
       if len(X.shape      ) != 2:
             raise ValueError('''Expected `X` to be a 2D vector'''      )
       if len(reference_distribution.shape      ) != 2:
             raise ValueError('''Expected `reference_distribution` to be a 2D vector'''      )
       if reference_distribution.shape[0] < 2:
             raise ValueError(
                 '''Expected `reference_distribution` to be a 2D vector with more than one element in the first dimension'''      )

       # Get mahalanobis distance for each prediction
       snake_case_ =   X - np.mean(_UpperCAmelCase      )
       snake_case_ =   np.cov(reference_distribution.T      )
       try:
             snake_case_ =   np.linalg.inv(_UpperCAmelCase      )
       except np.linalg.LinAlgError:
             snake_case_ =   np.linalg.pinv(_UpperCAmelCase      )
       snake_case_ =   np.dot(_UpperCAmelCase	,   _UpperCAmelCase      )
       snake_case_ =   np.dot(_UpperCAmelCase	,   X_minus_mu.T      ).diagonal()

       return {"mahalanobis": mahal_dist}