



"""simple docstring"""







import dataclasses
import json
import warnings
from dataclasses import dataclass, field
from time import time
from typing import List

from ..utils import logging


A    =			logging.get_logger(__name__)

def   lowerCAmelCase__			(     lowerCamelCase__=None				,      lowerCamelCase__=None			) ->							Dict:
							return field(default_factory=lambda: default				,      metadata=lowerCamelCase__			)






@dataclass
class      UpperCAmelCase__		:
		lowerCAmelCase_       :	List[str] 							=     list_field(
		    default=[]   ,metadata={
		        """help""": (
		            """Model checkpoints to be provided to the AutoModel classes. Leave blank to benchmark the base version"""
		            """ of all available models"""
		        )
		    }   ,)

		lowerCAmelCase_       :	List[int] 							=     list_field(
		    default=[8]   ,metadata={"""help""": """List of batch sizes for which memory and time performance will be evaluated"""}     )

		lowerCAmelCase_       :	List[int] 							=     list_field(
		    default=[8, 32, 1_28, 5_12]   ,metadata={"""help""": """List of sequence lengths for which memory and time performance will be evaluated"""}   ,)

		lowerCAmelCase_       :	bool 							=     field(
		    default=UpperCamelCase   ,metadata={"""help""": """Whether to benchmark inference of model. Inference can be disabled via --no-inference."""}   ,)
		lowerCAmelCase_       :	bool 							=     field(
		    default=UpperCamelCase   ,metadata={"""help""": """Whether to run on available cuda devices. Cuda can be disabled via --no-cuda."""}   ,)
		lowerCAmelCase_       :	bool 							=     field(
		    default=UpperCamelCase   ,metadata={"""help""": """Whether to run on available tpu devices. TPU can be disabled via --no-tpu."""}     )
		lowerCAmelCase_       :	bool 							=     field(default=UpperCamelCase   ,metadata={"""help""": """Use FP16 to accelerate inference."""}     )
		lowerCAmelCase_       :	bool 							=     field(default=UpperCamelCase   ,metadata={"""help""": """Benchmark training of model"""}     )
		lowerCAmelCase_       :	bool 							=     field(default=UpperCamelCase   ,metadata={"""help""": """Verbose memory tracing"""}     )
		lowerCAmelCase_       :	bool 							=     field(
		    default=UpperCamelCase   ,metadata={"""help""": """Whether to perform speed measurements. Speed measurements can be disabled via --no-speed."""}   ,)
		lowerCAmelCase_       :	bool 							=     field(
		    default=UpperCamelCase   ,metadata={
		        """help""": """Whether to perform memory measurements. Memory measurements can be disabled via --no-memory"""
		    }   ,)
		lowerCAmelCase_       :	bool 							=     field(default=UpperCamelCase   ,metadata={"""help""": """Trace memory line by line"""}     )
		lowerCAmelCase_       :	bool 							=     field(default=UpperCamelCase   ,metadata={"""help""": """Save result to a CSV file"""}     )
		lowerCAmelCase_       :	bool 							=     field(default=UpperCamelCase   ,metadata={"""help""": """Save all print statements in a log file"""}     )
		lowerCAmelCase_       :	bool 							=     field(default=UpperCamelCase   ,metadata={"""help""": """Whether to print environment information"""}     )
		lowerCAmelCase_       :	bool 							=     field(
		    default=UpperCamelCase   ,metadata={
		        """help""": (
		            """Whether to use multiprocessing for memory and speed measurement. It is highly recommended to use"""
		            """ multiprocessing for accurate CPU and GPU memory measurements. This option should only be disabled"""
		            """ for debugging / testing and on TPU."""
		        )
		    }   ,)
		lowerCAmelCase_       :	str 							=     field(
		    default=f"""inference_time_{round(time()     )}.csv"""   ,metadata={"""help""": """CSV filename used if saving time results to csv."""}   ,)
		lowerCAmelCase_       :	str 							=     field(
		    default=f"""inference_memory_{round(time()     )}.csv"""   ,metadata={"""help""": """CSV filename used if saving memory results to csv."""}   ,)
		lowerCAmelCase_       :	str 							=     field(
		    default=f"""train_time_{round(time()     )}.csv"""   ,metadata={"""help""": """CSV filename used if saving time results to csv for training."""}   ,)
		lowerCAmelCase_       :	str 							=     field(
		    default=f"""train_memory_{round(time()     )}.csv"""   ,metadata={"""help""": """CSV filename used if saving memory results to csv for training."""}   ,)
		lowerCAmelCase_       :	str 							=     field(
		    default=f"""env_info_{round(time()     )}.csv"""   ,metadata={"""help""": """CSV filename used if saving environment information."""}   ,)
		lowerCAmelCase_       :	str 							=     field(
		    default=f"""log_{round(time()     )}.csv"""   ,metadata={"""help""": """Log filename used if print statements are saved in log."""}   ,)
		lowerCAmelCase_       :	int 							=     field(default=3   ,metadata={"""help""": """Times an experiment will be run."""}     )
		lowerCAmelCase_       :	bool 							=     field(
		    default=UpperCamelCase   ,metadata={
		        """help""": (
		            """Instead of loading the model as defined in `config.architectures` if exists, just load the pretrain"""
		            """ model weights."""
		        )
		    }   ,)



		def 					A_		( self  :    Dict				)			->						Optional[int]:






									'''simple docstring'''

									warnings.warn(
									    f"""The class {self.__class__} is deprecated. Hugging Face Benchmarking utils"""
									    ' are deprecated in general and it is advised to use external Benchmarking libraries '
									    ' to benchmark Transformer models.'      ,							snake_case      ,							)



		def 					A_		( self  :    str				)			->						List[Any]:






									'''simple docstring'''

									return json.dumps(dataclasses.asdict(self				)      ,							indent=2				)



		@property
		def 					A_		( self  :    Union[str, Any]				)			->						List[str]:






									'''simple docstring'''

									if len(self.models				) <= 0:
																raise ValueError(
																    'Please make sure you provide at least one model name / model identifier, *e.g.* `--models'
																    ' bert-base-cased` or `args.models = [\'bert-base-cased\'].'				)
									return self.models



		@property
		def 					A_		( self  :    Tuple				)			->						List[str]:






									'''simple docstring'''

									if not self.multi_process:
																return False
									elif self.is_tpu:
																logger.info('Multiprocessing is currently not possible on TPU.'				)
																return False
									else:
																return True








"""simple docstring"""







import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation


A    =			logging.get_logger(__name__)

A    =			{'vocab_file': 'vocab.txt', 'emoji_file': 'emoji.json'}

A    =			{
    'vocab_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt',
    },
    'emoji_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json',
    },
}

A    =			{
    'abeja/gpt-neox-japanese-2.7b': 2_0_4_8,
}

def   lowerCAmelCase__			(     lowerCamelCase__				,      lowerCamelCase__			) ->							str:
							with open(lowerCamelCase__				,      'r'				,      encoding='utf-8'			) as f:
														A         =					json.loads(f.read()			)

							A         =					collections.OrderedDict()
							A         =					collections.OrderedDict()
							A         =					collections.OrderedDict()
							with open(lowerCamelCase__				,      'r'				,      encoding='utf-8'			) as f:
														A         =					f.readlines()
							A         =					[[t.rstrip('\n'			)] if (t == ',' or ',' not in t) else t.rstrip('\n'			).split(','			) for t in token]
							for idx, b in enumerate(lowerCamelCase__			):
														A         =					b
														A         =					idx
														for wd in b:
																					A         =					idx

							return vocab, raw_vocab, ids_to_tokens, emoji






class      UpperCAmelCase__		(      UpperCamelCase     ):
		lowerCAmelCase_       :	Optional[int] 							=     VOCAB_FILES_NAMES
		lowerCAmelCase_       :	Any 							=     PRETRAINED_VOCAB_FILES_MAP
		lowerCAmelCase_       :	Optional[Any] 							=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		lowerCAmelCase_       :	str 							=     ["""input_ids""", """attention_mask"""]



		def __init__( self  :    Optional[Any]      ,							snake_case  :    Tuple      ,							snake_case  :    Optional[Any]      ,							snake_case  :    Optional[Any]="<|endoftext|>"      ,							snake_case  :    List[str]="<|endoftext|>"      ,							snake_case  :    Any="<|startoftext|>"      ,							snake_case  :    Any="<|endoftext|>"      ,							snake_case  :    Tuple=False      ,							**snake_case  :    List[str]      ,							)			->						Optional[Any]:






									'''simple docstring'''

									super().__init__(
									    unk_token=snake_case      ,							pad_token=snake_case      ,							bos_token=snake_case      ,							eos_token=snake_case      ,							do_clean_text=snake_case      ,							**snake_case      ,							)
									if not os.path.isfile(snake_case				):
																raise ValueError(
																    f"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
																    ' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'				)
									if not os.path.isfile(snake_case				):
																raise ValueError(
																    f"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
																    ' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'				)
									A         =					do_clean_text
									A      ,  A      ,  A      ,  A         =					load_vocab_and_emoji(snake_case      ,							snake_case				)
									A         =					SubWordJapaneseTokenizer(
									    vocab=self.vocab      ,							ids_to_tokens=self.ids_to_tokens      ,							emoji=self.emoji				)



		@property
		def 					A_		( self  :    Any				)			->						List[str]:






									'''simple docstring'''

									return len(self.raw_vocab				)



		def 					A_		( self  :    str				)			->						str:






									'''simple docstring'''

									return dict(self.raw_vocab      ,							**self.added_tokens_encoder				)



		def 					A_		( self  :    List[str]      ,							snake_case  :    Tuple				)			->						Optional[Any]:






									'''simple docstring'''

									return self.subword_tokenizer.tokenize(snake_case      ,							clean=self.do_clean_text				)



		def 					A_		( self  :    int      ,							snake_case  :    Optional[int]				)			->						Dict:






									'''simple docstring'''

									return self.vocab.get(snake_case      ,							self.vocab.get(self.unk_token				)				)



		def 					A_		( self  :    int      ,							snake_case  :    List[Any]				)			->						Tuple:






									'''simple docstring'''

									return self.subword_tokenizer.convert_id_to_token(snake_case				)



		def 					A_		( self  :    str      ,							snake_case  :    int				)			->						Optional[Any]:






									'''simple docstring'''

									A         =					''.join(snake_case				).strip()
									return out_string



		def 					A_		( self  :    Optional[int]      ,							snake_case  :    "Conversation"				)			->						List[int]:






									'''simple docstring'''

									A         =					[]
									for is_user, text in conversation.iter_texts():
																input_ids.extend(self.encode(snake_case      ,							add_special_tokens=snake_case				) + [self.eos_token_id]				)

									if len(snake_case				) > self.model_max_length:
																A         =					input_ids[-self.model_max_length :]
									return input_ids



		def 					A_		( self  :    int      ,							snake_case  :    str      ,							snake_case  :    Optional[str] = None				)			->						Tuple[str]:






									'''simple docstring'''

									A         =					0
									if os.path.isdir(snake_case				):
																A         =					os.path.join(
																    snake_case      ,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']				)
																A         =					os.path.join(
																    snake_case      ,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['emoji_file']				)
									else:
																A         =					(
																    (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['vocab_file']
																)
																A         =					(
																    (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['emoji_file']
																)
									with open(snake_case      ,							'w'      ,							encoding='utf-8'				) as writer:
																for token_index, token in self.ids_to_tokens.items():
																							if index != token_index:
																														logger.warning(
																														    f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
																														    ' Please check that the vocabulary is not corrupted!'				)
																														A         =					token_index
																							writer.write(','.join(snake_case				) + '\n'				)
																							index += 1
									with open(snake_case      ,							'w'      ,							encoding='utf-8'				) as writer:
																json.dump(self.emoji      ,							snake_case				)
									return vocab_file, emoji_file






class      UpperCAmelCase__		(      UpperCamelCase     ):



		def __init__( self  :    str      ,							snake_case  :    Dict      ,							snake_case  :    Optional[Any]      ,							snake_case  :    List[Any]				)			->						int:






									'''simple docstring'''

									A         =					vocab  # same as swe
									A         =					ids_to_tokens  # same as bpe
									A         =					emoji
									A         =					np.max([len(snake_case				) for w in self.vocab.keys()]				)
									A         =					re.compile(r'(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'				)
									A         =					re.compile(r'[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'				)
									A         =					re.compile(r'[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'				)
									A         =					re.compile(
									    r'([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'				)
									A         =					re.compile(
									    r'(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'				)
									A         =					re.compile(
									    r'((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'				)
									A         =					'─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'
									A         =					'▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'
									A         =					str.maketrans({k: '<BLOCK>' for k in keisen + blocks}				)



		def __len__( self  :    List[str]				)			->						List[str]:






									'''simple docstring'''

									return len(self.ids_to_tokens				)



		def 					A_		( self  :    Tuple      ,							snake_case  :    Any				)			->						Optional[int]:






									'''simple docstring'''

									A         =					self.content_repattera.sub('<URL>'      ,							snake_case				)
									A         =					self.content_repattera.sub('<EMAIL>'      ,							snake_case				)
									A         =					self.content_repattera.sub('<TEL>'      ,							snake_case				)
									A         =					self.content_repattera.sub('<DATE>'      ,							snake_case				)
									A         =					self.content_repattera.sub('<DATE>'      ,							snake_case				)
									A         =					self.content_repattera.sub('<PRICE>'      ,							snake_case				)
									A         =					content.translate(self.content_transa				)
									while "<BLOCK><BLOCK>" in content:
																A         =					content.replace('<BLOCK><BLOCK>'      ,							'<BLOCK>'				)
									return content



		def 					A_		( self  :    Any      ,							snake_case  :    int      ,							snake_case  :    Any=False				)			->						Any:






									'''simple docstring'''

									A         =					text.replace(' '      ,							'<SP>'				)
									A         =					text.replace('　'      ,							'<SP>'				)
									A         =					text.replace('\r\n'      ,							'<BR>'				)
									A         =					text.replace('\n'      ,							'<BR>'				)
									A         =					text.replace('\r'      ,							'<BR>'				)
									A         =					text.replace('\t'      ,							'<TAB>'				)
									A         =					text.replace('—'      ,							'ー'				)
									A         =					text.replace('−'      ,							'ー'				)
									for k, v in self.emoji["emoji"].items():
																if k in text:
																							A         =					text.replace(snake_case      ,							snake_case				)
									if clean:
																A         =					self.clean_text(snake_case				)

									def check_simbol(snake_case  :    Union[str, Any]				):
																A         =					x.encode()
																if len(snake_case				) == 1 and len(snake_case				) == 2:
																							A         =					(int(e[0]				) << 8) + int(e[1]				)
																							if (
																							    (c >= 0xc2a1 and c <= 0xc2bf)
																							    or (c >= 0xc780 and c <= 0xc783)
																							    or (c >= 0xcab9 and c <= 0xcbbf)
																							    or (c >= 0xcc80 and c <= 0xcda2)
																							):
																														return True
																return False

									def checkuae(snake_case  :    List[Any]				):
																A         =					x.encode()
																if len(snake_case				) == 1 and len(snake_case				) == 3:
																							A         =					(int(e[0]				) << 16) + (int(e[1]				) << 8) + int(e[2]				)
																							if c >= 0xe2_8080 and c <= 0xe2_b07f:
																														return True
																return False

									A         =					0
									A         =					[]
									while pos < len(snake_case				):
																A         =					min(len(snake_case				)      ,							pos + self.maxlen + 1				) if text[pos] == '<' else pos + 3
																A         =					[]  # (token_id, token, pos)
																for e in range(snake_case      ,							snake_case      ,							-1				):
																							A         =					text[pos:e]
																							if wd in self.vocab:
																														if wd[0] == "<" and len(snake_case				) > 2:
																																					A         =					[(self.vocab[wd], wd, e)]
																																					break
																														else:
																																					candidates.append((self.vocab[wd], wd, e)				)
																if len(snake_case				) > 0:
																							# the smallest token_id is adopted
																							A      ,  A      ,  A         =					sorted(snake_case      ,							key=lambda snake_case				: x[0]				)[0]
																							result.append(snake_case				)
																							A         =					e
																else:
																							A         =					pos + 1
																							A         =					text[pos:end]
																							if check_simbol(snake_case				):
																														result.append('<KIGOU>'				)
																							elif checkuae(snake_case				):
																														result.append('<U2000U2BFF>'				)
																							else:
																														for i in wd.encode('utf-8'				):
																																					result.append('<|byte%d|>' % i				)
																							A         =					end
									return result



		def 					A_		( self  :    List[str]      ,							snake_case  :    Optional[Any]      ,							snake_case  :    Optional[Any]="\n"				)			->						List[Any]:






									'''simple docstring'''

									A         =					[]
									A         =					[]
									A         =					self.ids_to_tokens[index][0]
									if word[:6] == "<|byte" and word[-2:] == "|>":
																byte_tokens.append(int(word[6:-2]				)				)
									else:
																if len(snake_case				) > 0:
																							words.append(bytearray(snake_case				).decode('utf-8'      ,							errors='replace'				)				)
																							A         =					[]
																if word[:7] == "<|emoji" and word[-2:] == "|>":
																							words.append(self.emoji['emoji_inv'][word]				)
																elif word == "<SP>":
																							words.append(' '				)
																elif word == "<BR>":
																							words.append(snake_case				)
																elif word == "<TAB>":
																							words.append('\t'				)
																elif word == "<BLOCK>":
																							words.append('▀'				)
																elif word == "<KIGOU>":
																							words.append('ǀ'				)
																elif word == "<U2000U2BFF>":
																							words.append('‖'				)
																else:
																							words.append(snake_case				)
									if len(snake_case				) > 0:
																words.append(bytearray(snake_case				).decode('utf-8'      ,							errors='replace'				)				)
									A         =					''.join(snake_case				)
									return text



