


'''simple docstring'''
# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig







def       a_				(	UpperCamelCase_						,  UpperCamelCase_						,  UpperCamelCase_=0 ):
							# Format the message.
							if name is None:
														A_			    =		None
							else:
														A_			    =		"." * max(0						,  spaces - 2 ) + "# {:" + str(5_0 - spaces ) + "s}"
														A_			    =		fmt.format(UpperCamelCase_ )

							# Print and recurse (if needed).
							if isinstance(UpperCamelCase_						,  UpperCamelCase_ ):
														if msg is not None:
																					print(UpperCamelCase_ )
														for k in val.keys():
																					recursive_print(UpperCamelCase_						,  val[k]						,  spaces + 2 )
							elif isinstance(UpperCamelCase_						,  torch.Tensor ):
														print(UpperCamelCase_						,  ":"						,  val.size() )
							else:
														print(UpperCamelCase_						,  ":"						,  UpperCamelCase_ )







def       a_				(	UpperCamelCase_						,  UpperCamelCase_						,  UpperCamelCase_						,  UpperCamelCase_						,  UpperCamelCase_ ):
							# Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
							# for compatibility with later versions of NVIDIA Megatron-LM.
							# The inverse operation is performed inside Megatron-LM to read checkpoints:
							# https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
							# If param is the weight tensor of the self-attention block, the returned tensor
							# will have to be transposed one more time to be read by HuggingFace GPT2.
							A_			    =		param.size()
							if checkpoint_version == 1.0:
														# version 1.0 stores [num_heads * hidden_size * num_splits, :]
														A_			    =		(num_heads, hidden_size, num_splits) + input_shape[1:]
														A_			    =		param.view(*UpperCamelCase_ )
														A_			    =		param.transpose(0						,  2 )
														A_			    =		param.transpose(1						,  2 ).contiguous()
							elif checkpoint_version >= 2.0:
														# other versions store [num_heads * num_splits * hidden_size, :]
														A_			    =		(num_heads, num_splits, hidden_size) + input_shape[1:]
														A_			    =		param.view(*UpperCamelCase_ )
														A_			    =		param.transpose(0						,  1 ).contiguous()
							A_			    =		param.view(*UpperCamelCase_ )
							return param







def       a_				(	UpperCamelCase_						,  UpperCamelCase_						,  UpperCamelCase_ ):
							# The converted output model.
							A_			    =		{}

							# old versions did not store training args
							A_			    =		input_state_dict.get("args"						,  UpperCamelCase_ )
							if ds_args is not None:
														# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
														# from pprint import pprint
														# pprint(vars(ds_args))

														A_			    =		ds_args.padded_vocab_size
														A_			    =		ds_args.max_position_embeddings
														A_			    =		ds_args.hidden_size
														A_			    =		ds_args.num_layers
														A_			    =		ds_args.num_attention_heads
														A_			    =		ds_args.ffn_hidden_size
														# pprint(config)

							# The number of heads.
							A_			    =		config.n_head
							# The hidden_size per head.
							A_			    =		config.n_embd // config.n_head
							# Megatron-LM checkpoint version
							if "checkpoint_version" in input_state_dict.keys():
														A_			    =		input_state_dict["checkpoint_version"]
							else:
														A_			    =		0.0

							# The model.
							A_			    =		input_state_dict["model"]
							# The language model.
							A_			    =		model["language_model"]
							# The embeddings.
							A_			    =		lm["embedding"]

							# The word embeddings.
							A_			    =		embeddings["word_embeddings"]["weight"]
							# Truncate the embedding table to vocab_size rows.
							A_			    =		word_embeddings[: config.vocab_size, :]
							A_			    =		word_embeddings

							# The position embeddings.
							A_			    =		embeddings["position_embeddings"]["weight"]
							# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
							A_			    =		pos_embeddings.size(0 )
							if n_positions != config.n_positions:
														raise ValueError(
														    f"pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don't match" )
							# Store the position embeddings.
							A_			    =		pos_embeddings

							# The transformer.
							A_			    =		lm["transformer"] if "transformer" in lm.keys() else lm["encoder"]

							# The regex to extract layer names.
							A_			    =		re.compile(R"layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)" )

							# The simple map of names for "automated" rules.
							A_			    =		{
							    "attention.dense": ".attn.c_proj.",
							    "self_attention.dense": ".attn.c_proj.",
							    "mlp.dense_h_to_4h": ".mlp.c_fc.",
							    "mlp.dense_4h_to_h": ".mlp.c_proj.",
							}

							# Extract the layers.
							for key, val in transformer.items():
														# Match the name.
														A_			    =		layer_re.match(UpperCamelCase_ )

														# Stop if that's not a layer
														if m is None:
																					break

														# The index of the layer.
														A_			    =		int(m.group(1 ) )
														# The name of the operation.
														A_			    =		m.group(2 )
														# Is it a weight or a bias?
														A_			    =		m.group(3 )

														# The name of the layer.
														A_			    =		f"transformer.h.{layer_idx}"

														# For layernorm(s), simply store the layer norm.
														if op_name.endswith("layernorm" ):
																					A_			    =		"ln_1" if op_name.startswith("input" ) else "ln_2"
																					A_			    =		val

														# Transpose the QKV matrix.
														elif (
														    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
														) and weight_or_bias == "weight":
																					# Insert a tensor of 1x1xDxD bias.
																					A_			    =		torch.tril(torch.ones((n_positions, n_positions)						,  dtype=torch.floataa ) ).view(
																					    1						,  1						,  UpperCamelCase_						,  UpperCamelCase_ )
																					A_			    =		causal_mask

																					# Insert a "dummy" tensor for masked_bias.
																					A_			    =		torch.tensor(-1e4						,  dtype=torch.floataa )
																					A_			    =		masked_bias

																					A_			    =		fix_query_key_value_ordering(UpperCamelCase_						,  UpperCamelCase_						,  3						,  UpperCamelCase_						,  UpperCamelCase_ )
																					# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
																					A_			    =		out_val.transpose(0						,  1 ).contiguous()
																					# Store.
																					A_			    =		out_val

														# Transpose the bias.
														elif (
														    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
														) and weight_or_bias == "bias":
																					A_			    =		fix_query_key_value_ordering(UpperCamelCase_						,  UpperCamelCase_						,  3						,  UpperCamelCase_						,  UpperCamelCase_ )
																					# Store. No change of shape.
																					A_			    =		out_val

														# Transpose the weights.
														elif weight_or_bias == "weight":
																					A_			    =		megatron_to_transformers[op_name]
																					A_			    =		val.transpose(0						,  1 )

														# Copy the bias.
														elif weight_or_bias == "bias":
																					A_			    =		megatron_to_transformers[op_name]
																					A_			    =		val

    # DEBUG.
							assert config.n_layer == layer_idx + 1

							# The final layernorm.
							A_			    =		transformer["final_layernorm.weight"]
							A_			    =		transformer["final_layernorm.bias"]

							# For LM head, transformers' wants the matrix to weight embeddings.
							A_			    =		word_embeddings

							# It should be done!
							return output_state_dict







def       a_				(	):
							# Create the argument parser.
							A_			    =		argparse.ArgumentParser()
							parser.add_argument("--print-checkpoint-structure"						,  action="store_true" )
							parser.add_argument(
							    "path_to_checkpoint"						,  type=UpperCamelCase_						,  help="Path to the checkpoint file (.zip archive or direct .pt file)"						,  )
							parser.add_argument(
							    "--config_file"						,  default=""						,  type=UpperCamelCase_						,  help="An optional config json file describing the pre-trained model."						,  )
							A_			    =		parser.parse_args()

							# Extract the basename.
							A_			    =		os.path.dirname(args.path_to_checkpoint )

							# Load the model.
							# the .zip is very optional, let's keep it for backward compatibility
							print(f"Extracting PyTorch state dictionary from {args.path_to_checkpoint}" )
							if args.path_to_checkpoint.endswith(".zip" ):
														with zipfile.ZipFile(args.path_to_checkpoint						,  "r" ) as checkpoint:
																					with checkpoint.open("release/mp_rank_00/model_optim_rng.pt" ) as pytorch_dict:
																												A_			    =		torch.load(UpperCamelCase_						,  map_location="cpu" )
							else:
														A_			    =		torch.load(args.path_to_checkpoint						,  map_location="cpu" )

							A_			    =		input_state_dict.get("args"						,  UpperCamelCase_ )

							# Read the config, or default to the model released by NVIDIA.
							if args.config_file == "":
														if ds_args is not None:
																					if ds_args.bias_gelu_fusion:
																												A_			    =		"gelu_fast"
																					elif ds_args.openai_gelu:
																												A_			    =		"gelu_new"
																					else:
																												A_			    =		"gelu"
														else:
																					# in the very early days this used to be "gelu_new"
																					A_			    =		"gelu_new"

														# Spell out all parameters in case the defaults change.
														A_			    =		GPTaConfig(
														    vocab_size=5_0_2_5_7						,  n_positions=1_0_2_4						,  n_embd=1_0_2_4						,  n_layer=2_4						,  n_head=1_6						,  n_inner=4_0_9_6						,  activation_function=UpperCamelCase_						,  resid_pdrop=0.1						,  embd_pdrop=0.1						,  attn_pdrop=0.1						,  layer_norm_epsilon=1e-5						,  initializer_range=0.02						,  summary_type="cls_index"						,  summary_use_proj=UpperCamelCase_						,  summary_activation=UpperCamelCase_						,  summary_proj_to_labels=UpperCamelCase_						,  summary_first_dropout=0.1						,  scale_attn_weights=UpperCamelCase_						,  use_cache=UpperCamelCase_						,  bos_token_id=5_0_2_5_6						,  eos_token_id=5_0_2_5_6						,  )
							else:
														A_			    =		GPTaConfig.from_json_file(args.config_file )

							A_			    =		["GPT2LMHeadModel"]

							# Convert.
							print("Converting" )
							A_			    =		convert_megatron_checkpoint(UpperCamelCase_						,  UpperCamelCase_						,  UpperCamelCase_ )

							# Print the structure of converted state dict.
							if args.print_checkpoint_structure:
														recursive_print(UpperCamelCase_						,  UpperCamelCase_ )

							# Add tokenizer class info to config
							# see https://github.com/huggingface/transformers/issues/13906)
							if ds_args is not None:
														A_			    =		ds_args.tokenizer_type
														if tokenizer_type == "GPT2BPETokenizer":
																					A_			    =		"gpt2"
														elif tokenizer_type == "PretrainedFromHF":
																					A_			    =		ds_args.tokenizer_name_or_path
														else:
																					raise ValueError(f"Unrecognized tokenizer_type {tokenizer_type}" )
							else:
														A_			    =		"gpt2"

							A_			    =		AutoTokenizer.from_pretrained(UpperCamelCase_ )
							A_			    =		type(UpperCamelCase_ ).__name__
							A_			    =		tokenizer_class

							# Store the config to file.
							print("Saving config" )
							config.save_pretrained(UpperCamelCase_ )

							# Save tokenizer based on args
							print(f"Adding {tokenizer_class} tokenizer files" )
							tokenizer.save_pretrained(UpperCamelCase_ )

							# Store the state_dict to file.
							A_			    =		os.path.join(UpperCamelCase_						,  "pytorch_model.bin" )
							print(f"Saving checkpoint to \"{output_checkpoint_file}\"" )
							torch.save(UpperCamelCase_						,  UpperCamelCase_ )


####################################################################################################

if __name__ == "__main__":
							main()

####################################################################################################









'''simple docstring'''
import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation


__SCREAMING_SNAKE_CASE					:				List[str]									=		logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE					:				Optional[int]									=		{'''vocab_file''': '''vocab.txt''', '''emoji_file''': '''emoji.json'''}

__SCREAMING_SNAKE_CASE					:				str									=		{
    '''vocab_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt''',
    },
    '''emoji_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json''',
    },
}

__SCREAMING_SNAKE_CASE					:				List[Any]									=		{
    '''abeja/gpt-neox-japanese-2.7b''': 2048,
}







def       a_				(	UpperCamelCase_						,  UpperCamelCase_ ):
							with open(UpperCamelCase_						,  "r"						,  encoding="utf-8" ) as f:
														A_			    =		json.loads(f.read() )

							A_			    =		collections.OrderedDict()
							A_			    =		collections.OrderedDict()
							A_			    =		collections.OrderedDict()
							with open(UpperCamelCase_						,  "r"						,  encoding="utf-8" ) as f:
														A_			    =		f.readlines()
							A_			    =		[[t.rstrip("\n" )] if (t == "," or "," not in t) else t.rstrip("\n" ).split("," ) for t in token]
							for idx, b in enumerate(UpperCamelCase_ ):
														A_			    =		b
														A_			    =		idx
														for wd in b:
																					A_			    =		idx

							return vocab, raw_vocab, ids_to_tokens, emoji






class 					__lowerCAmelCase   (   lowercase						):

						"""simple docstring"""






						_UpperCAmelCase		:       str   						=VOCAB_FILES_NAMES
						_UpperCAmelCase		:       str   						=PRETRAINED_VOCAB_FILES_MAP
						_UpperCAmelCase		:       Dict   						=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_UpperCAmelCase		:       List[str]   						=["input_ids", "attention_mask"]






						def __init__(  self							:      str ,  lowerCAmelCase							:      List[str] ,  lowerCAmelCase							:      Optional[int] ,  lowerCAmelCase							:      List[Any]="<|endoftext|>" ,  lowerCAmelCase							:      Optional[int]="<|endoftext|>" ,  lowerCAmelCase							:      Tuple="<|startoftext|>" ,  lowerCAmelCase							:      Union[str, Any]="<|endoftext|>" ,  lowerCAmelCase							:      List[Any]=False ,  **lowerCAmelCase							:      Optional[Any] ,  ):
													super().__init__(
													    unk_token=lowerCAmelCase ,  pad_token=lowerCAmelCase ,  bos_token=lowerCAmelCase ,  eos_token=lowerCAmelCase ,  do_clean_text=lowerCAmelCase ,  **lowerCAmelCase ,  )
													if not os.path.isfile(lowerCAmelCase	):
																				raise ValueError(
																				    F"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"
																				    " model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"	)
													if not os.path.isfile(lowerCAmelCase	):
																				raise ValueError(
																				    F"Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"
																				    " pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"	)
													A_			    =		do_clean_text
													A_    ,					A_    ,					A_    ,					A_			    =		load_vocab_and_emoji(lowerCAmelCase ,  lowerCAmelCase	)
													A_			    =		SubWordJapaneseTokenizer(
													    vocab=self.vocab ,  ids_to_tokens=self.ids_to_tokens ,  emoji=self.emoji	)






						@property
						def 				_UpperCAmelCase       (  self							:      str	):
													# self.vocab contains support for character fluctuation unique to Japanese, and has a large number of vocab
													return len(self.raw_vocab	)






						def 				_UpperCAmelCase       (  self							:      Optional[int]	):
													return dict(self.raw_vocab ,  **self.added_tokens_encoder	)






						def 				_UpperCAmelCase       (  self							:      Optional[int] ,  lowerCAmelCase							:      Tuple	):
													return self.subword_tokenizer.tokenize(lowerCAmelCase ,  clean=self.do_clean_text	)






						def 				_UpperCAmelCase       (  self							:      str ,  lowerCAmelCase							:      int	):
													return self.vocab.get(lowerCAmelCase ,  self.vocab.get(self.unk_token	)	)






						def 				_UpperCAmelCase       (  self							:      List[Any] ,  lowerCAmelCase							:      List[str]	):
													return self.subword_tokenizer.convert_id_to_token(lowerCAmelCase	)






						def 				_UpperCAmelCase       (  self							:      Optional[Any] ,  lowerCAmelCase							:      Optional[Any]	):
													A_			    =		"".join(lowerCAmelCase	).strip()
													return out_string






						def 				_UpperCAmelCase       (  self							:      Any ,  lowerCAmelCase							:      "Conversation"	):
													A_			    =		[]
													for is_user, text in conversation.iter_texts():
																				input_ids.extend(self.encode(lowerCAmelCase ,  add_special_tokens=lowerCAmelCase	) + [self.eos_token_id]	)

													if len(lowerCAmelCase	) > self.model_max_length:
																				A_			    =		input_ids[-self.model_max_length :]
													return input_ids






						def 				_UpperCAmelCase       (  self							:      Dict ,  lowerCAmelCase							:      str ,  lowerCAmelCase							:      Optional[str] = None	):
													A_			    =		0
													if os.path.isdir(lowerCAmelCase	):
																				A_			    =		os.path.join(
																				    lowerCAmelCase ,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]	)
																				A_			    =		os.path.join(
																				    lowerCAmelCase ,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["emoji_file"]	)
													else:
																				A_			    =		(
																				    (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["vocab_file"]
																				)
																				A_			    =		(
																				    (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["emoji_file"]
																				)
													with open(lowerCAmelCase ,  "w" ,  encoding="utf-8"	) as writer:
																				for token_index, token in self.ids_to_tokens.items():
																											if index != token_index:
																																		logger.warning(
																																		    F"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
																																		    " Please check that the vocabulary is not corrupted!"	)
																																		A_			    =		token_index
																											writer.write(",".join(lowerCAmelCase	) + "\n"	)
																											index += 1
													with open(lowerCAmelCase ,  "w" ,  encoding="utf-8"	) as writer:
																				json.dump(self.emoji ,  lowerCAmelCase	)
													return vocab_file, emoji_file






class 					__lowerCAmelCase   (   lowercase						):

						"""simple docstring"""






						def __init__(  self							:      List[Any] ,  lowerCAmelCase							:      str ,  lowerCAmelCase							:      Any ,  lowerCAmelCase							:      List[Any]	):
													A_			    =		vocab  # same as swe
													A_			    =		ids_to_tokens  # same as bpe
													A_			    =		emoji
													A_			    =		np.max([len(lowerCAmelCase	) for w in self.vocab.keys()]	)
													A_			    =		re.compile(r"(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)"	)
													A_			    =		re.compile(r"[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*"	)
													A_			    =		re.compile(r"[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}"	)
													A_			    =		re.compile(
													    r"([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"	)
													A_			    =		re.compile(
													    r"(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"	)
													A_			    =		re.compile(
													    r"((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*"	)
													A_			    =		"─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"
													A_			    =		"▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"
													A_			    =		str.maketrans({k: "<BLOCK>" for k in keisen + blocks}	)






						def __len__(  self							:      Union[str, Any]	):
													return len(self.ids_to_tokens	)






						def 				_UpperCAmelCase       (  self							:      List[str] ,  lowerCAmelCase							:      int	):
													A_			    =		self.content_repattera.sub("<URL>" ,  lowerCAmelCase	)
													A_			    =		self.content_repattera.sub("<EMAIL>" ,  lowerCAmelCase	)
													A_			    =		self.content_repattera.sub("<TEL>" ,  lowerCAmelCase	)
													A_			    =		self.content_repattera.sub("<DATE>" ,  lowerCAmelCase	)
													A_			    =		self.content_repattera.sub("<DATE>" ,  lowerCAmelCase	)
													A_			    =		self.content_repattera.sub("<PRICE>" ,  lowerCAmelCase	)
													A_			    =		content.translate(self.content_transa	)
													while "<BLOCK><BLOCK>" in content:
																				A_			    =		content.replace("<BLOCK><BLOCK>" ,  "<BLOCK>"	)
													return content






						def 				_UpperCAmelCase       (  self							:      Any ,  lowerCAmelCase							:      Optional[Any] ,  lowerCAmelCase							:      Tuple=False	):
													A_			    =		text.replace(" " ,  "<SP>"	)
													A_			    =		text.replace("　" ,  "<SP>"	)
													A_			    =		text.replace("\r\n" ,  "<BR>"	)
													A_			    =		text.replace("\n" ,  "<BR>"	)
													A_			    =		text.replace("\r" ,  "<BR>"	)
													A_			    =		text.replace("\t" ,  "<TAB>"	)
													A_			    =		text.replace("—" ,  "ー"	)
													A_			    =		text.replace("−" ,  "ー"	)
													for k, v in self.emoji["emoji"].items():
																				if k in text:
																											A_			    =		text.replace(lowerCAmelCase ,  lowerCAmelCase	)
													if clean:
																				A_			    =		self.clean_text(lowerCAmelCase	)

													def check_simbol(lowerCAmelCase							:      Tuple	):
																				A_			    =		x.encode()
																				if len(lowerCAmelCase	) == 1 and len(lowerCAmelCase	) == 2:
																											A_			    =		(int(e[0]	) << 8) + int(e[1]	)
																											if (
																											    (c >= 0xC2A1 and c <= 0xC2BF)
																											    or (c >= 0xC780 and c <= 0xC783)
																											    or (c >= 0xCAB9 and c <= 0xCBBF)
																											    or (c >= 0xCC80 and c <= 0xCDA2)
																											):
																																		return True
																				return False

													def checkuae(lowerCAmelCase							:      Tuple	):
																				A_			    =		x.encode()
																				if len(lowerCAmelCase	) == 1 and len(lowerCAmelCase	) == 3:
																											A_			    =		(int(e[0]	) << 16) + (int(e[1]	) << 8) + int(e[2]	)
																											if c >= 0xE2_8080 and c <= 0xE2_B07F:
																																		return True
																				return False

													A_			    =		0
													A_			    =		[]
													while pos < len(lowerCAmelCase	):
																				A_			    =		min(len(lowerCAmelCase	) ,  pos + self.maxlen + 1	) if text[pos] == "<" else pos + 3
																				A_			    =		[]  # (token_id, token, pos)
																				for e in range(lowerCAmelCase ,  lowerCAmelCase ,  -1	):
																											A_			    =		text[pos:e]
																											if wd in self.vocab:
																																		if wd[0] == "<" and len(lowerCAmelCase	) > 2:
																																									A_			    =		[(self.vocab[wd], wd, e)]
																																									break
																																		else:
																																									candidates.append((self.vocab[wd], wd, e)	)
																				if len(lowerCAmelCase	) > 0:
																											# the smallest token_id is adopted
																											A_    ,					A_    ,					A_			    =		sorted(lowerCAmelCase ,  key=lambda lowerCAmelCase	: x[0]	)[0]
																											result.append(lowerCAmelCase	)
																											A_			    =		e
																				else:
																											A_			    =		pos + 1
																											A_			    =		text[pos:end]
																											if check_simbol(lowerCAmelCase	):
																																		result.append("<KIGOU>"	)
																											elif checkuae(lowerCAmelCase	):
																																		result.append("<U2000U2BFF>"	)
																											else:
																																		for i in wd.encode("utf-8"	):
																																									result.append("<|byte%d|>" % i	)
																											A_			    =		end
													return result






						def 				_UpperCAmelCase       (  self							:      Dict ,  lowerCAmelCase							:      Any ,  lowerCAmelCase							:      Union[str, Any]="\n"	):
													A_			    =		[]
													A_			    =		[]
													A_			    =		self.ids_to_tokens[index][0]
													if word[:6] == "<|byte" and word[-2:] == "|>":
																				byte_tokens.append(int(word[6:-2]	)	)
													else:
																				if len(lowerCAmelCase	) > 0:
																											words.append(bytearray(lowerCAmelCase	).decode("utf-8" ,  errors="replace"	)	)
																											A_			    =		[]
																				if word[:7] == "<|emoji" and word[-2:] == "|>":
																											words.append(self.emoji["emoji_inv"][word]	)
																				elif word == "<SP>":
																											words.append(" "	)
																				elif word == "<BR>":
																											words.append(lowerCAmelCase	)
																				elif word == "<TAB>":
																											words.append("\t"	)
																				elif word == "<BLOCK>":
																											words.append("▀"	)
																				elif word == "<KIGOU>":
																											words.append("ǀ"	)
																				elif word == "<U2000U2BFF>":
																											words.append("‖"	)
																				else:
																											words.append(lowerCAmelCase	)
													if len(lowerCAmelCase	) > 0:
																				words.append(bytearray(lowerCAmelCase	).decode("utf-8" ,  errors="replace"	)	)
													A_			    =		"".join(lowerCAmelCase	)
													return text





