






import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(42)

A__      :     Optional[Any]       =					'bert-base-cased'
A__      :     int       =					'fp16'
A__      :     Optional[int]       =					'bf16'
A__      :     List[str]       =					[FPaa, BFaa]
@require_fsdp
@require_cuda
class  _UpperCAmelCase   (			A__      ):

				"""simple docstring"""




				def 							lowercase__  (	self		:       int    ):

								'''simple docstring'''
								super().setUp()

								lowercase__  					=     dict(
								    ACCELERATE_USE_FSDP='''true''',    MASTER_ADDR='''localhost''',    MASTER_PORT='''10999''',    RANK='''0''',    LOCAL_RANK='''0''',    WORLD_SIZE='''1''',    )




				def 							lowercase__  (	self		:       Any    ):

								'''simple docstring'''
								from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

								for i, strategy in enumerate(lowerCamelCase    ):
												lowercase__  					=     self.dist_env.copy()
												lowercase__  					=     F"""{i + 1}"""
												lowercase__  					=     strategy
												with mockenv_context(**lowerCamelCase    ):
																lowercase__  					=     FullyShardedDataParallelPlugin()
																self.assertEqual(fsdp_plugin.sharding_strategy,    ShardingStrategy(i + 1    )    )




				def 							lowercase__  (	self		:       List[str]    ):

								'''simple docstring'''
								from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

								for i, prefetch_policy in enumerate(lowerCamelCase    ):
												lowercase__  					=     self.dist_env.copy()
												lowercase__  					=     prefetch_policy
												with mockenv_context(**lowerCamelCase    ):
																lowercase__  					=     FullyShardedDataParallelPlugin()
																if prefetch_policy == "NO_PREFETCH":
																				self.assertIsNone(fsdp_plugin.backward_prefetch    )
																else:
																				self.assertEqual(fsdp_plugin.backward_prefetch,    BackwardPrefetch(i + 1    )    )




				def 							lowercase__  (	self		:       str    ):

								'''simple docstring'''
								from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

								for i, state_dict_type in enumerate(lowerCamelCase    ):
												lowercase__  					=     self.dist_env.copy()
												lowercase__  					=     state_dict_type
												with mockenv_context(**lowerCamelCase    ):
																lowercase__  					=     FullyShardedDataParallelPlugin()
																self.assertEqual(fsdp_plugin.state_dict_type,    StateDictType(i + 1    )    )
																if state_dict_type == "FULL_STATE_DICT":
																				self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu    )
																				self.assertTrue(fsdp_plugin.state_dict_config.ranka_only    )




				def 							lowercase__  (	self		:       str    ):

								'''simple docstring'''
								lowercase__  					=     AutoModel.from_pretrained(lowerCamelCase    )
								for policy in FSDP_AUTO_WRAP_POLICY:
												lowercase__  					=     self.dist_env.copy()
												lowercase__  					=     policy
												if policy == "TRANSFORMER_BASED_WRAP":
																lowercase__  					=     '''BertLayer'''
												elif policy == "SIZE_BASED_WRAP":
																lowercase__  					=     '''2000'''
												with mockenv_context(**lowerCamelCase    ):
																lowercase__  					=     FullyShardedDataParallelPlugin()
																fsdp_plugin.set_auto_wrap_policy(lowerCamelCase    )
																if policy == "NO_WRAP":
																				self.assertIsNone(fsdp_plugin.auto_wrap_policy    )
																else:
																				self.assertIsNotNone(fsdp_plugin.auto_wrap_policy    )

								lowercase__  					=     self.dist_env.copy()
								lowercase__  					=     '''TRANSFORMER_BASED_WRAP'''
								lowercase__  					=     '''T5Layer'''
								with mockenv_context(**lowerCamelCase    ):
												lowercase__  					=     FullyShardedDataParallelPlugin()
												with self.assertRaises(lowerCamelCase    ) as cm:
																fsdp_plugin.set_auto_wrap_policy(lowerCamelCase    )
												self.assertTrue('''Could not find the transformer layer class to wrap in the model.''' in str(cm.exception    )    )

								lowercase__  					=     self.dist_env.copy()
								lowercase__  					=     '''SIZE_BASED_WRAP'''
								lowercase__  					=     '''0'''
								with mockenv_context(**lowerCamelCase    ):
												lowercase__  					=     FullyShardedDataParallelPlugin()
												fsdp_plugin.set_auto_wrap_policy(lowerCamelCase    )
												self.assertIsNone(fsdp_plugin.auto_wrap_policy    )




				def 							lowercase__  (	self		:       List[str]    ):

								'''simple docstring'''
								from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
								from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

								for mp_dtype in dtypes:
												lowercase__  					=     self.dist_env.copy()
												lowercase__  					=     mp_dtype
												with mockenv_context(**lowerCamelCase    ):
																lowercase__  					=     Accelerator()
																if mp_dtype == "fp16":
																				lowercase__  					=     torch.floataa
																elif mp_dtype == "bf16":
																				lowercase__  					=     torch.bfloataa
																lowercase__  					=     MixedPrecision(param_dtype=lowerCamelCase,    reduce_dtype=lowerCamelCase,    buffer_dtype=lowerCamelCase    )
																self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy,    lowerCamelCase    )
																if mp_dtype == FPaa:
																				self.assertTrue(isinstance(accelerator.scaler,    lowerCamelCase    )    )
																elif mp_dtype == BFaa:
																				self.assertIsNone(accelerator.scaler    )
																AcceleratorState._reset_state(lowerCamelCase    )




				def 							lowercase__  (	self		:       Optional[int]    ):

								'''simple docstring'''
								from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

								for flag in [True, False]:
												lowercase__  					=     self.dist_env.copy()
												lowercase__  					=     str(lowerCamelCase    ).lower()
												with mockenv_context(**lowerCamelCase    ):
																lowercase__  					=     FullyShardedDataParallelPlugin()
																self.assertEqual(fsdp_plugin.cpu_offload,    CPUOffload(offload_params=lowerCamelCase    )    )



@require_fsdp
@require_multi_gpu
@slow
class  _UpperCAmelCase   (			A__      ):

				"""simple docstring"""




				def 							lowercase__  (	self		:       Optional[int]    ):

								'''simple docstring'''
								super().setUp()
								lowercase__  					=     0.82
								lowercase__  					=     [
								    '''fsdp_shard_grad_op_transformer_based_wrap''',
								    '''fsdp_full_shard_transformer_based_wrap''',
								]
								lowercase__  					=     {
								    '''multi_gpu_fp16''': 3_200,
								    '''fsdp_shard_grad_op_transformer_based_wrap_fp16''': 2_000,
								    '''fsdp_full_shard_transformer_based_wrap_fp16''': 1_900,
								    # Disabling below test as it overwhelms the RAM memory usage
								    # on CI self-hosted runner leading to tests getting killed.
								    # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
								}
								lowercase__  					=     160
								lowercase__  					=     160

								lowercase__  					=     inspect.getfile(accelerate.test_utils    )
								lowercase__  					=     os.path.sep.join(mod_file.split(os.path.sep    )[:-1] + ['''scripts''', '''external_deps''']    )




				def 							lowercase__  (	self		:       str    ):

								'''simple docstring'''
								lowercase__  					=     os.path.join(self.test_scripts_folder,    '''test_performance.py'''    )
								lowercase__  					=     ['''accelerate''', '''launch''', '''--num_processes=2''', '''--num_machines=1''', '''--machine_rank=0''', '''--use_fsdp''']
								for config in self.performance_configs:
												lowercase__  					=     cmd.copy()
												for i, strategy in enumerate(lowerCamelCase    ):
																if strategy.lower() in config:
																				cmd_config.append(F"""--fsdp_sharding_strategy={i+1}"""    )
																				break

												if "fp32" in config:
																cmd_config.append('''--mixed_precision=no'''    )
												else:
																cmd_config.append('''--mixed_precision=fp16'''    )

												if "cpu_offload" in config:
																cmd_config.append('''--fsdp_offload_params=True'''    )

												for policy in FSDP_AUTO_WRAP_POLICY:
																if policy.lower() in config:
																				cmd_config.append(F"""--fsdp_auto_wrap_policy={policy}"""    )
																				break

												if policy == "TRANSFORMER_BASED_WRAP":
																cmd_config.append('''--fsdp_transformer_layer_cls_to_wrap=BertLayer'''    )
												elif policy == "SIZE_BASED_WRAP":
																cmd_config.append('''--fsdp_min_num_params=2000'''    )

												cmd_config.extend(
												    [
												        self.test_file_path,
												        F"""--output_dir={self.tmpdir}""",
												        F"""--performance_lower_bound={self.performance_lower_bound}""",
												    ]    )
												with patch_environment(omp_num_threads=1    ):
																execute_subprocess_async(lowerCamelCase,    env=os.environ.copy()    )




				def 							lowercase__  (	self		:       int    ):

								'''simple docstring'''
								lowercase__  					=     os.path.join(self.test_scripts_folder,    '''test_checkpointing.py'''    )
								lowercase__  					=     [
								    '''accelerate''',
								    '''launch''',
								    '''--num_processes=2''',
								    '''--num_machines=1''',
								    '''--machine_rank=0''',
								    '''--use_fsdp''',
								    '''--mixed_precision=fp16''',
								    '''--fsdp_transformer_layer_cls_to_wrap=BertLayer''',
								]

								for i, strategy in enumerate(lowerCamelCase    ):
												lowercase__  					=     cmd.copy()
												cmd_config.append(F"""--fsdp_sharding_strategy={i+1}"""    )
												if strategy != "FULL_SHARD":
																continue
												lowercase__  					=     len(lowerCamelCase    )
												for state_dict_type in FSDP_STATE_DICT_TYPE:
																lowercase__  					=     cmd_config[:state_dict_config_index]
																cmd_config.append(F"""--fsdp_state_dict_type={state_dict_type}"""    )
																cmd_config.extend(
																    [
																        self.test_file_path,
																        F"""--output_dir={self.tmpdir}""",
																        '''--partial_train_epoch=1''',
																    ]    )
																with patch_environment(omp_num_threads=1    ):
																				execute_subprocess_async(lowerCamelCase,    env=os.environ.copy()    )

																lowercase__  					=     cmd_config[:-1]
																lowercase__  					=     os.path.join(self.tmpdir,    '''epoch_0'''    )
																cmd_config.extend(
																    [
																        F"""--resume_from_checkpoint={resume_from_checkpoint}""",
																    ]    )
																with patch_environment(omp_num_threads=1    ):
																				execute_subprocess_async(lowerCamelCase,    env=os.environ.copy()    )




				def 							lowercase__  (	self		:       int    ):

								'''simple docstring'''
								lowercase__  					=     os.path.join(self.test_scripts_folder,    '''test_peak_memory_usage.py'''    )
								lowercase__  					=     [
								    '''accelerate''',
								    '''launch''',
								    '''--num_processes=2''',
								    '''--num_machines=1''',
								    '''--machine_rank=0''',
								]
								for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
												lowercase__  					=     cmd.copy()
												if "fp16" in spec:
																cmd_config.extend(['''--mixed_precision=fp16''']    )
												else:
																cmd_config.extend(['''--mixed_precision=no''']    )

												if "multi_gpu" in spec:
																continue
												else:
																cmd_config.extend(['''--use_fsdp''']    )
																for i, strategy in enumerate(lowerCamelCase    ):
																				if strategy.lower() in spec:
																								cmd_config.append(F"""--fsdp_sharding_strategy={i+1}"""    )
																								break

																if "cpu_offload" in spec:
																				cmd_config.append('''--fsdp_offload_params=True'''    )

																for policy in FSDP_AUTO_WRAP_POLICY:
																				if policy.lower() in spec:
																								cmd_config.append(F"""--fsdp_auto_wrap_policy={policy}"""    )
																								break

																if policy == "TRANSFORMER_BASED_WRAP":
																				cmd_config.append('''--fsdp_transformer_layer_cls_to_wrap=BertLayer'''    )
																elif policy == "SIZE_BASED_WRAP":
																				cmd_config.append('''--fsdp_min_num_params=2000'''    )

												cmd_config.extend(
												    [
												        self.test_file_path,
												        F"""--output_dir={self.tmpdir}""",
												        F"""--peak_memory_upper_bound={peak_mem_upper_bound}""",
												        F"""--n_train={self.n_train}""",
												        F"""--n_val={self.n_val}""",
												    ]    )
												with patch_environment(omp_num_threads=1    ):
																execute_subprocess_async(lowerCamelCase,    env=os.environ.copy()    )












import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation


A__      :     Any       =					logging.get_logger(__name__)

A__      :     Tuple       =					{'vocab_file': 'vocab.txt', 'emoji_file': 'emoji.json'}

A__      :     str       =					{
    'vocab_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt',
    },
    'emoji_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json',
    },
}

A__      :     List[Any]       =					{
    'abeja/gpt-neox-japanese-2.7b': 20_48,
}






def     a     (		lowerCamelCase_						,  lowerCamelCase_							):






				'''simple docstring'''
				with open(lowerCamelCase_						,  '''r'''						,  encoding='''utf-8'''							) as f:
								lowercase__  					=     json.loads(f.read()							)

				lowercase__  					=     collections.OrderedDict()
				lowercase__  					=     collections.OrderedDict()
				lowercase__  					=     collections.OrderedDict()
				with open(lowerCamelCase_						,  '''r'''						,  encoding='''utf-8'''							) as f:
								lowercase__  					=     f.readlines()
				lowercase__  					=     [[t.rstrip('''\n'''							)] if (t == ''',''' or ''',''' not in t) else t.rstrip('''\n'''							).split(''','''							) for t in token]
				for idx, b in enumerate(lowerCamelCase_							):
								lowercase__  					=     b
								lowercase__  					=     idx
								for wd in b:
												lowercase__  					=     idx

				return vocab, raw_vocab, ids_to_tokens, emoji
class  _UpperCAmelCase   (			A__      ):

				"""simple docstring"""




				lowercase__											=			VOCAB_FILES_NAMES
				lowercase__											=			PRETRAINED_VOCAB_FILES_MAP
				lowercase__											=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowercase__											=			["""input_ids""", """attention_mask"""]




				def __init__(	self		:       Any,    lowerCamelCase		:       int,    lowerCamelCase		:       Optional[int],    lowerCamelCase		:       Optional[int]="<|endoftext|>",    lowerCamelCase		:       str="<|endoftext|>",    lowerCamelCase		:       str="<|startoftext|>",    lowerCamelCase		:       Tuple="<|endoftext|>",    lowerCamelCase		:       str=False,    **lowerCamelCase		:       List[str],    ):

								'''simple docstring'''
								super().__init__(
								    unk_token=lowerCamelCase,    pad_token=lowerCamelCase,    bos_token=lowerCamelCase,    eos_token=lowerCamelCase,    do_clean_text=lowerCamelCase,    **lowerCamelCase,    )
								if not os.path.isfile(lowerCamelCase    ):
												raise ValueError(
												    F"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
												    ''' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'''    )
								if not os.path.isfile(lowerCamelCase    ):
												raise ValueError(
												    F"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
												    ''' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'''    )
								lowercase__  					=     do_clean_text
								lowercase__					,		lowercase__					,		lowercase__					,		lowercase__  					=     load_vocab_and_emoji(lowerCamelCase,    lowerCamelCase    )
								lowercase__  					=     SubWordJapaneseTokenizer(
								    vocab=self.vocab,    ids_to_tokens=self.ids_to_tokens,    emoji=self.emoji    )




				@property
				def 							lowercase__  (	self		:       int    ):

								'''simple docstring'''
								# self.vocab contains support for character fluctuation unique to Japanese, and has a large number of vocab
								return len(self.raw_vocab    )




				def 							lowercase__  (	self		:       List[str]    ):

								'''simple docstring'''
								return dict(self.raw_vocab,    **self.added_tokens_encoder    )




				def 							lowercase__  (	self		:       str,    lowerCamelCase		:       Any    ):

								'''simple docstring'''
								return self.subword_tokenizer.tokenize(lowerCamelCase,    clean=self.do_clean_text    )




				def 							lowercase__  (	self		:       List[Any],    lowerCamelCase		:       List[str]    ):

								'''simple docstring'''
								return self.vocab.get(lowerCamelCase,    self.vocab.get(self.unk_token    )    )




				def 							lowercase__  (	self		:       Dict,    lowerCamelCase		:       Any    ):

								'''simple docstring'''
								return self.subword_tokenizer.convert_id_to_token(lowerCamelCase    )




				def 							lowercase__  (	self		:       Dict,    lowerCamelCase		:       List[Any]    ):

								'''simple docstring'''
								lowercase__  					=     ''''''.join(lowerCamelCase    ).strip()
								return out_string




				def 							lowercase__  (	self		:       Any,    lowerCamelCase		:       "Conversation"    ):

								'''simple docstring'''
								lowercase__  					=     []
								for is_user, text in conversation.iter_texts():
												input_ids.extend(self.encode(lowerCamelCase,    add_special_tokens=lowerCamelCase    ) + [self.eos_token_id]    )

								if len(lowerCamelCase    ) > self.model_max_length:
												lowercase__  					=     input_ids[-self.model_max_length :]
								return input_ids




				def 							lowercase__  (	self		:       List[Any],    lowerCamelCase		:       str,    lowerCamelCase		:       Optional[str] = None    ):

								'''simple docstring'''
								lowercase__  					=     0
								if os.path.isdir(lowerCamelCase    ):
												lowercase__  					=     os.path.join(
												    lowerCamelCase,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )
												lowercase__  					=     os.path.join(
												    lowerCamelCase,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''emoji_file''']    )
								else:
												lowercase__  					=     (
												    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''vocab_file''']
												)
												lowercase__  					=     (
												    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''emoji_file''']
												)
								with open(lowerCamelCase,    '''w''',    encoding='''utf-8'''    ) as writer:
												for token_index, token in self.ids_to_tokens.items():
																if index != token_index:
																				logger.warning(
																				    F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
																				    ''' Please check that the vocabulary is not corrupted!'''    )
																				lowercase__  					=     token_index
																writer.write(''','''.join(lowerCamelCase    ) + '''\n'''    )
																index += 1
								with open(lowerCamelCase,    '''w''',    encoding='''utf-8'''    ) as writer:
												json.dump(self.emoji,    lowerCamelCase    )
								return vocab_file, emoji_file



class  _UpperCAmelCase   (			A__      ):

				"""simple docstring"""




				def __init__(	self		:       Union[str, Any],    lowerCamelCase		:       Optional[int],    lowerCamelCase		:       int,    lowerCamelCase		:       List[str]    ):

								'''simple docstring'''
								lowercase__  					=     vocab  # same as swe
								lowercase__  					=     ids_to_tokens  # same as bpe
								lowercase__  					=     emoji
								lowercase__  					=     np.max([len(lowerCamelCase    ) for w in self.vocab.keys()]    )
								lowercase__  					=     re.compile(R'''(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'''    )
								lowercase__  					=     re.compile(R'''[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'''    )
								lowercase__  					=     re.compile(R'''[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'''    )
								lowercase__  					=     re.compile(
								    R'''([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'''    )
								lowercase__  					=     re.compile(
								    R'''(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'''    )
								lowercase__  					=     re.compile(
								    R'''((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'''    )
								lowercase__  					=     '''─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'''
								lowercase__  					=     '''▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'''
								lowercase__  					=     str.maketrans({k: '''<BLOCK>''' for k in keisen + blocks}    )




				def __len__(	self		:       Optional[int]    ):

								'''simple docstring'''
								return len(self.ids_to_tokens    )




				def 							lowercase__  (	self		:       Union[str, Any],    lowerCamelCase		:       Dict    ):

								'''simple docstring'''
								lowercase__  					=     self.content_repattera.sub('''<URL>''',    lowerCamelCase    )
								lowercase__  					=     self.content_repattera.sub('''<EMAIL>''',    lowerCamelCase    )
								lowercase__  					=     self.content_repattera.sub('''<TEL>''',    lowerCamelCase    )
								lowercase__  					=     self.content_repattera.sub('''<DATE>''',    lowerCamelCase    )
								lowercase__  					=     self.content_repattera.sub('''<DATE>''',    lowerCamelCase    )
								lowercase__  					=     self.content_repattera.sub('''<PRICE>''',    lowerCamelCase    )
								lowercase__  					=     content.translate(self.content_transa    )
								while "<BLOCK><BLOCK>" in content:
												lowercase__  					=     content.replace('''<BLOCK><BLOCK>''',    '''<BLOCK>'''    )
								return content




				def 							lowercase__  (	self		:       Union[str, Any],    lowerCamelCase		:       List[Any],    lowerCamelCase		:       str=False    ):

								'''simple docstring'''
								lowercase__  					=     text.replace(''' ''',    '''<SP>'''    )
								lowercase__  					=     text.replace('''　''',    '''<SP>'''    )
								lowercase__  					=     text.replace('''\r\n''',    '''<BR>'''    )
								lowercase__  					=     text.replace('''\n''',    '''<BR>'''    )
								lowercase__  					=     text.replace('''\r''',    '''<BR>'''    )
								lowercase__  					=     text.replace('''\t''',    '''<TAB>'''    )
								lowercase__  					=     text.replace('''—''',    '''ー'''    )
								lowercase__  					=     text.replace('''−''',    '''ー'''    )
								for k, v in self.emoji["emoji"].items():
												if k in text:
																lowercase__  					=     text.replace(lowerCamelCase,    lowerCamelCase    )
								if clean:
												lowercase__  					=     self.clean_text(lowerCamelCase    )

								def check_simbol(lowerCamelCase		:       Any    ):
												lowercase__  					=     x.encode()
												if len(lowerCamelCase    ) == 1 and len(lowerCamelCase    ) == 2:
																lowercase__  					=     (int(e[0]    ) << 8) + int(e[1]    )
																if (
																    (c >= 0xC_2_A_1 and c <= 0xC_2_B_F)
																    or (c >= 0xC_7_8_0 and c <= 0xC_7_8_3)
																    or (c >= 0xC_A_B_9 and c <= 0xC_B_B_F)
																    or (c >= 0xC_C_8_0 and c <= 0xC_D_A_2)
																):
																				return True
												return False

								def checkuae(lowerCamelCase		:       Optional[Any]    ):
												lowercase__  					=     x.encode()
												if len(lowerCamelCase    ) == 1 and len(lowerCamelCase    ) == 3:
																lowercase__  					=     (int(e[0]    ) << 16) + (int(e[1]    ) << 8) + int(e[2]    )
																if c >= 0xE_2_8_0_8_0 and c <= 0xE_2_B_0_7_F:
																				return True
												return False

								lowercase__  					=     0
								lowercase__  					=     []
								while pos < len(lowerCamelCase    ):
												lowercase__  					=     min(len(lowerCamelCase    ),    pos + self.maxlen + 1    ) if text[pos] == '''<''' else pos + 3
												lowercase__  					=     []  # (token_id, token, pos)
												for e in range(lowerCamelCase,    lowerCamelCase,    -1    ):
																lowercase__  					=     text[pos:e]
																if wd in self.vocab:
																				if wd[0] == "<" and len(lowerCamelCase    ) > 2:
																								lowercase__  					=     [(self.vocab[wd], wd, e)]
																								break
																				else:
																								candidates.append((self.vocab[wd], wd, e)    )
												if len(lowerCamelCase    ) > 0:
																# the smallest token_id is adopted
																lowercase__					,		lowercase__					,		lowercase__  					=     sorted(lowerCamelCase,    key=lambda lowerCamelCase    : x[0]    )[0]
																result.append(lowerCamelCase    )
																lowercase__  					=     e
												else:
																lowercase__  					=     pos + 1
																lowercase__  					=     text[pos:end]
																if check_simbol(lowerCamelCase    ):
																				result.append('''<KIGOU>'''    )
																elif checkuae(lowerCamelCase    ):
																				result.append('''<U2000U2BFF>'''    )
																else:
																				for i in wd.encode('''utf-8'''    ):
																								result.append('''<|byte%d|>''' % i    )
																lowercase__  					=     end
								return result




				def 							lowercase__  (	self		:       Optional[Any],    lowerCamelCase		:       List[Any],    lowerCamelCase		:       Dict="\n"    ):

								'''simple docstring'''
								lowercase__  					=     []
								lowercase__  					=     []
								lowercase__  					=     self.ids_to_tokens[index][0]
								if word[:6] == "<|byte" and word[-2:] == "|>":
												byte_tokens.append(int(word[6:-2]    )    )
								else:
												if len(lowerCamelCase    ) > 0:
																words.append(bytearray(lowerCamelCase    ).decode('''utf-8''',    errors='''replace'''    )    )
																lowercase__  					=     []
												if word[:7] == "<|emoji" and word[-2:] == "|>":
																words.append(self.emoji['''emoji_inv'''][word]    )
												elif word == "<SP>":
																words.append(''' '''    )
												elif word == "<BR>":
																words.append(lowerCamelCase    )
												elif word == "<TAB>":
																words.append('''\t'''    )
												elif word == "<BLOCK>":
																words.append('''▀'''    )
												elif word == "<KIGOU>":
																words.append('''ǀ'''    )
												elif word == "<U2000U2BFF>":
																words.append('''‖'''    )
												else:
																words.append(lowerCamelCase    )
								if len(lowerCamelCase    ) > 0:
												words.append(bytearray(lowerCamelCase    ).decode('''utf-8''',    errors='''replace'''    )    )
								lowercase__  					=     ''''''.join(lowerCamelCase    )
								return text




