






import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


A__      :     Dict       =					logging.get_logger(__name__)

A__      :     Dict       =					'▁'

A__      :     Dict       =					{
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
    'tokenizer_config_file': 'tokenizer_config.json',
}

A__      :     Optional[Any]       =					{
    'vocab_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json',
    },
    'spm_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model',
    },
    'tokenizer_config_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json',
    },
}

A__      :     Union[str, Any]       =					{
    'facebook/m2m100_418M': 10_24,
}

# fmt: off
A__      :     List[str]       =					{
    'm2m100': ['af', 'am', 'ar', 'ast', 'az', 'ba', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'ceb', 'cs', 'cy', 'da', 'de', 'el', 'en', 'es', 'et', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl', 'gu', 'ha', 'he', 'hi', 'hr', 'ht', 'hu', 'hy', 'id', 'ig', 'ilo', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'lb', 'lg', 'ln', 'lo', 'lt', 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'ne', 'nl', 'no', 'ns', 'oc', 'or', 'pa', 'pl', 'ps', 'pt', 'ro', 'ru', 'sd', 'si', 'sk', 'sl', 'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'th', 'tl', 'tn', 'tr', 'uk', 'ur', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo', 'zh', 'zu'],
    'wmt21': ['en', 'ha', 'is', 'ja', 'cs', 'ru', 'zh', 'de']
}
class  _UpperCAmelCase   (			A__      ):

				"""simple docstring"""




				lowercase__											=			VOCAB_FILES_NAMES
				lowercase__											=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowercase__											=			PRETRAINED_VOCAB_FILES_MAP
				lowercase__											=			["""input_ids""", """attention_mask"""]

				lowercase__											=			[]
				lowercase__											=			[]




				def __init__(	self		:       Optional[int],    lowerCamelCase		:       Tuple,    lowerCamelCase		:       str,    lowerCamelCase		:       List[str]=None,    lowerCamelCase		:       Union[str, Any]=None,    lowerCamelCase		:       Union[str, Any]="<s>",    lowerCamelCase		:       Union[str, Any]="</s>",    lowerCamelCase		:       Dict="</s>",    lowerCamelCase		:       int="<pad>",    lowerCamelCase		:       int="<unk>",    lowerCamelCase		:       str="m2m100",    lowerCamelCase		:       Optional[Dict[str, Any]] = None,    lowerCamelCase		:       Optional[Any]=8,    **lowerCamelCase		:       List[Any],    ):

								'''simple docstring'''
								lowercase__  					=     {} if sp_model_kwargs is None else sp_model_kwargs

								lowercase__  					=     language_codes
								lowercase__  					=     FAIRSEQ_LANGUAGE_CODES[language_codes]
								lowercase__  					=     {lang_code: F"""__{lang_code}__""" for lang_code in fairseq_language_code}

								lowercase__  					=     kwargs.get('''additional_special_tokens''',    []    )
								kwargs["additional_special_tokens"] += [
								    self.get_lang_token(lowerCamelCase    )
								    for lang_code in fairseq_language_code
								    if self.get_lang_token(lowerCamelCase    ) not in kwargs["additional_special_tokens"]
								]

								super().__init__(
								    src_lang=lowerCamelCase,    tgt_lang=lowerCamelCase,    bos_token=lowerCamelCase,    eos_token=lowerCamelCase,    sep_token=lowerCamelCase,    unk_token=lowerCamelCase,    pad_token=lowerCamelCase,    language_codes=lowerCamelCase,    sp_model_kwargs=self.sp_model_kwargs,    num_madeup_words=lowerCamelCase,    **lowerCamelCase,    )

								lowercase__  					=     vocab_file
								lowercase__  					=     load_json(lowerCamelCase    )
								lowercase__  					=     {v: k for k, v in self.encoder.items()}
								lowercase__  					=     spm_file
								lowercase__  					=     load_spm(lowerCamelCase,    self.sp_model_kwargs    )

								lowercase__  					=     len(self.encoder    )

								lowercase__  					=     {
								    self.get_lang_token(lowerCamelCase    ): self.encoder_size + i for i, lang_code in enumerate(lowerCamelCase    )
								}
								lowercase__  					=     {lang_code: self.encoder_size + i for i, lang_code in enumerate(lowerCamelCase    )}
								lowercase__  					=     {v: k for k, v in self.lang_token_to_id.items()}

								lowercase__  					=     src_lang if src_lang is not None else '''en'''
								lowercase__  					=     tgt_lang
								lowercase__  					=     self.get_lang_id(self._src_lang    )
								self.set_src_lang_special_tokens(self._src_lang    )

								lowercase__  					=     num_madeup_words




				@property
				def 							lowercase__  (	self		:       str    ):

								'''simple docstring'''
								return len(self.encoder    ) + len(self.lang_token_to_id    )




				@property
				def 							lowercase__  (	self		:       Optional[Any]    ):

								'''simple docstring'''
								return self._src_lang




				@src_lang.setter
				def 							lowercase__  (	self		:       Dict,    lowerCamelCase		:       str    ):

								'''simple docstring'''
								lowercase__  					=     new_src_lang
								self.set_src_lang_special_tokens(self._src_lang    )




				def 							lowercase__  (	self		:       Tuple,    lowerCamelCase		:       str    ):

								'''simple docstring'''
								return self.sp_model.encode(lowerCamelCase,    out_type=lowerCamelCase    )




				def 							lowercase__  (	self		:       List[str],    lowerCamelCase		:       int    ):

								'''simple docstring'''
								if token in self.lang_token_to_id:
												return self.lang_token_to_id[token]
								return self.encoder.get(lowerCamelCase,    self.encoder[self.unk_token]    )




				def 							lowercase__  (	self		:       Tuple,    lowerCamelCase		:       int    ):

								'''simple docstring'''
								if index in self.id_to_lang_token:
												return self.id_to_lang_token[index]
								return self.decoder.get(lowerCamelCase,    self.unk_token    )




				def 							lowercase__  (	self		:       Dict,    lowerCamelCase		:       Optional[Any]    ):

								'''simple docstring'''
								lowercase__  					=     []
								lowercase__  					=     ''''''
								for token in tokens:
												# make sure that special tokens are not decoded using sentencepiece model
												if token in self.all_special_tokens:
																out_string += self.sp_model.decode(lowerCamelCase    ) + token
																lowercase__  					=     []
												else:
																current_sub_tokens.append(lowerCamelCase    )
								out_string += self.sp_model.decode(lowerCamelCase    )
								return out_string.strip()




				def 							lowercase__  (	self		:       Any,    lowerCamelCase		:       List[int],    lowerCamelCase		:       Optional[List[int]] = None,    lowerCamelCase		:       bool = False    ):

								'''simple docstring'''

								if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=lowerCamelCase,    token_ids_a=lowerCamelCase,    already_has_special_tokens=lowerCamelCase    )

								lowercase__  					=     [1] * len(self.prefix_tokens    )
								lowercase__  					=     [1] * len(self.suffix_tokens    )
								if token_ids_a is None:
												return prefix_ones + ([0] * len(lowerCamelCase    )) + suffix_ones
								return prefix_ones + ([0] * len(lowerCamelCase    )) + ([0] * len(lowerCamelCase    )) + suffix_ones




				def 							lowercase__  (	self		:       List[str],    lowerCamelCase		:       List[int],    lowerCamelCase		:       Optional[List[int]] = None    ):

								'''simple docstring'''
								if token_ids_a is None:
												return self.prefix_tokens + token_ids_a + self.suffix_tokens
								# We don't expect to process pairs, but leave the pair logic for API consistency
								return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




				def 							lowercase__  (	self		:       Tuple    ):

								'''simple docstring'''
								lowercase__  					=     {self.convert_ids_to_tokens(lowerCamelCase    ): i for i in range(self.vocab_size    )}
								vocab.update(self.added_tokens_encoder    )
								return vocab




				def __getstate__(	self		:       Dict    ):

								'''simple docstring'''
								lowercase__  					=     self.__dict__.copy()
								lowercase__  					=     None
								return state




				def __setstate__(	self		:       Optional[Any],    lowerCamelCase		:       Dict    ):

								'''simple docstring'''
								lowercase__  					=     d

								# for backward compatibility
								if not hasattr(self,    '''sp_model_kwargs'''    ):
												lowercase__  					=     {}

								lowercase__  					=     load_spm(self.spm_file,    self.sp_model_kwargs    )




				def 							lowercase__  (	self		:       Tuple,    lowerCamelCase		:       str,    lowerCamelCase		:       Optional[str] = None    ):

								'''simple docstring'''
								lowercase__  					=     Path(lowerCamelCase    )
								if not save_dir.is_dir():
												raise OSError(F"""{save_directory} should be a directory"""    )
								lowercase__  					=     save_dir / (
								    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
								)
								lowercase__  					=     save_dir / (
								    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
								)

								save_json(self.encoder,    lowerCamelCase    )

								if os.path.abspath(self.spm_file    ) != os.path.abspath(lowerCamelCase    ) and os.path.isfile(self.spm_file    ):
												copyfile(self.spm_file,    lowerCamelCase    )
								elif not os.path.isfile(self.spm_file    ):
												with open(lowerCamelCase,    '''wb'''    ) as fi:
																lowercase__  					=     self.sp_model.serialized_model_proto()
																fi.write(lowerCamelCase    )

								return (str(lowerCamelCase    ), str(lowerCamelCase    ))




				def 							lowercase__  (	self		:       Tuple,    lowerCamelCase		:       List[str],    lowerCamelCase		:       str = "en",    lowerCamelCase		:       Optional[List[str]] = None,    lowerCamelCase		:       str = "ro",    **lowerCamelCase		:       Dict,    ):

								'''simple docstring'''
								lowercase__  					=     src_lang
								lowercase__  					=     tgt_lang
								self.set_src_lang_special_tokens(self.src_lang    )
								return super().prepare_seqaseq_batch(lowerCamelCase,    lowerCamelCase,    **lowerCamelCase    )




				def 							lowercase__  (	self		:       int,    lowerCamelCase		:       Any,    lowerCamelCase		:       Optional[str],    lowerCamelCase		:       Optional[str],    **lowerCamelCase		:       List[Any]    ):

								'''simple docstring'''
								if src_lang is None or tgt_lang is None:
												raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''    )
								lowercase__  					=     src_lang
								lowercase__  					=     self(lowerCamelCase,    add_special_tokens=lowerCamelCase,    **lowerCamelCase    )
								lowercase__  					=     self.get_lang_id(lowerCamelCase    )
								lowercase__  					=     tgt_lang_id
								return inputs




				def 							lowercase__  (	self		:       Union[str, Any]    ):

								'''simple docstring'''
								self.set_src_lang_special_tokens(self.src_lang    )




				def 							lowercase__  (	self		:       str    ):

								'''simple docstring'''
								self.set_tgt_lang_special_tokens(self.tgt_lang    )




				def 							lowercase__  (	self		:       Tuple,    lowerCamelCase		:       str    ):

								'''simple docstring'''
								lowercase__  					=     self.get_lang_token(lowerCamelCase    )
								lowercase__  					=     self.lang_token_to_id[lang_token]
								lowercase__  					=     [self.cur_lang_id]
								lowercase__  					=     [self.eos_token_id]




				def 							lowercase__  (	self		:       List[Any],    lowerCamelCase		:       str    ):

								'''simple docstring'''
								lowercase__  					=     self.get_lang_token(lowerCamelCase    )
								lowercase__  					=     self.lang_token_to_id[lang_token]
								lowercase__  					=     [self.cur_lang_id]
								lowercase__  					=     [self.eos_token_id]




				def 							lowercase__  (	self		:       List[str],    lowerCamelCase		:       str    ):

								'''simple docstring'''
								return self.lang_code_to_token[lang]




				def 							lowercase__  (	self		:       Optional[int],    lowerCamelCase		:       str    ):

								'''simple docstring'''
								lowercase__  					=     self.get_lang_token(lowerCamelCase    )
								return self.lang_token_to_id[lang_token]






def     a     (		lowerCamelCase_						,  lowerCamelCase_							):






				'''simple docstring'''
				lowercase__  					=     sentencepiece.SentencePieceProcessor(**lowerCamelCase_							)
				spm.Load(str(lowerCamelCase_							)							)
				return spm






def     a     (		lowerCamelCase_							):






				'''simple docstring'''
				with open(lowerCamelCase_						,  '''r'''							) as f:
								return json.load(lowerCamelCase_							)






def     a     (		lowerCamelCase_						,  lowerCamelCase_							):






				'''simple docstring'''
				with open(lowerCamelCase_						,  '''w'''							) as f:
								json.dump(lowerCamelCase_						,  lowerCamelCase_						,  indent=2							)












import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation


A__      :     Any       =					logging.get_logger(__name__)

A__      :     Tuple       =					{'vocab_file': 'vocab.txt', 'emoji_file': 'emoji.json'}

A__      :     str       =					{
    'vocab_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt',
    },
    'emoji_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json',
    },
}

A__      :     List[Any]       =					{
    'abeja/gpt-neox-japanese-2.7b': 20_48,
}






def     a     (		lowerCamelCase_						,  lowerCamelCase_							):






				'''simple docstring'''
				with open(lowerCamelCase_						,  '''r'''						,  encoding='''utf-8'''							) as f:
								lowercase__  					=     json.loads(f.read()							)

				lowercase__  					=     collections.OrderedDict()
				lowercase__  					=     collections.OrderedDict()
				lowercase__  					=     collections.OrderedDict()
				with open(lowerCamelCase_						,  '''r'''						,  encoding='''utf-8'''							) as f:
								lowercase__  					=     f.readlines()
				lowercase__  					=     [[t.rstrip('''\n'''							)] if (t == ''',''' or ''',''' not in t) else t.rstrip('''\n'''							).split(''','''							) for t in token]
				for idx, b in enumerate(lowerCamelCase_							):
								lowercase__  					=     b
								lowercase__  					=     idx
								for wd in b:
												lowercase__  					=     idx

				return vocab, raw_vocab, ids_to_tokens, emoji
class  _UpperCAmelCase   (			A__      ):

				"""simple docstring"""




				lowercase__											=			VOCAB_FILES_NAMES
				lowercase__											=			PRETRAINED_VOCAB_FILES_MAP
				lowercase__											=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowercase__											=			["""input_ids""", """attention_mask"""]




				def __init__(	self		:       Any,    lowerCamelCase		:       int,    lowerCamelCase		:       Optional[int],    lowerCamelCase		:       Optional[int]="<|endoftext|>",    lowerCamelCase		:       str="<|endoftext|>",    lowerCamelCase		:       str="<|startoftext|>",    lowerCamelCase		:       Tuple="<|endoftext|>",    lowerCamelCase		:       str=False,    **lowerCamelCase		:       List[str],    ):

								'''simple docstring'''
								super().__init__(
								    unk_token=lowerCamelCase,    pad_token=lowerCamelCase,    bos_token=lowerCamelCase,    eos_token=lowerCamelCase,    do_clean_text=lowerCamelCase,    **lowerCamelCase,    )
								if not os.path.isfile(lowerCamelCase    ):
												raise ValueError(
												    F"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
												    ''' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'''    )
								if not os.path.isfile(lowerCamelCase    ):
												raise ValueError(
												    F"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
												    ''' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'''    )
								lowercase__  					=     do_clean_text
								lowercase__					,		lowercase__					,		lowercase__					,		lowercase__  					=     load_vocab_and_emoji(lowerCamelCase,    lowerCamelCase    )
								lowercase__  					=     SubWordJapaneseTokenizer(
								    vocab=self.vocab,    ids_to_tokens=self.ids_to_tokens,    emoji=self.emoji    )




				@property
				def 							lowercase__  (	self		:       int    ):

								'''simple docstring'''
								# self.vocab contains support for character fluctuation unique to Japanese, and has a large number of vocab
								return len(self.raw_vocab    )




				def 							lowercase__  (	self		:       List[str]    ):

								'''simple docstring'''
								return dict(self.raw_vocab,    **self.added_tokens_encoder    )




				def 							lowercase__  (	self		:       str,    lowerCamelCase		:       Any    ):

								'''simple docstring'''
								return self.subword_tokenizer.tokenize(lowerCamelCase,    clean=self.do_clean_text    )




				def 							lowercase__  (	self		:       List[Any],    lowerCamelCase		:       List[str]    ):

								'''simple docstring'''
								return self.vocab.get(lowerCamelCase,    self.vocab.get(self.unk_token    )    )




				def 							lowercase__  (	self		:       Dict,    lowerCamelCase		:       Any    ):

								'''simple docstring'''
								return self.subword_tokenizer.convert_id_to_token(lowerCamelCase    )




				def 							lowercase__  (	self		:       Dict,    lowerCamelCase		:       List[Any]    ):

								'''simple docstring'''
								lowercase__  					=     ''''''.join(lowerCamelCase    ).strip()
								return out_string




				def 							lowercase__  (	self		:       Any,    lowerCamelCase		:       "Conversation"    ):

								'''simple docstring'''
								lowercase__  					=     []
								for is_user, text in conversation.iter_texts():
												input_ids.extend(self.encode(lowerCamelCase,    add_special_tokens=lowerCamelCase    ) + [self.eos_token_id]    )

								if len(lowerCamelCase    ) > self.model_max_length:
												lowercase__  					=     input_ids[-self.model_max_length :]
								return input_ids




				def 							lowercase__  (	self		:       List[Any],    lowerCamelCase		:       str,    lowerCamelCase		:       Optional[str] = None    ):

								'''simple docstring'''
								lowercase__  					=     0
								if os.path.isdir(lowerCamelCase    ):
												lowercase__  					=     os.path.join(
												    lowerCamelCase,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )
												lowercase__  					=     os.path.join(
												    lowerCamelCase,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''emoji_file''']    )
								else:
												lowercase__  					=     (
												    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''vocab_file''']
												)
												lowercase__  					=     (
												    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''emoji_file''']
												)
								with open(lowerCamelCase,    '''w''',    encoding='''utf-8'''    ) as writer:
												for token_index, token in self.ids_to_tokens.items():
																if index != token_index:
																				logger.warning(
																				    F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
																				    ''' Please check that the vocabulary is not corrupted!'''    )
																				lowercase__  					=     token_index
																writer.write(''','''.join(lowerCamelCase    ) + '''\n'''    )
																index += 1
								with open(lowerCamelCase,    '''w''',    encoding='''utf-8'''    ) as writer:
												json.dump(self.emoji,    lowerCamelCase    )
								return vocab_file, emoji_file



class  _UpperCAmelCase   (			A__      ):

				"""simple docstring"""




				def __init__(	self		:       Union[str, Any],    lowerCamelCase		:       Optional[int],    lowerCamelCase		:       int,    lowerCamelCase		:       List[str]    ):

								'''simple docstring'''
								lowercase__  					=     vocab  # same as swe
								lowercase__  					=     ids_to_tokens  # same as bpe
								lowercase__  					=     emoji
								lowercase__  					=     np.max([len(lowerCamelCase    ) for w in self.vocab.keys()]    )
								lowercase__  					=     re.compile(R'''(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'''    )
								lowercase__  					=     re.compile(R'''[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'''    )
								lowercase__  					=     re.compile(R'''[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'''    )
								lowercase__  					=     re.compile(
								    R'''([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'''    )
								lowercase__  					=     re.compile(
								    R'''(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'''    )
								lowercase__  					=     re.compile(
								    R'''((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'''    )
								lowercase__  					=     '''─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'''
								lowercase__  					=     '''▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'''
								lowercase__  					=     str.maketrans({k: '''<BLOCK>''' for k in keisen + blocks}    )




				def __len__(	self		:       Optional[int]    ):

								'''simple docstring'''
								return len(self.ids_to_tokens    )




				def 							lowercase__  (	self		:       Union[str, Any],    lowerCamelCase		:       Dict    ):

								'''simple docstring'''
								lowercase__  					=     self.content_repattera.sub('''<URL>''',    lowerCamelCase    )
								lowercase__  					=     self.content_repattera.sub('''<EMAIL>''',    lowerCamelCase    )
								lowercase__  					=     self.content_repattera.sub('''<TEL>''',    lowerCamelCase    )
								lowercase__  					=     self.content_repattera.sub('''<DATE>''',    lowerCamelCase    )
								lowercase__  					=     self.content_repattera.sub('''<DATE>''',    lowerCamelCase    )
								lowercase__  					=     self.content_repattera.sub('''<PRICE>''',    lowerCamelCase    )
								lowercase__  					=     content.translate(self.content_transa    )
								while "<BLOCK><BLOCK>" in content:
												lowercase__  					=     content.replace('''<BLOCK><BLOCK>''',    '''<BLOCK>'''    )
								return content




				def 							lowercase__  (	self		:       Union[str, Any],    lowerCamelCase		:       List[Any],    lowerCamelCase		:       str=False    ):

								'''simple docstring'''
								lowercase__  					=     text.replace(''' ''',    '''<SP>'''    )
								lowercase__  					=     text.replace('''　''',    '''<SP>'''    )
								lowercase__  					=     text.replace('''\r\n''',    '''<BR>'''    )
								lowercase__  					=     text.replace('''\n''',    '''<BR>'''    )
								lowercase__  					=     text.replace('''\r''',    '''<BR>'''    )
								lowercase__  					=     text.replace('''\t''',    '''<TAB>'''    )
								lowercase__  					=     text.replace('''—''',    '''ー'''    )
								lowercase__  					=     text.replace('''−''',    '''ー'''    )
								for k, v in self.emoji["emoji"].items():
												if k in text:
																lowercase__  					=     text.replace(lowerCamelCase,    lowerCamelCase    )
								if clean:
												lowercase__  					=     self.clean_text(lowerCamelCase    )

								def check_simbol(lowerCamelCase		:       Any    ):
												lowercase__  					=     x.encode()
												if len(lowerCamelCase    ) == 1 and len(lowerCamelCase    ) == 2:
																lowercase__  					=     (int(e[0]    ) << 8) + int(e[1]    )
																if (
																    (c >= 0xC_2_A_1 and c <= 0xC_2_B_F)
																    or (c >= 0xC_7_8_0 and c <= 0xC_7_8_3)
																    or (c >= 0xC_A_B_9 and c <= 0xC_B_B_F)
																    or (c >= 0xC_C_8_0 and c <= 0xC_D_A_2)
																):
																				return True
												return False

								def checkuae(lowerCamelCase		:       Optional[Any]    ):
												lowercase__  					=     x.encode()
												if len(lowerCamelCase    ) == 1 and len(lowerCamelCase    ) == 3:
																lowercase__  					=     (int(e[0]    ) << 16) + (int(e[1]    ) << 8) + int(e[2]    )
																if c >= 0xE_2_8_0_8_0 and c <= 0xE_2_B_0_7_F:
																				return True
												return False

								lowercase__  					=     0
								lowercase__  					=     []
								while pos < len(lowerCamelCase    ):
												lowercase__  					=     min(len(lowerCamelCase    ),    pos + self.maxlen + 1    ) if text[pos] == '''<''' else pos + 3
												lowercase__  					=     []  # (token_id, token, pos)
												for e in range(lowerCamelCase,    lowerCamelCase,    -1    ):
																lowercase__  					=     text[pos:e]
																if wd in self.vocab:
																				if wd[0] == "<" and len(lowerCamelCase    ) > 2:
																								lowercase__  					=     [(self.vocab[wd], wd, e)]
																								break
																				else:
																								candidates.append((self.vocab[wd], wd, e)    )
												if len(lowerCamelCase    ) > 0:
																# the smallest token_id is adopted
																lowercase__					,		lowercase__					,		lowercase__  					=     sorted(lowerCamelCase,    key=lambda lowerCamelCase    : x[0]    )[0]
																result.append(lowerCamelCase    )
																lowercase__  					=     e
												else:
																lowercase__  					=     pos + 1
																lowercase__  					=     text[pos:end]
																if check_simbol(lowerCamelCase    ):
																				result.append('''<KIGOU>'''    )
																elif checkuae(lowerCamelCase    ):
																				result.append('''<U2000U2BFF>'''    )
																else:
																				for i in wd.encode('''utf-8'''    ):
																								result.append('''<|byte%d|>''' % i    )
																lowercase__  					=     end
								return result




				def 							lowercase__  (	self		:       Optional[Any],    lowerCamelCase		:       List[Any],    lowerCamelCase		:       Dict="\n"    ):

								'''simple docstring'''
								lowercase__  					=     []
								lowercase__  					=     []
								lowercase__  					=     self.ids_to_tokens[index][0]
								if word[:6] == "<|byte" and word[-2:] == "|>":
												byte_tokens.append(int(word[6:-2]    )    )
								else:
												if len(lowerCamelCase    ) > 0:
																words.append(bytearray(lowerCamelCase    ).decode('''utf-8''',    errors='''replace'''    )    )
																lowercase__  					=     []
												if word[:7] == "<|emoji" and word[-2:] == "|>":
																words.append(self.emoji['''emoji_inv'''][word]    )
												elif word == "<SP>":
																words.append(''' '''    )
												elif word == "<BR>":
																words.append(lowerCamelCase    )
												elif word == "<TAB>":
																words.append('''\t'''    )
												elif word == "<BLOCK>":
																words.append('''▀'''    )
												elif word == "<KIGOU>":
																words.append('''ǀ'''    )
												elif word == "<U2000U2BFF>":
																words.append('''‖'''    )
												else:
																words.append(lowerCamelCase    )
								if len(lowerCamelCase    ) > 0:
												words.append(bytearray(lowerCamelCase    ).decode('''utf-8''',    errors='''replace'''    )    )
								lowercase__  					=     ''''''.join(lowerCamelCase    )
								return text




