


'''simple docstring'''



import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__A		=logging.get_logger(__name__)


__A		={'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

__A		={
    'vocab_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
    },
}

__A		={
    'allenai/longformer-base-4096': 40_96,
    'allenai/longformer-large-4096': 40_96,
    'allenai/longformer-large-4096-finetuned-triviaqa': 40_96,
    'allenai/longformer-base-4096-extra.pos.embd.only': 40_96,
    'allenai/longformer-large-4096-extra.pos.embd.only': 40_96,
}







@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def       _UpperCamelCase (						):
			UpperCAmelCase__					:   Optional[int]       				=       (
			    list(range(ord("""!"""   )	, ord("""~"""   ) + 1   )   ) + list(range(ord("""¡"""   )	, ord("""¬"""   ) + 1   )   ) + list(range(ord("""®"""   )	, ord("""ÿ"""   ) + 1   )   )
			)
			UpperCAmelCase__					:   Any       				=       bs[:]
			UpperCAmelCase__					:   Union[str, Any]       				=       0
			for b in range(2**8   ):
						if b not in bs:
									bs.append(UpperCamelCase__   )
									cs.append(2**8 + n   )
									n += 1
			UpperCAmelCase__					:   List[str]       				=       [chr(UpperCamelCase__   ) for n in cs]
			return dict(zip(UpperCamelCase__	, UpperCamelCase__   )   )







def       _UpperCamelCase (						UpperCamelCase__   ):
			UpperCAmelCase__					:   Union[str, Any]       				=       set()
			UpperCAmelCase__					:   Any       				=       word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)   )
						UpperCAmelCase__					:   Tuple       				=       char
			return pairs


class 						_snake_case   (       a__		):
					lowerCAmelCase :Dict						=   VOCAB_FILES_NAMES
					lowerCAmelCase :Any						=   PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase :Tuple						=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase :Optional[int]						=   ['''input_ids''', '''attention_mask''']




					def __init__( self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase="replace"							,							_lowerCamelCase="<s>"							,							_lowerCamelCase="</s>"							,							_lowerCamelCase="</s>"							,							_lowerCamelCase="<s>"							,							_lowerCamelCase="<unk>"							,							_lowerCamelCase="<pad>"							,							_lowerCamelCase="<mask>"							,							_lowerCamelCase=False							,							**_lowerCamelCase							,							):
								UpperCAmelCase__					:   int       				=       AddedToken(_lowerCamelCase							,							lstrip=_lowerCamelCase							,							rstrip=_lowerCamelCase) if isinstance(_lowerCamelCase							,							_lowerCamelCase) else bos_token
								UpperCAmelCase__					:   str       				=       AddedToken(_lowerCamelCase							,							lstrip=_lowerCamelCase							,							rstrip=_lowerCamelCase) if isinstance(_lowerCamelCase							,							_lowerCamelCase) else eos_token
								UpperCAmelCase__					:   List[str]       				=       AddedToken(_lowerCamelCase							,							lstrip=_lowerCamelCase							,							rstrip=_lowerCamelCase) if isinstance(_lowerCamelCase							,							_lowerCamelCase) else sep_token
								UpperCAmelCase__					:   str       				=       AddedToken(_lowerCamelCase							,							lstrip=_lowerCamelCase							,							rstrip=_lowerCamelCase) if isinstance(_lowerCamelCase							,							_lowerCamelCase) else cls_token
								UpperCAmelCase__					:   Optional[int]       				=       AddedToken(_lowerCamelCase							,							lstrip=_lowerCamelCase							,							rstrip=_lowerCamelCase) if isinstance(_lowerCamelCase							,							_lowerCamelCase) else unk_token
								UpperCAmelCase__					:   List[Any]       				=       AddedToken(_lowerCamelCase							,							lstrip=_lowerCamelCase							,							rstrip=_lowerCamelCase) if isinstance(_lowerCamelCase							,							_lowerCamelCase) else pad_token

								# Mask token behave like a normal word, i.e. include the space before it
								UpperCAmelCase__					:   int       				=       AddedToken(_lowerCamelCase							,							lstrip=_lowerCamelCase							,							rstrip=_lowerCamelCase) if isinstance(_lowerCamelCase							,							_lowerCamelCase) else mask_token

								super().__init__(
								    errors=_lowerCamelCase							,							bos_token=_lowerCamelCase							,							eos_token=_lowerCamelCase							,							unk_token=_lowerCamelCase							,							sep_token=_lowerCamelCase							,							cls_token=_lowerCamelCase							,							pad_token=_lowerCamelCase							,							mask_token=_lowerCamelCase							,							add_prefix_space=_lowerCamelCase							,							**_lowerCamelCase							,							)

								with open(_lowerCamelCase							,							encoding="""utf-8""") as vocab_handle:
											UpperCAmelCase__					:   str       				=       json.load(_lowerCamelCase)
								UpperCAmelCase__					:   Any       				=       {v: k for k, v in self.encoder.items()}
								UpperCAmelCase__					:   str       				=       errors  # how to handle errors in decoding
								UpperCAmelCase__					:   List[Any]       				=       bytes_to_unicode()
								UpperCAmelCase__					:   int       				=       {v: k for k, v in self.byte_encoder.items()}
								with open(_lowerCamelCase							,							encoding="""utf-8""") as merges_handle:
											UpperCAmelCase__					:   List[Any]       				=       merges_handle.read().split("""\n""")[1:-1]
								UpperCAmelCase__					:   str       				=       [tuple(merge.split()) for merge in bpe_merges]
								UpperCAmelCase__					:   Dict       				=       dict(zip(_lowerCamelCase							,							range(len(_lowerCamelCase))))
								UpperCAmelCase__					:   Dict       				=       {}
								UpperCAmelCase__					:   str       				=       add_prefix_space

								# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
								UpperCAmelCase__					:   Optional[int]       				=       re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")




					@property
					def 			snake_case__  ( self):
								return len(self.encoder)




					def 			snake_case__  ( self):
								return dict(self.encoder							,							**self.added_tokens_encoder)




					def 			snake_case__  ( self							,							_lowerCamelCase):
								if token in self.cache:
											return self.cache[token]
								UpperCAmelCase__					:   Tuple       				=       tuple(_lowerCamelCase)
								UpperCAmelCase__					:   Optional[Any]       				=       get_pairs(_lowerCamelCase)

								if not pairs:
											return token

								while True:
											UpperCAmelCase__					:   List[str]       				=       min(_lowerCamelCase							,							key=lambda _lowerCamelCase: self.bpe_ranks.get(_lowerCamelCase							,							float("""inf""")))
											if bigram not in self.bpe_ranks:
														break
											UpperCAmelCase__					:   Any       				=       bigram
											UpperCAmelCase__					:   Union[str, Any]       				=       []
											UpperCAmelCase__					:   List[str]       				=       0
											while i < len(_lowerCamelCase):
														try:
																	UpperCAmelCase__					:   int       				=       word.index(_lowerCamelCase							,							_lowerCamelCase)
														except ValueError:
																	new_word.extend(word[i:])
																	break
														else:
																	new_word.extend(word[i:j])
																	UpperCAmelCase__					:   int       				=       j

														if word[i] == first and i < len(_lowerCamelCase) - 1 and word[i + 1] == second:
																	new_word.append(first + second)
																	i += 2
														else:
																	new_word.append(word[i])
																	i += 1
											UpperCAmelCase__					:   Any       				=       tuple(_lowerCamelCase)
											UpperCAmelCase__					:   Optional[Any]       				=       new_word
											if len(_lowerCamelCase) == 1:
														break
											else:
														UpperCAmelCase__					:   int       				=       get_pairs(_lowerCamelCase)
								UpperCAmelCase__					:   Dict       				=       """ """.join(_lowerCamelCase)
								UpperCAmelCase__					:   Dict       				=       word
								return word




					def 			snake_case__  ( self							,							_lowerCamelCase):
								UpperCAmelCase__					:   Dict       				=       []
								for token in re.findall(self.pat							,							_lowerCamelCase):
											UpperCAmelCase__					:   Dict       				=       """""".join(
											    self.byte_encoder[b] for b in token.encode("""utf-8"""))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
											bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowerCamelCase).split(""" """))
								return bpe_tokens




					def 			snake_case__  ( self							,							_lowerCamelCase):
								return self.encoder.get(_lowerCamelCase							,							self.encoder.get(self.unk_token))




					def 			snake_case__  ( self							,							_lowerCamelCase):
								return self.decoder.get(_lowerCamelCase)




					def 			snake_case__  ( self							,							_lowerCamelCase):
								UpperCAmelCase__					:   Optional[int]       				=       """""".join(_lowerCamelCase)
								UpperCAmelCase__					:   Tuple       				=       bytearray([self.byte_decoder[c] for c in text]).decode("""utf-8"""							,							errors=self.errors)
								return text




					def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase = None):
								if not os.path.isdir(_lowerCamelCase):
											logger.error(f'''Vocabulary path ({save_directory}) should be a directory''')
											return
								UpperCAmelCase__					:   Dict       				=       os.path.join(
								    _lowerCamelCase							,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])
								UpperCAmelCase__					:   Optional[int]       				=       os.path.join(
								    _lowerCamelCase							,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""])

								with open(_lowerCamelCase							,							"""w"""							,							encoding="""utf-8""") as f:
											f.write(json.dumps(self.encoder							,							indent=2							,							sort_keys=_lowerCamelCase							,							ensure_ascii=_lowerCamelCase) + """\n""")

								UpperCAmelCase__					:   List[str]       				=       0
								with open(_lowerCamelCase							,							"""w"""							,							encoding="""utf-8""") as writer:
											writer.write("""#version: 0.2\n""")
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,							key=lambda _lowerCamelCase: kv[1]):
														if index != token_index:
																	logger.warning(
																	    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																	    """ Please check that the tokenizer is not corrupted!""")
																	UpperCAmelCase__					:   Tuple       				=       token_index
														writer.write(""" """.join(_lowerCamelCase) + """\n""")
														index += 1

								return vocab_file, merge_file




					def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase = None):
								if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								UpperCAmelCase__					:   Optional[Any]       				=       [self.cls_token_id]
								UpperCAmelCase__					:   Dict       				=       [self.sep_token_id]
								return cls + token_ids_a + sep + sep + token_ids_a + sep




					def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase = None							,							_lowerCamelCase = False):
								if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=_lowerCamelCase							,							token_ids_a=_lowerCamelCase							,							already_has_special_tokens=_lowerCamelCase)

								if token_ids_a is None:
											return [1] + ([0] * len(_lowerCamelCase)) + [1]
								return [1] + ([0] * len(_lowerCamelCase)) + [1, 1] + ([0] * len(_lowerCamelCase)) + [1]




					def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase = None):
								UpperCAmelCase__					:   List[str]       				=       [self.sep_token_id]
								UpperCAmelCase__					:   int       				=       [self.cls_token_id]

								if token_ids_a is None:
											return len(cls + token_ids_a + sep) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]




					def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase=False							,							**_lowerCamelCase):
								UpperCAmelCase__					:   Optional[int]       				=       kwargs.pop("""add_prefix_space"""							,							self.add_prefix_space)
								if (is_split_into_words or add_prefix_space) and (len(_lowerCamelCase) > 0 and not text[0].isspace()):
											UpperCAmelCase__					:   Tuple       				=       """ """ + text
								return (text, kwargs)



'''simple docstring'''



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


__A		=logging.get_logger(__name__)

__A		={'vocab_file': 'spiece.model'}

__A		={
    'vocab_file': {
        'TsinghuaAI/CPM-Generate': 'https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model',
    }
}


class 						_snake_case   (       a__		):




					def __init__( self							,							_lowerCamelCase							,							_lowerCamelCase=False							,							_lowerCamelCase=True							,							_lowerCamelCase=False							,							_lowerCamelCase="<s>"							,							_lowerCamelCase="</s>"							,							_lowerCamelCase="<unk>"							,							_lowerCamelCase="<sep>"							,							_lowerCamelCase="<pad>"							,							_lowerCamelCase="<cls>"							,							_lowerCamelCase="<mask>"							,							_lowerCamelCase=["<eop>", "<eod>"]							,							_lowerCamelCase = None							,							**_lowerCamelCase							,							):
								UpperCAmelCase__					:   Dict       				=       AddedToken(_lowerCamelCase							,							lstrip=_lowerCamelCase							,							rstrip=_lowerCamelCase) if isinstance(_lowerCamelCase							,							_lowerCamelCase) else mask_token

								UpperCAmelCase__					:   Any       				=       {} if sp_model_kwargs is None else sp_model_kwargs

								super().__init__(
								    do_lower_case=_lowerCamelCase							,							remove_space=_lowerCamelCase							,							keep_accents=_lowerCamelCase							,							bos_token=_lowerCamelCase							,							eos_token=_lowerCamelCase							,							unk_token=_lowerCamelCase							,							sep_token=_lowerCamelCase							,							pad_token=_lowerCamelCase							,							cls_token=_lowerCamelCase							,							mask_token=_lowerCamelCase							,							additional_special_tokens=_lowerCamelCase							,							sp_model_kwargs=self.sp_model_kwargs							,							**_lowerCamelCase							,							)

								UpperCAmelCase__					:   List[str]       				=       3

								UpperCAmelCase__					:   str       				=       do_lower_case
								UpperCAmelCase__					:   Tuple       				=       remove_space
								UpperCAmelCase__					:   List[str]       				=       keep_accents
								UpperCAmelCase__					:   Any       				=       vocab_file

								UpperCAmelCase__					:   str       				=       spm.SentencePieceProcessor(**self.sp_model_kwargs)
								self.sp_model.Load(_lowerCamelCase)

								try:
											import jieba
								except ModuleNotFoundError as error:
											raise error.__class__(
											    """You need to install jieba to use CpmTokenizer or CpmTokenizerFast. """
											    """See https://pypi.org/project/jieba/ for installation.""")
								UpperCAmelCase__					:   Union[str, Any]       				=       jieba
								UpperCAmelCase__					:   int       				=       str.maketrans(""" \n"""							,							"""\u2582\u2583""")




					@property
					# Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
					def 			snake_case__  ( self):
								return len(self.sp_model)




					def 			snake_case__  ( self):
								UpperCAmelCase__					:   Optional[Any]       				=       {self.convert_ids_to_tokens(_lowerCamelCase): i for i in range(self.vocab_size)}
								vocab.update(self.added_tokens_encoder)
								return vocab




					def __getstate__( self):
								UpperCAmelCase__					:   List[str]       				=       self.__dict__.copy()
								UpperCAmelCase__					:   Optional[int]       				=       None
								return state




					def __setstate__( self							,							_lowerCamelCase):
								UpperCAmelCase__					:   Optional[Any]       				=       d

								# for backward compatibility
								if not hasattr(self							,							"""sp_model_kwargs"""):
											UpperCAmelCase__					:   Tuple       				=       {}

								UpperCAmelCase__					:   Union[str, Any]       				=       spm.SentencePieceProcessor(**self.sp_model_kwargs)
								self.sp_model.Load(self.vocab_file)




					def 			snake_case__  ( self							,							_lowerCamelCase):
								if self.remove_space:
											UpperCAmelCase__					:   Tuple       				=       """ """.join(inputs.strip().split())
								else:
											UpperCAmelCase__					:   str       				=       inputs
								UpperCAmelCase__					:   List[Any]       				=       outputs.replace("""``"""							,							"""\"""").replace("""''"""							,							"""\"""")

								if not self.keep_accents:
											UpperCAmelCase__					:   str       				=       unicodedata.normalize("""NFKD"""							,							_lowerCamelCase)
											UpperCAmelCase__					:   Any       				=       """""".join([c for c in outputs if not unicodedata.combining(_lowerCamelCase)])
								if self.do_lower_case:
											UpperCAmelCase__					:   Optional[int]       				=       outputs.lower()

								return outputs




					def 			snake_case__  ( self							,							_lowerCamelCase):
								UpperCAmelCase__					:   List[str]       				=       self.preprocess_text(_lowerCamelCase)
								UpperCAmelCase__					:   Dict       				=       self.sp_model.encode(_lowerCamelCase							,							out_type=_lowerCamelCase)
								UpperCAmelCase__					:   List[Any]       				=       []
								for piece in pieces:
											if len(_lowerCamelCase) > 1 and piece[-1] == str(""",""") and piece[-2].isdigit():
														UpperCAmelCase__					:   str       				=       self.sp_model.EncodeAsPieces(piece[:-1].replace(_lowerCamelCase							,							""""""))
														if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																	if len(cur_pieces[0]) == 1:
																				UpperCAmelCase__					:   Optional[int]       				=       cur_pieces[1:]
																	else:
																				UpperCAmelCase__					:   Optional[int]       				=       cur_pieces[0][1:]
														cur_pieces.append(piece[-1])
														new_pieces.extend(_lowerCamelCase)
											else:
														new_pieces.append(_lowerCamelCase)

								return new_pieces




					def 			snake_case__  ( self							,							_lowerCamelCase):
								return self.sp_model.PieceToId(_lowerCamelCase)




					def 			snake_case__  ( self							,							_lowerCamelCase):
								return self.sp_model.IdToPiece(_lowerCamelCase)




					def 			snake_case__  ( self							,							_lowerCamelCase):
								UpperCAmelCase__					:   str       				=       """""".join(_lowerCamelCase).replace(_lowerCamelCase							,							""" """).strip()
								return out_string




					def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase = None):
								UpperCAmelCase__					:   Optional[Any]       				=       [self.sep_token_id]
								UpperCAmelCase__					:   Optional[int]       				=       [self.cls_token_id]
								if token_ids_a is None:
											return token_ids_a + sep + cls
								return token_ids_a + sep + token_ids_a + sep + cls




					def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase = None							,							_lowerCamelCase = False):

								if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=_lowerCamelCase							,							token_ids_a=_lowerCamelCase							,							already_has_special_tokens=_lowerCamelCase)

								if token_ids_a is not None:
											return ([0] * len(_lowerCamelCase)) + [1] + ([0] * len(_lowerCamelCase)) + [1, 1]
								return ([0] * len(_lowerCamelCase)) + [1, 1]




					def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase = None):
								UpperCAmelCase__					:   int       				=       [self.sep_token_id]
								UpperCAmelCase__					:   Tuple       				=       [2]

								if token_ids_a is None:
											return len(token_ids_a + sep) * [0] + cls_segment_id
								return len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1] + cls_segment_id




					def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase = None):
								if not os.path.isdir(_lowerCamelCase):
											logger.error(f'''Vocabulary path ({save_directory}) should be a directory''')
											return
								UpperCAmelCase__					:   str       				=       os.path.join(
								    _lowerCamelCase							,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])

								if os.path.abspath(self.vocab_file) != os.path.abspath(_lowerCamelCase) and os.path.isfile(self.vocab_file):
											copyfile(self.vocab_file							,							_lowerCamelCase)
								elif not os.path.isfile(self.vocab_file):
											with open(_lowerCamelCase							,							"""wb""") as fi:
														UpperCAmelCase__					:   str       				=       self.sp_model.serialized_model_proto()
														fi.write(_lowerCamelCase)

								return (out_vocab_file,)




					def 			snake_case__  ( self							,							*_lowerCamelCase							,							**_lowerCamelCase):
								UpperCAmelCase__					:   Union[str, Any]       				=       super()._decode(*_lowerCamelCase							,							**_lowerCamelCase)
								UpperCAmelCase__					:   Any       				=       text.replace(""" """							,							"""""").replace("""\u2582"""							,							""" """).replace("""\u2583"""							,							"""\n""")
								return text