




import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCamelCase					:     List[str]					  =  logging.get_logger(__name__)


_lowerCamelCase					:     Any					  =  {"vocab_file": "vocab.json", "merges_file": "merges.txt"}

# See all MVP models at https://huggingface.co/models?filter=mvp
_lowerCamelCase					:     List[str]					  =  {
    "vocab_file": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json",
    },
    "added_tokens.json": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json",
    },
    "merges_file": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt",
    },
}

_lowerCamelCase					:     Optional[Any]					  =  {
    "RUCAIBox/mvp": 1_0_2_4,
}




@lru_cache()
def 							_UpperCAmelCase     ():
				'''simple docstring'''
				_lowerCAmelCase			:			Dict        =					(
				    list(range(ord("""!"""				)					,			ord("""~"""				) + 1				)				) + list(range(ord("""¡"""				)					,			ord("""¬"""				) + 1				)				) + list(range(ord("""®"""				)					,			ord("""ÿ"""				) + 1				)				)
				)
				_lowerCAmelCase			:			Dict        =					bs[:]
				_lowerCAmelCase			:			List[Any]        =					0
				for b in range(2**8				):
								if b not in bs:
												bs.append(lowercase_				)
												cs.append(2**8 + n				)
												n += 1
				_lowerCAmelCase			:			Optional[Any]        =					[chr(lowercase_				) for n in cs]
				return dict(zip(lowercase_					,			lowercase_				)				)




def 							_UpperCAmelCase     (UpperCamelCase_       :       Optional[Any]				):
				'''simple docstring'''
				_lowerCAmelCase			:			Optional[int]        =					set()
				_lowerCAmelCase			:			str        =					word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)				)
								_lowerCAmelCase			:			Optional[Any]        =					char
				return pairs





class 				__snake_case     (_SCREAMING_SNAKE_CASE						):
				lowerCAmelCase__			=   VOCAB_FILES_NAMES
				lowerCAmelCase__			=   PRETRAINED_VOCAB_FILES_MAP
				lowerCAmelCase__			=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCAmelCase__			=   ["input_ids", "attention_mask"]



				def __init__(					self					:						List[Any]						,		_UpperCAmelCase					:						List[str]						,		_UpperCAmelCase					:						Any						,		_UpperCAmelCase					:						int="replace"						,		_UpperCAmelCase					:						Optional[int]="<s>"						,		_UpperCAmelCase					:						List[Any]="</s>"						,		_UpperCAmelCase					:						str="</s>"						,		_UpperCAmelCase					:						Optional[Any]="<s>"						,		_UpperCAmelCase					:						Any="<unk>"						,		_UpperCAmelCase					:						List[Any]="<pad>"						,		_UpperCAmelCase					:						int="<mask>"						,		_UpperCAmelCase					:						List[Any]=False						,		**_UpperCAmelCase					:						Tuple						,		)     ->			str:





								'''simple docstring'''




								_lowerCAmelCase			:			Optional[int]        =					AddedToken(_UpperCAmelCase						,		lstrip=_UpperCAmelCase						,		rstrip=_UpperCAmelCase					) if isinstance(_UpperCAmelCase						,		_UpperCAmelCase					) else bos_token
								_lowerCAmelCase			:			List[Any]        =					AddedToken(_UpperCAmelCase						,		lstrip=_UpperCAmelCase						,		rstrip=_UpperCAmelCase					) if isinstance(_UpperCAmelCase						,		_UpperCAmelCase					) else eos_token
								_lowerCAmelCase			:			int        =					AddedToken(_UpperCAmelCase						,		lstrip=_UpperCAmelCase						,		rstrip=_UpperCAmelCase					) if isinstance(_UpperCAmelCase						,		_UpperCAmelCase					) else sep_token
								_lowerCAmelCase			:			Union[str, Any]        =					AddedToken(_UpperCAmelCase						,		lstrip=_UpperCAmelCase						,		rstrip=_UpperCAmelCase					) if isinstance(_UpperCAmelCase						,		_UpperCAmelCase					) else cls_token
								_lowerCAmelCase			:			Optional[int]        =					AddedToken(_UpperCAmelCase						,		lstrip=_UpperCAmelCase						,		rstrip=_UpperCAmelCase					) if isinstance(_UpperCAmelCase						,		_UpperCAmelCase					) else unk_token
								_lowerCAmelCase			:			Optional[int]        =					AddedToken(_UpperCAmelCase						,		lstrip=_UpperCAmelCase						,		rstrip=_UpperCAmelCase					) if isinstance(_UpperCAmelCase						,		_UpperCAmelCase					) else pad_token

								# Mask token behave like a normal word, i.e. include the space before it
								_lowerCAmelCase			:			str        =					AddedToken(_UpperCAmelCase						,		lstrip=_UpperCAmelCase						,		rstrip=_UpperCAmelCase					) if isinstance(_UpperCAmelCase						,		_UpperCAmelCase					) else mask_token

								super().__init__(
								    errors=_UpperCAmelCase						,		bos_token=_UpperCAmelCase						,		eos_token=_UpperCAmelCase						,		unk_token=_UpperCAmelCase						,		sep_token=_UpperCAmelCase						,		cls_token=_UpperCAmelCase						,		pad_token=_UpperCAmelCase						,		mask_token=_UpperCAmelCase						,		add_prefix_space=_UpperCAmelCase						,		**_UpperCAmelCase						,		)
								with open(_UpperCAmelCase						,		encoding="""utf-8"""					) as vocab_handle:
												_lowerCAmelCase			:			str        =					json.load(_UpperCAmelCase					)
								_lowerCAmelCase			:			str        =					{v: k for k, v in self.encoder.items()}
								_lowerCAmelCase			:			Optional[Any]        =					errors  # how to handle errors in decoding
								_lowerCAmelCase			:			List[Any]        =					bytes_to_unicode()
								_lowerCAmelCase			:			Union[str, Any]        =					{v: k for k, v in self.byte_encoder.items()}
								with open(_UpperCAmelCase						,		encoding="""utf-8"""					) as merges_handle:
												_lowerCAmelCase			:			Dict        =					merges_handle.read().split("""\n"""					)[1:-1]
								_lowerCAmelCase			:			Dict        =					[tuple(merge.split()					) for merge in bpe_merges]
								_lowerCAmelCase			:			List[str]        =					dict(zip(_UpperCAmelCase						,		range(len(_UpperCAmelCase					)					)					)					)
								_lowerCAmelCase			:			Tuple        =					{}
								_lowerCAmelCase			:			Optional[int]        =					add_prefix_space

								# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
								_lowerCAmelCase			:			Union[str, Any]        =					re.compile(R"""\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""					)



				@property
				def 					SCREAMING_SNAKE_CASE				(					self					:						List[str]					)     ->			List[str]:





								'''simple docstring'''




								return len(self.encoder					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Dict					)     ->			Optional[int]:





								'''simple docstring'''




								return dict(self.encoder						,		**self.added_tokens_encoder					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Tuple						,		_UpperCAmelCase					:						Union[str, Any]					)     ->			List[str]:





								'''simple docstring'''




								if token in self.cache:
												return self.cache[token]
								_lowerCAmelCase			:			Optional[int]        =					tuple(_UpperCAmelCase					)
								_lowerCAmelCase			:			List[Any]        =					get_pairs(_UpperCAmelCase					)

								if not pairs:
												return token

								while True:
												_lowerCAmelCase			:			Union[str, Any]        =					min(_UpperCAmelCase						,		key=lambda _UpperCAmelCase					: self.bpe_ranks.get(_UpperCAmelCase						,		float("""inf"""					)					)					)
												if bigram not in self.bpe_ranks:
																break
												_lowerCAmelCase    ,			_lowerCAmelCase			:			int        =					bigram
												_lowerCAmelCase			:			List[str]        =					[]
												_lowerCAmelCase			:			List[str]        =					0
												while i < len(_UpperCAmelCase					):
																try:
																				_lowerCAmelCase			:			Optional[Any]        =					word.index(_UpperCAmelCase						,		_UpperCAmelCase					)
																except ValueError:
																				new_word.extend(word[i:]					)
																				break
																else:
																				new_word.extend(word[i:j]					)
																				_lowerCAmelCase			:			List[str]        =					j

																if word[i] == first and i < len(_UpperCAmelCase					) - 1 and word[i + 1] == second:
																				new_word.append(first + second					)
																				i += 2
																else:
																				new_word.append(word[i]					)
																				i += 1
												_lowerCAmelCase			:			Optional[int]        =					tuple(_UpperCAmelCase					)
												_lowerCAmelCase			:			int        =					new_word
												if len(_UpperCAmelCase					) == 1:
																break
												else:
																_lowerCAmelCase			:			List[Any]        =					get_pairs(_UpperCAmelCase					)
								_lowerCAmelCase			:			Tuple        =					""" """.join(_UpperCAmelCase					)
								_lowerCAmelCase			:			List[str]        =					word
								return word



				def 					SCREAMING_SNAKE_CASE				(					self					:						List[str]						,		_UpperCAmelCase					:						Any					)     ->			Dict:





								'''simple docstring'''




								_lowerCAmelCase			:			Any        =					[]
								for token in re.findall(self.pat						,		_UpperCAmelCase					):
												_lowerCAmelCase			:			Optional[int]        =					"""""".join(
												    self.byte_encoder[b] for b in token.encode("""utf-8"""					)					)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
												bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_UpperCAmelCase					).split(""" """					)					)
								return bpe_tokens



				def 					SCREAMING_SNAKE_CASE				(					self					:						Optional[int]						,		_UpperCAmelCase					:						Union[str, Any]					)     ->			Optional[Any]:





								'''simple docstring'''




								return self.encoder.get(_UpperCAmelCase						,		self.encoder.get(self.unk_token					)					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Optional[Any]						,		_UpperCAmelCase					:						List[str]					)     ->			Any:





								'''simple docstring'''




								return self.decoder.get(_UpperCAmelCase					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						str						,		_UpperCAmelCase					:						Optional[int]					)     ->			Union[str, Any]:





								'''simple docstring'''




								_lowerCAmelCase			:			Union[str, Any]        =					"""""".join(_UpperCAmelCase					)
								_lowerCAmelCase			:			List[Any]        =					bytearray([self.byte_decoder[c] for c in text]					).decode("""utf-8"""						,		errors=self.errors					)
								return text



				def 					SCREAMING_SNAKE_CASE				(					self					:						str						,		_UpperCAmelCase					:						str						,		_UpperCAmelCase					:						Optional[str] = None					)     ->			int:





								'''simple docstring'''




								if not os.path.isdir(_UpperCAmelCase					):
												logger.error(f"Vocabulary path ({save_directory}) should be a directory"					)
												return
								_lowerCAmelCase			:			Optional[Any]        =					os.path.join(
								    _UpperCAmelCase						,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)
								_lowerCAmelCase			:			Any        =					os.path.join(
								    _UpperCAmelCase						,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]					)

								with open(_UpperCAmelCase						,		"""w"""						,		encoding="""utf-8"""					) as f:
												f.write(json.dumps(self.encoder						,		indent=2						,		sort_keys=_UpperCAmelCase						,		ensure_ascii=_UpperCAmelCase					) + """\n"""					)

								_lowerCAmelCase			:			Tuple        =					0
								with open(_UpperCAmelCase						,		"""w"""						,		encoding="""utf-8"""					) as writer:
												writer.write("""#version: 0.2\n"""					)
												for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,		key=lambda _UpperCAmelCase					: kv[1]					):
																if index != token_index:
																				logger.warning(
																				    f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
																				    """ Please check that the tokenizer is not corrupted!"""					)
																				_lowerCAmelCase			:			Optional[Any]        =					token_index
																writer.write(""" """.join(_UpperCAmelCase					) + """\n"""					)
																index += 1

								return vocab_file, merge_file



				def 					SCREAMING_SNAKE_CASE				(					self					:						Union[str, Any]						,		_UpperCAmelCase					:						List[int]						,		_UpperCAmelCase					:						Optional[List[int]] = None					)     ->			str:





								'''simple docstring'''




								if token_ids_a is None:
												return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								_lowerCAmelCase			:			Optional[Any]        =					[self.cls_token_id]
								_lowerCAmelCase			:			Tuple        =					[self.sep_token_id]
								return cls + token_ids_a + sep + sep + token_ids_a + sep



				def 					SCREAMING_SNAKE_CASE				(					self					:						Any						,		_UpperCAmelCase					:						List[int]						,		_UpperCAmelCase					:						Optional[List[int]] = None						,		_UpperCAmelCase					:						bool = False					)     ->			List[Any]:





								'''simple docstring'''




								if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=_UpperCAmelCase						,		token_ids_a=_UpperCAmelCase						,		already_has_special_tokens=_UpperCAmelCase					)

								if token_ids_a is None:
												return [1] + ([0] * len(_UpperCAmelCase					)) + [1]
								return [1] + ([0] * len(_UpperCAmelCase					)) + [1, 1] + ([0] * len(_UpperCAmelCase					)) + [1]



				def 					SCREAMING_SNAKE_CASE				(					self					:						int						,		_UpperCAmelCase					:						List[int]						,		_UpperCAmelCase					:						Optional[List[int]] = None					)     ->			Optional[Any]:





								'''simple docstring'''




								_lowerCAmelCase			:			Optional[int]        =					[self.sep_token_id]
								_lowerCAmelCase			:			Tuple        =					[self.cls_token_id]

								if token_ids_a is None:
												return len(cls + token_ids_a + sep					) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]



				def 					SCREAMING_SNAKE_CASE				(					self					:						int						,		_UpperCAmelCase					:						Union[str, Any]						,		_UpperCAmelCase					:						int=False						,		**_UpperCAmelCase					:						Any					)     ->			Any:





								'''simple docstring'''




								_lowerCAmelCase			:			List[str]        =					kwargs.pop("""add_prefix_space"""						,		self.add_prefix_space					)
								if (is_split_into_words or add_prefix_space) and (len(_UpperCAmelCase					) > 0 and not text[0].isspace()):
												_lowerCAmelCase			:			Dict        =					""" """ + text
								return (text, kwargs)







import os
from shutil import copyfile
from typing import Optional, Tuple

from ...file_utils import is_sentencepiece_available
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if is_sentencepiece_available():
 from .tokenization_deberta_va import DebertaVaTokenizer
else:
 a_      						=   None

a_      						=   logging.get_logger(__name__)

a_      						=   {"vocab_file": "spm.model", "tokenizer_file": "tokenizer.json"}

a_      						=   {
    "vocab_file": {
        "microsoft/deberta-v2-xlarge": "https://huggingface.co/microsoft/deberta-v2-xlarge/resolve/main/spm.model",
        "microsoft/deberta-v2-xxlarge": "https://huggingface.co/microsoft/deberta-v2-xxlarge/resolve/main/spm.model",
        "microsoft/deberta-v2-xlarge-mnli": (
            "https://huggingface.co/microsoft/deberta-v2-xlarge-mnli/resolve/main/spm.model"
        ),
        "microsoft/deberta-v2-xxlarge-mnli": (
            "https://huggingface.co/microsoft/deberta-v2-xxlarge-mnli/resolve/main/spm.model"
        ),
    }
}

a_      						=   {
    "microsoft/deberta-v2-xlarge": 512,
    "microsoft/deberta-v2-xxlarge": 512,
    "microsoft/deberta-v2-xlarge-mnli": 512,
    "microsoft/deberta-v2-xxlarge-mnli": 512,
}

a_      						=   {
    "microsoft/deberta-v2-xlarge": {"do_lower_case": False},
    "microsoft/deberta-v2-xxlarge": {"do_lower_case": False},
    "microsoft/deberta-v2-xlarge-mnli": {"do_lower_case": False},
    "microsoft/deberta-v2-xxlarge-mnli": {"do_lower_case": False},
}

class        _lowerCamelCase  (	_SCREAMING_SNAKE_CASE      ):

      """simple docstring"""

      lowerCAmelCase__	:       int			=    VOCAB_FILES_NAMES
      lowerCAmelCase__	:       Any			=    PRETRAINED_VOCAB_FILES_MAP
      lowerCAmelCase__	:       Optional[Any]			=    PRETRAINED_INIT_CONFIGURATION
      lowerCAmelCase__	:       int			=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowerCAmelCase__	:       Dict			=    DebertaVaTokenizer







      def __init__(   self    :       int       ,				snake_case    :       int=None       ,				snake_case    :       Optional[int]=None       ,				snake_case    :       Any=False       ,				snake_case    :       Optional[int]=False       ,				snake_case    :       Dict="[CLS]"       ,				snake_case    :       Any="[SEP]"       ,				snake_case    :       Union[str, Any]="[UNK]"       ,				snake_case    :       List[Any]="[SEP]"       ,				snake_case    :       Union[str, Any]="[PAD]"       ,				snake_case    :       Tuple="[CLS]"       ,				snake_case    :       Dict="[MASK]"       ,				**snake_case    :       int       ,				):
          super().__init__(
              snake_case       ,				tokenizer_file=snake_case       ,				do_lower_case=snake_case       ,				bos_token=snake_case       ,				eos_token=snake_case       ,				unk_token=snake_case       ,				sep_token=snake_case       ,				pad_token=snake_case       ,				cls_token=snake_case       ,				mask_token=snake_case       ,				split_by_punct=snake_case       ,				**snake_case       ,				)

          __UpperCamelCase     						=  do_lower_case
          __UpperCamelCase     						=  split_by_punct
          __UpperCamelCase     						=  vocab_file
          __UpperCamelCase     						=  False if not self.vocab_file else True







      def 					snake_case		(   self    :       Optional[int]       ,				snake_case    :       int       ,				snake_case    :       Tuple=None ):

          if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
          __UpperCamelCase     						=  [self.cls_token_id]
          __UpperCamelCase     						=  [self.sep_token_id]
          return cls + token_ids_a + sep + token_ids_a + sep







      def 					snake_case		(   self    :       Any       ,				snake_case    :       Union[str, Any]       ,				snake_case    :       Optional[Any]=None       ,				snake_case    :       Union[str, Any]=False ):

          if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=snake_case       ,				token_ids_a=snake_case       ,				already_has_special_tokens=snake_case )

          if token_ids_a is not None:
              return [1] + ([0] * len(snake_case )) + [1] + ([0] * len(snake_case )) + [1]
          return [1] + ([0] * len(snake_case )) + [1]







      def 					snake_case		(   self    :       Optional[int]       ,				snake_case    :       Optional[int]       ,				snake_case    :       Optional[Any]=None ):
          __UpperCamelCase     						=  [self.sep_token_id]
          __UpperCamelCase     						=  [self.cls_token_id]
          if token_ids_a is None:
              return len(cls + token_ids_a + sep ) * [0]
          return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]







      def 					snake_case		(   self    :       List[str]       ,				snake_case    :       str       ,				snake_case    :       Optional[str] = None ):
          if not self.can_save_slow_tokenizer:
              raise ValueError(
                  '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
                  '''tokenizer.''' )

          if not os.path.isdir(snake_case ):
              logger.error(F"Vocabulary path ({save_directory}) should be a directory" )
              return
          __UpperCamelCase     						=  os.path.join(
              snake_case       ,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )

          if os.path.abspath(self.vocab_file ) != os.path.abspath(snake_case ):
              copyfile(self.vocab_file       ,				snake_case )

          return (out_vocab_file,)






