





'''simple docstring'''







import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation

_a					: str      =    logging.get_logger(__name__)

_a					: List[Any]      =    {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

_a					: Dict      =    {
    """vocab_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"""
        ),
    },
}

_a					: List[str]      =    {
    """microsoft/deberta-base""": 5_1_2,
    """microsoft/deberta-large""": 5_1_2,
    """microsoft/deberta-xlarge""": 5_1_2,
    """microsoft/deberta-base-mnli""": 5_1_2,
    """microsoft/deberta-large-mnli""": 5_1_2,
    """microsoft/deberta-xlarge-mnli""": 5_1_2,
}

_a					: List[Any]      =    {
    """microsoft/deberta-base""": {"""do_lower_case""": False},
    """microsoft/deberta-large""": {"""do_lower_case""": False},
}





def    _lowerCAmelCase						(						)					->					Optional[int]:
			__lowerCAmelCase											=				(
			    list(range(ord("""!"""	)					,	ord("""~"""	) + 1	)	) + list(range(ord("""¡"""	)					,	ord("""¬"""	) + 1	)	) + list(range(ord("""®"""	)					,	ord("""ÿ"""	) + 1	)	)
			)
			__lowerCAmelCase											=				bs[:]
			__lowerCAmelCase											=				0
			for b in range(2**8	):
						if b not in bs:
									bs.append(lowercase	)
									cs.append(2**8 + n	)
									n += 1
			__lowerCAmelCase											=				[chr(lowercase	) for n in cs]
			return dict(zip(lowercase					,	lowercase	)	)





def    _lowerCAmelCase						(						lowercase	)					->					Dict:
			__lowerCAmelCase											=				set()
			__lowerCAmelCase											=				word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)	)
						__lowerCAmelCase											=				char
			return pairs







class     _UpperCAmelCase							(						lowerCAmelCase_   ):
		a   :   Tuple     =VOCAB_FILES_NAMES
		a   :   Dict     =PRETRAINED_VOCAB_FILES_MAP
		a   :   List[Any]     =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		a   :   Union[str, Any]     =["""input_ids""", """attention_mask""", """token_type_ids"""]
		def __init__(  self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE="replace",__SCREAMING_SNAKE_CASE="[CLS]",__SCREAMING_SNAKE_CASE="[SEP]",__SCREAMING_SNAKE_CASE="[SEP]",__SCREAMING_SNAKE_CASE="[CLS]",__SCREAMING_SNAKE_CASE="[UNK]",__SCREAMING_SNAKE_CASE="[PAD]",__SCREAMING_SNAKE_CASE="[MASK]",__SCREAMING_SNAKE_CASE=False,__SCREAMING_SNAKE_CASE=False,**__SCREAMING_SNAKE_CASE,):



					'''simple docstring'''

					__lowerCAmelCase											=				AddedToken(__SCREAMING_SNAKE_CASE,lstrip=__SCREAMING_SNAKE_CASE,rstrip=__SCREAMING_SNAKE_CASE			) if isinstance(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE			) else bos_token
					__lowerCAmelCase											=				AddedToken(__SCREAMING_SNAKE_CASE,lstrip=__SCREAMING_SNAKE_CASE,rstrip=__SCREAMING_SNAKE_CASE			) if isinstance(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE			) else eos_token
					__lowerCAmelCase											=				AddedToken(__SCREAMING_SNAKE_CASE,lstrip=__SCREAMING_SNAKE_CASE,rstrip=__SCREAMING_SNAKE_CASE			) if isinstance(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE			) else sep_token
					__lowerCAmelCase											=				AddedToken(__SCREAMING_SNAKE_CASE,lstrip=__SCREAMING_SNAKE_CASE,rstrip=__SCREAMING_SNAKE_CASE			) if isinstance(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE			) else cls_token
					__lowerCAmelCase											=				AddedToken(__SCREAMING_SNAKE_CASE,lstrip=__SCREAMING_SNAKE_CASE,rstrip=__SCREAMING_SNAKE_CASE			) if isinstance(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE			) else unk_token
					__lowerCAmelCase											=				AddedToken(__SCREAMING_SNAKE_CASE,lstrip=__SCREAMING_SNAKE_CASE,rstrip=__SCREAMING_SNAKE_CASE			) if isinstance(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE			) else pad_token

					# Mask token behave like a normal word, i.e. include the space before it
					__lowerCAmelCase											=				AddedToken(__SCREAMING_SNAKE_CASE,lstrip=__SCREAMING_SNAKE_CASE,rstrip=__SCREAMING_SNAKE_CASE			) if isinstance(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE			) else mask_token

					super().__init__(
					    errors=__SCREAMING_SNAKE_CASE,bos_token=__SCREAMING_SNAKE_CASE,eos_token=__SCREAMING_SNAKE_CASE,unk_token=__SCREAMING_SNAKE_CASE,sep_token=__SCREAMING_SNAKE_CASE,cls_token=__SCREAMING_SNAKE_CASE,pad_token=__SCREAMING_SNAKE_CASE,mask_token=__SCREAMING_SNAKE_CASE,add_prefix_space=__SCREAMING_SNAKE_CASE,add_bos_token=__SCREAMING_SNAKE_CASE,**__SCREAMING_SNAKE_CASE,)
					__lowerCAmelCase											=				add_bos_token

					with open(__SCREAMING_SNAKE_CASE,encoding="""utf-8"""			) as vocab_handle:
								__lowerCAmelCase											=				json.load(__SCREAMING_SNAKE_CASE			)
					__lowerCAmelCase											=				{v: k for k, v in self.encoder.items()}
					__lowerCAmelCase											=				errors  # how to handle errors in decoding
					__lowerCAmelCase											=				bytes_to_unicode()
					__lowerCAmelCase											=				{v: k for k, v in self.byte_encoder.items()}
					with open(__SCREAMING_SNAKE_CASE,encoding="""utf-8"""			) as merges_handle:
								__lowerCAmelCase											=				merges_handle.read().split("""\n"""			)[1:-1]
					__lowerCAmelCase											=				[tuple(merge.split()			) for merge in bpe_merges]
					__lowerCAmelCase											=				dict(zip(__SCREAMING_SNAKE_CASE,range(len(__SCREAMING_SNAKE_CASE			)			)			)			)
					__lowerCAmelCase											=				{}
					__lowerCAmelCase											=				add_prefix_space

					# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
					__lowerCAmelCase											=				re.compile(R"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""			)
		@property
		# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
		def   lowerCamelCase__			(  self			):



					'''simple docstring'''

					return len(self.encoder			)
		def   lowerCamelCase__			(  self			):



					'''simple docstring'''

					return dict(self.encoder,**self.added_tokens_encoder			)
		def   lowerCamelCase__			(  self,__SCREAMING_SNAKE_CASE			):



					'''simple docstring'''

					if token in self.cache:
								return self.cache[token]
					__lowerCAmelCase											=				tuple(__SCREAMING_SNAKE_CASE			)
					__lowerCAmelCase											=				get_pairs(__SCREAMING_SNAKE_CASE			)

					if not pairs:
								return token

					while True:
								__lowerCAmelCase											=				min(__SCREAMING_SNAKE_CASE,key=lambda __SCREAMING_SNAKE_CASE			: self.bpe_ranks.get(__SCREAMING_SNAKE_CASE,float("""inf"""			)			)			)
								if bigram not in self.bpe_ranks:
											break
								__lowerCAmelCase						,       __lowerCAmelCase											=				bigram
								__lowerCAmelCase											=				[]
								__lowerCAmelCase											=				0
								while i < len(__SCREAMING_SNAKE_CASE			):
											try:
														__lowerCAmelCase											=				word.index(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE			)
											except ValueError:
														new_word.extend(word[i:]			)
														break
											else:
														new_word.extend(word[i:j]			)
														__lowerCAmelCase											=				j

											if word[i] == first and i < len(__SCREAMING_SNAKE_CASE			) - 1 and word[i + 1] == second:
														new_word.append(first + second			)
														i += 2
											else:
														new_word.append(word[i]			)
														i += 1
								__lowerCAmelCase											=				tuple(__SCREAMING_SNAKE_CASE			)
								__lowerCAmelCase											=				new_word
								if len(__SCREAMING_SNAKE_CASE			) == 1:
											break
								else:
											__lowerCAmelCase											=				get_pairs(__SCREAMING_SNAKE_CASE			)
					__lowerCAmelCase											=				""" """.join(__SCREAMING_SNAKE_CASE			)
					__lowerCAmelCase											=				word
					return word
		def   lowerCamelCase__			(  self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE = None			):



					'''simple docstring'''

					if token_ids_a is None:
								return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
					__lowerCAmelCase											=				[self.cls_token_id]
					__lowerCAmelCase											=				[self.sep_token_id]
					return cls + token_ids_a + sep + token_ids_a + sep
		def   lowerCamelCase__			(  self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE = None,__SCREAMING_SNAKE_CASE = False			):



					'''simple docstring'''

					if already_has_special_tokens:
								return super().get_special_tokens_mask(
								    token_ids_a=__SCREAMING_SNAKE_CASE,token_ids_a=__SCREAMING_SNAKE_CASE,already_has_special_tokens=__SCREAMING_SNAKE_CASE			)

					if token_ids_a is None:
								return [1] + ([0] * len(__SCREAMING_SNAKE_CASE			)) + [1]
					return [1] + ([0] * len(__SCREAMING_SNAKE_CASE			)) + [1] + ([0] * len(__SCREAMING_SNAKE_CASE			)) + [1]
		def   lowerCamelCase__			(  self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE = None			):



					'''simple docstring'''

					__lowerCAmelCase											=				[self.sep_token_id]
					__lowerCAmelCase											=				[self.cls_token_id]

					if token_ids_a is None:
								return len(cls + token_ids_a + sep			) * [0]
					return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]
		def   lowerCamelCase__			(  self,__SCREAMING_SNAKE_CASE			):



					'''simple docstring'''

					__lowerCAmelCase											=				[]
					for token in re.findall(self.pat,__SCREAMING_SNAKE_CASE			):
								__lowerCAmelCase											=				"""""".join(
								    self.byte_encoder[b] for b in token.encode("""utf-8"""			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
								bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__SCREAMING_SNAKE_CASE			).split(""" """			)			)
					return bpe_tokens
		def   lowerCamelCase__			(  self,__SCREAMING_SNAKE_CASE			):



					'''simple docstring'''

					return self.encoder.get(__SCREAMING_SNAKE_CASE,self.encoder.get(self.unk_token			)			)
		def   lowerCamelCase__			(  self,__SCREAMING_SNAKE_CASE			):



					'''simple docstring'''

					return self.decoder.get(__SCREAMING_SNAKE_CASE			)
		def   lowerCamelCase__			(  self,__SCREAMING_SNAKE_CASE			):



					'''simple docstring'''

					__lowerCAmelCase											=				"""""".join(__SCREAMING_SNAKE_CASE			)
					__lowerCAmelCase											=				bytearray([self.byte_decoder[c] for c in text]			).decode("""utf-8""",errors=self.errors			)
					return text
		def   lowerCamelCase__			(  self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE = None			):



					'''simple docstring'''

					if not os.path.isdir(__SCREAMING_SNAKE_CASE			):
								logger.error(f'Vocabulary path ({save_directory}) should be a directory'			)
								return
					__lowerCAmelCase											=				os.path.join(
					    __SCREAMING_SNAKE_CASE,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)
					__lowerCAmelCase											=				os.path.join(
					    __SCREAMING_SNAKE_CASE,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]			)

					with open(__SCREAMING_SNAKE_CASE,"""w""",encoding="""utf-8"""			) as f:
								f.write(json.dumps(self.encoder,indent=2,sort_keys=__SCREAMING_SNAKE_CASE,ensure_ascii=__SCREAMING_SNAKE_CASE			) + """\n"""			)

					__lowerCAmelCase											=				0
					with open(__SCREAMING_SNAKE_CASE,"""w""",encoding="""utf-8"""			) as writer:
								writer.write("""#version: 0.2\n"""			)
								for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),key=lambda __SCREAMING_SNAKE_CASE			: kv[1]			):
											if index != token_index:
														logger.warning(
														    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
														    """ Please check that the tokenizer is not corrupted!"""			)
														__lowerCAmelCase											=				token_index
											writer.write(""" """.join(__SCREAMING_SNAKE_CASE			) + """\n"""			)
											index += 1

					return vocab_file, merge_file
		def   lowerCamelCase__			(  self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE=False,**__SCREAMING_SNAKE_CASE			):



					'''simple docstring'''

					__lowerCAmelCase											=				kwargs.pop("""add_prefix_space""",self.add_prefix_space			)
					if (is_split_into_words or add_prefix_space) and (len(__SCREAMING_SNAKE_CASE			) > 0 and not text[0].isspace()):
								__lowerCAmelCase											=				""" """ + text
					return (text, kwargs)







		def   lowerCamelCase__			(  self,__SCREAMING_SNAKE_CASE			):



					'''simple docstring'''

					__lowerCAmelCase											=				[]
					for is_user, text in conversation.iter_texts():
								input_ids.extend(self.encode(__SCREAMING_SNAKE_CASE,add_special_tokens=__SCREAMING_SNAKE_CASE			) + [self.eos_token_id]			)
					if len(__SCREAMING_SNAKE_CASE			) > self.model_max_length:
								__lowerCAmelCase											=				input_ids[-self.model_max_length :]
					return input_ids








'''simple docstring'''







# Lint as: python3

import hashlib
from typing import Union





def    _lowerCAmelCase						(						lowercase	)					->					bytes:
			if isinstance(lowercase					,	lowercase	):
						# Data already in bytes, returns as it as
						return hash_data
			elif isinstance(lowercase					,	lowercase	):
						# We keep the data as it as for it ot be later encoded to UTF-8
						# However replace `\\` with `/` for Windows compatibility
						__lowerCAmelCase											=				hash_data.replace("""\\"""					,	"""/"""	)
			elif isinstance(lowercase					,	lowercase	):
						__lowerCAmelCase											=				str(lowercase	)
			else:
						# If data is not of the required type, raise error
						raise InvalidKeyError(lowercase	)

			return hash_data.encode("""utf-8"""	)





class     _UpperCAmelCase							(						lowerCAmelCase_   ):
		def __init__(  self,__SCREAMING_SNAKE_CASE			):



					'''simple docstring'''

					__lowerCAmelCase											=				"""\nFAILURE TO GENERATE DATASET: Invalid key type detected"""
					__lowerCAmelCase											=				f'\nFound Key {hash_data} of type {type(__SCREAMING_SNAKE_CASE			)}'
					__lowerCAmelCase											=				"""\nKeys should be either str, int or bytes type"""
					super().__init__(f'{self.prefix}{self.err_msg}{self.suffix}'			)





class     _UpperCAmelCase							(						lowerCAmelCase_   ):
		def __init__(  self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE=""			):



					'''simple docstring'''

					__lowerCAmelCase											=				key
					__lowerCAmelCase											=				duplicate_key_indices
					__lowerCAmelCase											=				fix_msg
					__lowerCAmelCase											=				"""Found multiple examples generated with the same key"""
					if len(__SCREAMING_SNAKE_CASE			) <= 20:
								__lowerCAmelCase											=				f'\nThe examples at index {", ".join(__SCREAMING_SNAKE_CASE			)} have the key {key}'
					else:
								__lowerCAmelCase											=				f'\nThe examples at index {", ".join(duplicate_key_indices[:20]			)}... ({len(__SCREAMING_SNAKE_CASE			) - 20} more) have the key {key}'
					__lowerCAmelCase											=				"""\n""" + fix_msg if fix_msg else """"""
					super().__init__(f'{self.prefix}{self.err_msg}{self.suffix}'			)





class     _UpperCAmelCase							:
		def __init__(  self,__SCREAMING_SNAKE_CASE			):



					'''simple docstring'''

					__lowerCAmelCase											=				hashlib.mda(_as_bytes(__SCREAMING_SNAKE_CASE			)			)







		def   lowerCamelCase__			(  self,__SCREAMING_SNAKE_CASE			):



					'''simple docstring'''

					__lowerCAmelCase											=				self._split_mda.copy()
					__lowerCAmelCase											=				_as_bytes(__SCREAMING_SNAKE_CASE			)
					mda.update(__SCREAMING_SNAKE_CASE			)
					# Convert to integer with hexadecimal conversion
					return int(mda.hexdigest(),16			)

