





import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation

UpperCAmelCase__  :	List[str]     =logging.get_logger(__name__)

UpperCAmelCase__  :	Any     ={'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

UpperCAmelCase__  :	Dict     ={
    '''vocab_file''': {
        '''microsoft/deberta-base''': '''https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json''',
        '''microsoft/deberta-large''': '''https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json''',
        '''microsoft/deberta-xlarge''': '''https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json''',
        '''microsoft/deberta-base-mnli''': '''https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json''',
        '''microsoft/deberta-large-mnli''': '''https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json''',
        '''microsoft/deberta-xlarge-mnli''': (
            '''https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''microsoft/deberta-base''': '''https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt''',
        '''microsoft/deberta-large''': '''https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt''',
        '''microsoft/deberta-xlarge''': '''https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt''',
        '''microsoft/deberta-base-mnli''': '''https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt''',
        '''microsoft/deberta-large-mnli''': '''https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt''',
        '''microsoft/deberta-xlarge-mnli''': (
            '''https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt'''
        ),
    },
}

UpperCAmelCase__  :	Optional[int]     ={
    '''microsoft/deberta-base''': 5_12,
    '''microsoft/deberta-large''': 5_12,
    '''microsoft/deberta-xlarge''': 5_12,
    '''microsoft/deberta-base-mnli''': 5_12,
    '''microsoft/deberta-large-mnli''': 5_12,
    '''microsoft/deberta-xlarge-mnli''': 5_12,
}

UpperCAmelCase__  :	List[Any]     ={
    '''microsoft/deberta-base''': {'''do_lower_case''': False},
    '''microsoft/deberta-large''': {'''do_lower_case''': False},
}





def    _lowercase    (  )  ->     Optional[Any]:
				lowerCamelCase		       =(
				    list(range(ord("""!""" )	,					ord("""~""" ) + 1 ) ) + list(range(ord("""¡""" )	,					ord("""¬""" ) + 1 ) ) + list(range(ord("""®""" )	,					ord("""ÿ""" ) + 1 ) )
				)
				lowerCamelCase		       =bs[:]
				lowerCamelCase		       =0
				for b in range(2**8 ):
								if b not in bs:
												bs.append(_UpperCAmelCase )
												cs.append(2**8 + n )
												n += 1
				lowerCamelCase		       =[chr(_UpperCAmelCase ) for n in cs]
				return dict(zip(_UpperCAmelCase	,					_UpperCAmelCase ) )





def    _lowercase    (  _UpperCAmelCase )  ->     Union[str, Any]:
				lowerCamelCase		       =set()
				lowerCamelCase		       =word[0]
				for char in word[1:]:
								pairs.add((prev_char, char) )
								lowerCamelCase		       =char
				return pairs




class 					__A   (  a			):
				__A            = VOCAB_FILES_NAMES
				__A            = PRETRAINED_VOCAB_FILES_MAP
				__A            = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__A            = ["""input_ids""", """attention_mask""", """token_type_ids"""]


				def __init__(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_      ,       UpperCAmelCase_="replace"      ,       UpperCAmelCase_="[CLS]"      ,       UpperCAmelCase_="[SEP]"      ,       UpperCAmelCase_="[SEP]"      ,       UpperCAmelCase_="[CLS]"      ,       UpperCAmelCase_="[UNK]"      ,       UpperCAmelCase_="[PAD]"      ,       UpperCAmelCase_="[MASK]"      ,       UpperCAmelCase_=False      ,       UpperCAmelCase_=False      ,       **UpperCAmelCase_      ,       ):
								lowerCamelCase		       =AddedToken(UpperCAmelCase_      ,       lstrip=UpperCAmelCase_      ,       rstrip=UpperCAmelCase_							) if isinstance(UpperCAmelCase_      ,       UpperCAmelCase_							) else bos_token
								lowerCamelCase		       =AddedToken(UpperCAmelCase_      ,       lstrip=UpperCAmelCase_      ,       rstrip=UpperCAmelCase_							) if isinstance(UpperCAmelCase_      ,       UpperCAmelCase_							) else eos_token
								lowerCamelCase		       =AddedToken(UpperCAmelCase_      ,       lstrip=UpperCAmelCase_      ,       rstrip=UpperCAmelCase_							) if isinstance(UpperCAmelCase_      ,       UpperCAmelCase_							) else sep_token
								lowerCamelCase		       =AddedToken(UpperCAmelCase_      ,       lstrip=UpperCAmelCase_      ,       rstrip=UpperCAmelCase_							) if isinstance(UpperCAmelCase_      ,       UpperCAmelCase_							) else cls_token
								lowerCamelCase		       =AddedToken(UpperCAmelCase_      ,       lstrip=UpperCAmelCase_      ,       rstrip=UpperCAmelCase_							) if isinstance(UpperCAmelCase_      ,       UpperCAmelCase_							) else unk_token
								lowerCamelCase		       =AddedToken(UpperCAmelCase_      ,       lstrip=UpperCAmelCase_      ,       rstrip=UpperCAmelCase_							) if isinstance(UpperCAmelCase_      ,       UpperCAmelCase_							) else pad_token

								# Mask token behave like a normal word, i.e. include the space before it
								lowerCamelCase		       =AddedToken(UpperCAmelCase_      ,       lstrip=UpperCAmelCase_      ,       rstrip=UpperCAmelCase_							) if isinstance(UpperCAmelCase_      ,       UpperCAmelCase_							) else mask_token

								super().__init__(
								    errors=UpperCAmelCase_      ,       bos_token=UpperCAmelCase_      ,       eos_token=UpperCAmelCase_      ,       unk_token=UpperCAmelCase_      ,       sep_token=UpperCAmelCase_      ,       cls_token=UpperCAmelCase_      ,       pad_token=UpperCAmelCase_      ,       mask_token=UpperCAmelCase_      ,       add_prefix_space=UpperCAmelCase_      ,       add_bos_token=UpperCAmelCase_      ,       **UpperCAmelCase_      ,       )
								lowerCamelCase		       =add_bos_token

								with open(UpperCAmelCase_      ,       encoding="""utf-8"""							) as vocab_handle:
												lowerCamelCase		       =json.load(UpperCAmelCase_							)
								lowerCamelCase		       ={v: k for k, v in self.encoder.items()}
								lowerCamelCase		       =errors  # how to handle errors in decoding
								lowerCamelCase		       =bytes_to_unicode()
								lowerCamelCase		       ={v: k for k, v in self.byte_encoder.items()}
								with open(UpperCAmelCase_      ,       encoding="""utf-8"""							) as merges_handle:
												lowerCamelCase		       =merges_handle.read().split("""\n"""							)[1:-1]
								lowerCamelCase		       =[tuple(merge.split()							) for merge in bpe_merges]
								lowerCamelCase		       =dict(zip(UpperCAmelCase_      ,       range(len(UpperCAmelCase_							)							)							)							)
								lowerCamelCase		       ={}
								lowerCamelCase		       =add_prefix_space

								# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
								lowerCamelCase		       =re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""							)


				@property
				# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
				def _snake_case				(						self							):
								return len(self.encoder							)


				def _snake_case				(						self							):
								return dict(self.encoder      ,       **self.added_tokens_encoder							)


				def _snake_case				(						self      ,       UpperCAmelCase_							):
								if token in self.cache:
												return self.cache[token]
								lowerCamelCase		       =tuple(UpperCAmelCase_							)
								lowerCamelCase		       =get_pairs(UpperCAmelCase_							)

								if not pairs:
												return token

								while True:
												lowerCamelCase		       =min(UpperCAmelCase_      ,       key=lambda UpperCAmelCase_							: self.bpe_ranks.get(UpperCAmelCase_      ,       float("""inf"""							)							)							)
												if bigram not in self.bpe_ranks:
																break
												lowerCamelCase       ,     lowerCamelCase		       =bigram
												lowerCamelCase		       =[]
												lowerCamelCase		       =0
												while i < len(UpperCAmelCase_							):
																try:
																				lowerCamelCase		       =word.index(UpperCAmelCase_      ,       UpperCAmelCase_							)
																except ValueError:
																				new_word.extend(word[i:]							)
																				break
																else:
																				new_word.extend(word[i:j]							)
																				lowerCamelCase		       =j

																if word[i] == first and i < len(UpperCAmelCase_							) - 1 and word[i + 1] == second:
																				new_word.append(first + second							)
																				i += 2
																else:
																				new_word.append(word[i]							)
																				i += 1
												lowerCamelCase		       =tuple(UpperCAmelCase_							)
												lowerCamelCase		       =new_word
												if len(UpperCAmelCase_							) == 1:
																break
												else:
																lowerCamelCase		       =get_pairs(UpperCAmelCase_							)
								lowerCamelCase		       =""" """.join(UpperCAmelCase_							)
								lowerCamelCase		       =word
								return word


				def _snake_case				(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_ = None							):
								if token_ids_a is None:
												return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								lowerCamelCase		       =[self.cls_token_id]
								lowerCamelCase		       =[self.sep_token_id]
								return cls + token_ids_a + sep + token_ids_a + sep


				def _snake_case				(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_ = None      ,       UpperCAmelCase_ = False							):
								if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=UpperCAmelCase_      ,       token_ids_a=UpperCAmelCase_      ,       already_has_special_tokens=UpperCAmelCase_							)

								if token_ids_a is None:
												return [1] + ([0] * len(UpperCAmelCase_							)) + [1]
								return [1] + ([0] * len(UpperCAmelCase_							)) + [1] + ([0] * len(UpperCAmelCase_							)) + [1]


				def _snake_case				(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_ = None							):
								lowerCamelCase		       =[self.sep_token_id]
								lowerCamelCase		       =[self.cls_token_id]

								if token_ids_a is None:
												return len(cls + token_ids_a + sep							) * [0]
								return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]


				def _snake_case				(						self      ,       UpperCAmelCase_							):
								lowerCamelCase		       =[]
								for token in re.findall(self.pat      ,       UpperCAmelCase_							):
												lowerCamelCase		       ="""""".join(
												    self.byte_encoder[b] for b in token.encode("""utf-8"""							)							)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
												bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCAmelCase_							).split(""" """							)							)
								return bpe_tokens


				def _snake_case				(						self      ,       UpperCAmelCase_							):
								return self.encoder.get(UpperCAmelCase_      ,       self.encoder.get(self.unk_token							)							)


				def _snake_case				(						self      ,       UpperCAmelCase_							):
								return self.decoder.get(UpperCAmelCase_							)


				def _snake_case				(						self      ,       UpperCAmelCase_							):
								lowerCamelCase		       ="""""".join(UpperCAmelCase_							)
								lowerCamelCase		       =bytearray([self.byte_decoder[c] for c in text]							).decode("""utf-8"""      ,       errors=self.errors							)
								return text


				def _snake_case				(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_ = None							):
								if not os.path.isdir(UpperCAmelCase_							):
												logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
												return
								lowerCamelCase		       =os.path.join(
								    UpperCAmelCase_      ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)
								lowerCamelCase		       =os.path.join(
								    UpperCAmelCase_      ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]							)

								with open(UpperCAmelCase_      ,       """w"""      ,       encoding="""utf-8"""							) as f:
												f.write(json.dumps(self.encoder      ,       indent=2      ,       sort_keys=UpperCAmelCase_      ,       ensure_ascii=UpperCAmelCase_							) + """\n"""							)

								lowerCamelCase		       =0
								with open(UpperCAmelCase_      ,       """w"""      ,       encoding="""utf-8"""							) as writer:
												writer.write("""#version: 0.2\n"""							)
												for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,       key=lambda UpperCAmelCase_							: kv[1]							):
																if index != token_index:
																				logger.warning(
																				    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																				    """ Please check that the tokenizer is not corrupted!"""							)
																				lowerCamelCase		       =token_index
																writer.write(""" """.join(UpperCAmelCase_							) + """\n"""							)
																index += 1

								return vocab_file, merge_file


				def _snake_case				(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_=False      ,       **UpperCAmelCase_							):
								lowerCamelCase		       =kwargs.pop("""add_prefix_space"""      ,       self.add_prefix_space							)
								if (is_split_into_words or add_prefix_space) and (len(UpperCAmelCase_							) > 0 and not text[0].isspace()):
												lowerCamelCase		       =""" """ + text
								return (text, kwargs)


				def _snake_case				(						self      ,       UpperCAmelCase_							):
								lowerCamelCase		       =[]
								for is_user, text in conversation.iter_texts():
												input_ids.extend(self.encode(UpperCAmelCase_      ,       add_special_tokens=UpperCAmelCase_							) + [self.eos_token_id]							)
								if len(UpperCAmelCase_							) > self.model_max_length:
												lowerCamelCase		       =input_ids[-self.model_max_length :]
								return input_ids








import importlib.metadata
import subprocess
import sys





def 	snake_case_      (__A     :						bool = False     )			->	str:
   __lowerCAmelCase  :			Any					    =						False
   if "IPython" in sys.modules:
      __lowerCAmelCase  :			Optional[int]					    =						"""google.colab""" in str(sys.modules["""IPython"""].get_ipython()     )

   if in_colab:
      if upgrade:
         __lowerCAmelCase  :			int					    =						["""pip""", """install""", """-U""", """torch"""]
         subprocess.run(__A					,   check=__A     )
      # get the current version of torch
      __lowerCAmelCase  :			List[Any]					    =						importlib.metadata.version("""torch"""     )
      __lowerCAmelCase  :			List[Any]					    =						torch_version[: torch_version.rindex("""."""     )]
      __lowerCAmelCase  :			int					    =						f'''https://storage.googleapis.com/tpu-pytorch/wheels/colab/torch_xla-{torch_version_trunc}-cp37-cp37m-linux_x86_64.whl'''
      __lowerCAmelCase  :			List[Any]					    =						["""pip""", """install""", xla_wheel]
      subprocess.run(__A					,   check=__A     )
   else:
      raise RuntimeError("""`install_xla` utility works only on google colab."""     )


