


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__ 						=					logging.get_logger(__name__)
lowerCAmelCase__ 						=					{"vocab_file": "spiece.model"}

lowerCAmelCase__ 						=					{
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

lowerCAmelCase__ 						=					{
    "albert-base-v1": 5_12,
    "albert-large-v1": 5_12,
    "albert-xlarge-v1": 5_12,
    "albert-xxlarge-v1": 5_12,
    "albert-base-v2": 5_12,
    "albert-large-v2": 5_12,
    "albert-xlarge-v2": 5_12,
    "albert-xxlarge-v2": 5_12,
}

lowerCAmelCase__ 						=					"▁"







class 	lowerCAmelCase__			(							UpperCAmelCase__):




     '''simple docstring'''
     __SCREAMING_SNAKE_CASE         =					VOCAB_FILES_NAMES
     __SCREAMING_SNAKE_CASE         =					PRETRAINED_VOCAB_FILES_MAP
     __SCREAMING_SNAKE_CASE         =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES


     def __init__(		self					,		__lowerCamelCase					,		__lowerCamelCase=True					,		__lowerCamelCase=True					,		__lowerCamelCase=False					,		__lowerCamelCase="[CLS]"					,		__lowerCamelCase="[SEP]"					,		__lowerCamelCase="<unk>"					,		__lowerCamelCase="[SEP]"					,		__lowerCamelCase="<pad>"					,		__lowerCamelCase="[CLS]"					,		__lowerCamelCase="[MASK]"					,		__lowerCamelCase = None					,		**__lowerCamelCase					,		)     ->							str:
            _A :			Any							       =					(
                AddedToken(__lowercase					,		lstrip=__lowercase					,		rstrip=__lowercase					,		normalized=__lowercase)
                if isinstance(__lowercase					,		__lowercase)
                else mask_token
            )

            _A :			Union[str, Any]							       =					{} if sp_model_kwargs is None else sp_model_kwargs

            super().__init__(
                do_lower_case=__lowercase					,		remove_space=__lowercase					,		keep_accents=__lowercase					,		bos_token=__lowercase					,		eos_token=__lowercase					,		unk_token=__lowercase					,		sep_token=__lowercase					,		pad_token=__lowercase					,		cls_token=__lowercase					,		mask_token=__lowercase					,		sp_model_kwargs=self.sp_model_kwargs					,		**__lowercase					,		)

            _A :			Union[str, Any]							       =					do_lower_case
            _A :			Union[str, Any]							       =					remove_space
            _A :			Tuple							       =					keep_accents
            _A :			List[Any]							       =					vocab_file

            _A :			Optional[Any]							       =					spm.SentencePieceProcessor(**self.sp_model_kwargs)
            self.sp_model.Load(__lowercase)


     @property
     def 					_lowerCamelCase					(		self)     ->							int:
            return len(self.sp_model)


     def 					_lowerCamelCase					(		self)     ->							List[str]:
            _A :			List[Any]							       =					{self.convert_ids_to_tokens(__lowercase): i for i in range(self.vocab_size)}
            vocab.update(self.added_tokens_encoder)
            return vocab


     def __getstate__(		self)     ->							Optional[Any]:
            _A :			str							       =					self.__dict__.copy()
            _A :			Union[str, Any]							       =					None
            return state


     def __setstate__(		self					,		__lowerCamelCase)     ->							int:
            _A :			int							       =					d

            # for backward compatibility
            if not hasattr(self					,		"sp_model_kwargs"):
                   _A :			str							       =					{}

            _A :			Dict							       =					spm.SentencePieceProcessor(**self.sp_model_kwargs)
            self.sp_model.Load(self.vocab_file)


     def 					_lowerCamelCase					(		self					,		__lowerCamelCase)     ->							Dict:






            if self.remove_space:
                   _A :			int							       =					" ".join(inputs.strip().split())
            else:
                   _A :			List[str]							       =					inputs
            _A :			List[str]							       =					outputs.replace("``"					,		"\"").replace("''"					,		"\"")

            if not self.keep_accents:
                   _A :			int							       =					unicodedata.normalize("NFKD"					,		__lowercase)
                   _A :			Union[str, Any]							       =					"".join([c for c in outputs if not unicodedata.combining(__lowercase)])
            if self.do_lower_case:
                   _A :			Any							       =					outputs.lower()

            return outputs


     def 					_lowerCamelCase					(		self					,		__lowerCamelCase)     ->							Dict:
            _A :			Tuple							       =					self.preprocess_text(__lowercase)
            _A :			str							       =					self.sp_model.encode(__lowercase					,		out_type=__lowercase)
            _A :			Optional[int]							       =					[]
            for piece in pieces:
                   if len(__lowercase) > 1 and piece[-1] == str(",") and piece[-2].isdigit():
                          _A :			Dict							       =					self.sp_model.EncodeAsPieces(piece[:-1].replace(__lowercase					,		""))
                          if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                                 if len(cur_pieces[0]) == 1:
                                        _A :			Any							       =					cur_pieces[1:]
                                 else:
                                        _A :			Any							       =					cur_pieces[0][1:]
                          cur_pieces.append(piece[-1])
                          new_pieces.extend(__lowercase)
                   else:
                          new_pieces.append(__lowercase)

            return new_pieces


     def 					_lowerCamelCase					(		self					,		__lowerCamelCase)     ->							Tuple:
            return self.sp_model.PieceToId(__lowercase)


     def 					_lowerCamelCase					(		self					,		__lowerCamelCase)     ->							Tuple:
            return self.sp_model.IdToPiece(__lowercase)


     def 					_lowerCamelCase					(		self					,		__lowerCamelCase)     ->							Dict:
            _A :			int							       =					[]
            _A :			List[Any]							       =					""
            _A :			Any							       =					False
            for token in tokens:
                   # make sure that special tokens are not decoded using sentencepiece model
                   if token in self.all_special_tokens:
                          if not prev_is_special:
                                 out_string += " "
                          out_string += self.sp_model.decode(__lowercase) + token
                          _A :			Union[str, Any]							       =					True
                          _A :			Tuple							       =					[]
                   else:
                          current_sub_tokens.append(__lowercase)
                          _A :			Optional[Any]							       =					False
            out_string += self.sp_model.decode(__lowercase)
            return out_string.strip()


     def 					_lowerCamelCase					(		self					,		__lowerCamelCase					,		__lowerCamelCase = None)     ->							Optional[int]:
            _A :			Tuple							       =					[self.sep_token_id]
            _A :			List[str]							       =					[self.cls_token_id]
            if token_ids_a is None:
                   return cls + token_ids_a + sep
            return cls + token_ids_a + sep + token_ids_a + sep


     def 					_lowerCamelCase					(		self					,		__lowerCamelCase					,		__lowerCamelCase = None					,		__lowerCamelCase = False)     ->							str:






            if already_has_special_tokens:
                   return super().get_special_tokens_mask(
                       token_ids_a=__lowercase					,		token_ids_a=__lowercase					,		already_has_special_tokens=__lowercase)

            if token_ids_a is not None:
                   return [1] + ([0] * len(__lowercase)) + [1] + ([0] * len(__lowercase)) + [1]
            return [1] + ([0] * len(__lowercase)) + [1]


     def 					_lowerCamelCase					(		self					,		__lowerCamelCase					,		__lowerCamelCase = None)     ->							List[Any]:
            _A :			Optional[int]							       =					[self.sep_token_id]
            _A :			Optional[Any]							       =					[self.cls_token_id]

            if token_ids_a is None:
                   return len(cls + token_ids_a + sep) * [0]
            return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]


     def 					_lowerCamelCase					(		self					,		__lowerCamelCase					,		__lowerCamelCase = None)     ->							List[str]:






            if not os.path.isdir(__lowercase):
                   logger.error(F"Vocabulary path ({save_directory}) should be a directory")
                   return
            _A :			Optional[int]							       =					os.path.join(
                __lowercase					,		(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])

            if os.path.abspath(self.vocab_file) != os.path.abspath(__lowercase) and os.path.isfile(self.vocab_file):
                   copyfile(self.vocab_file					,		__lowercase)
            elif not os.path.isfile(self.vocab_file):
                   with open(__lowercase					,		"wb") as fi:
                          _A :			Optional[int]							       =					self.sp_model.serialized_model_proto()
                          fi.write(__lowercase)

            return (out_vocab_file,)






import unittest

from transformers import DebertaConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    DebertaForMaskedLM,
		    DebertaForQuestionAnswering,
		    DebertaForSequenceClassification,
		    DebertaForTokenClassification,
		    DebertaModel,
		)
		from transformers.models.deberta.modeling_deberta import DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST
class 						UpperCAmelCase  (							UpperCAmelCase__     ):







						'''simple docstring'''





						def __init__(		self				:	Dict			,      __lowercase				:	Any			,      __lowercase				:	str=13			,      __lowercase				:	Optional[Any]=7			,      __lowercase				:	Tuple=True			,      __lowercase				:	Optional[int]=True			,      __lowercase				:	List[str]=True			,      __lowercase				:	Dict=True			,      __lowercase				:	Any=99			,      __lowercase				:	Any=32			,      __lowercase				:	List[Any]=5			,      __lowercase				:	Any=4			,      __lowercase				:	List[str]=37			,      __lowercase				:	Union[str, Any]="gelu"			,      __lowercase				:	Union[str, Any]=0.1			,      __lowercase				:	str=0.1			,      __lowercase				:	List[str]=5_12			,      __lowercase				:	Tuple=16			,      __lowercase				:	List[str]=2			,      __lowercase				:	Tuple=0.02			,      __lowercase				:	Optional[Any]=False			,      __lowercase				:	Optional[int]=True			,      __lowercase				:	Union[str, Any]="None"			,      __lowercase				:	Union[str, Any]=3			,      __lowercase				:	int=4			,      __lowercase				:	Tuple=None			,      ):






							"""simple docstring"""






							snake_case_						       =	parent
							snake_case_						       =	batch_size
							snake_case_						       =	seq_length
							snake_case_						       =	is_training
							snake_case_						       =	use_input_mask
							snake_case_						       =	use_token_type_ids
							snake_case_						       =	use_labels
							snake_case_						       =	vocab_size
							snake_case_						       =	hidden_size
							snake_case_						       =	num_hidden_layers
							snake_case_						       =	num_attention_heads
							snake_case_						       =	intermediate_size
							snake_case_						       =	hidden_act
							snake_case_						       =	hidden_dropout_prob
							snake_case_						       =	attention_probs_dropout_prob
							snake_case_						       =	max_position_embeddings
							snake_case_						       =	type_vocab_size
							snake_case_						       =	type_sequence_label_size
							snake_case_						       =	initializer_range
							snake_case_						       =	num_labels
							snake_case_						       =	num_choices
							snake_case_						       =	relative_attention
							snake_case_						       =	position_biased_input
							snake_case_						       =	pos_att_type
							snake_case_						       =	scope






						def 			snake_case__		(		self				:	str		):






							"""simple docstring"""






							snake_case_						       =	ids_tensor([self.batch_size, self.seq_length]			,      self.vocab_size		)

							snake_case_						       =	None
							if self.use_input_mask:
								snake_case_						       =	ids_tensor([self.batch_size, self.seq_length]			,      vocab_size=2		)

							snake_case_						       =	None
							if self.use_token_type_ids:
								snake_case_						       =	ids_tensor([self.batch_size, self.seq_length]			,      self.type_vocab_size		)

							snake_case_						       =	None
							snake_case_						       =	None
							snake_case_						       =	None
							if self.use_labels:
								snake_case_						       =	ids_tensor([self.batch_size]			,      self.type_sequence_label_size		)
								snake_case_						       =	ids_tensor([self.batch_size, self.seq_length]			,      self.num_labels		)
								snake_case_						       =	ids_tensor([self.batch_size]			,      self.num_choices		)

							snake_case_						       =	self.get_config()

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






						def 			snake_case__		(		self				:	Optional[int]		):






							"""simple docstring"""






							return DebertaConfig(
							    vocab_size=self.vocab_size			,      hidden_size=self.hidden_size			,      num_hidden_layers=self.num_hidden_layers			,      num_attention_heads=self.num_attention_heads			,      intermediate_size=self.intermediate_size			,      hidden_act=self.hidden_act			,      hidden_dropout_prob=self.hidden_dropout_prob			,      attention_probs_dropout_prob=self.attention_probs_dropout_prob			,      max_position_embeddings=self.max_position_embeddings			,      type_vocab_size=self.type_vocab_size			,      initializer_range=self.initializer_range			,      relative_attention=self.relative_attention			,      position_biased_input=self.position_biased_input			,      pos_att_type=self.pos_att_type			,      )






						def 			snake_case__		(		self				:	int		):






							"""simple docstring"""






							snake_case_						       =	self.get_config()
							snake_case_						       =	3_00
							return config






						def 			snake_case__		(		self				:	List[Any]			,      __lowercase				:	List[Any]		):






							"""simple docstring"""






							self.parent.assertListEqual(list(result.loss.size()		)			,      []		)






						def 			snake_case__		(		self				:	List[str]			,      __lowercase				:	Tuple			,      __lowercase				:	Union[str, Any]			,      __lowercase				:	Tuple			,      __lowercase				:	Dict			,      __lowercase				:	str			,      __lowercase				:	Optional[int]			,      __lowercase				:	int		):






							"""simple docstring"""






							snake_case_						       =	DebertaModel(config=__lowercase		)
							model.to(__lowercase		)
							model.eval()
							snake_case_						       =	model(__lowercase			,      attention_mask=__lowercase			,      token_type_ids=__lowercase		)[0]
							snake_case_						       =	model(__lowercase			,      token_type_ids=__lowercase		)[0]
							snake_case_						       =	model(__lowercase		)[0]

							self.parent.assertListEqual(list(sequence_output.size()		)			,      [self.batch_size, self.seq_length, self.hidden_size]		)






						def 			snake_case__		(		self				:	Any			,      __lowercase				:	Any			,      __lowercase				:	Dict			,      __lowercase				:	Any			,      __lowercase				:	Any			,      __lowercase				:	Tuple			,      __lowercase				:	Union[str, Any]			,      __lowercase				:	List[str]		):






							"""simple docstring"""






							snake_case_						       =	DebertaForMaskedLM(config=__lowercase		)
							model.to(__lowercase		)
							model.eval()
							snake_case_						       =	model(__lowercase			,      attention_mask=__lowercase			,      token_type_ids=__lowercase			,      labels=__lowercase		)

							self.parent.assertEqual(result.logits.shape			,      (self.batch_size, self.seq_length, self.vocab_size)		)






						def 			snake_case__		(		self				:	List[str]			,      __lowercase				:	Tuple			,      __lowercase				:	List[str]			,      __lowercase				:	Dict			,      __lowercase				:	str			,      __lowercase				:	Optional[Any]			,      __lowercase				:	Optional[Any]			,      __lowercase				:	Union[str, Any]		):






							"""simple docstring"""






							snake_case_						       =	self.num_labels
							snake_case_						       =	DebertaForSequenceClassification(__lowercase		)
							model.to(__lowercase		)
							model.eval()
							snake_case_						       =	model(__lowercase			,      attention_mask=__lowercase			,      token_type_ids=__lowercase			,      labels=__lowercase		)
							self.parent.assertListEqual(list(result.logits.size()		)			,      [self.batch_size, self.num_labels]		)
							self.check_loss_output(__lowercase		)






						def 			snake_case__		(		self				:	List[Any]			,      __lowercase				:	List[str]			,      __lowercase				:	Any			,      __lowercase				:	List[Any]			,      __lowercase				:	str			,      __lowercase				:	List[str]			,      __lowercase				:	Optional[Any]			,      __lowercase				:	Optional[int]		):






							"""simple docstring"""






							snake_case_						       =	self.num_labels
							snake_case_						       =	DebertaForTokenClassification(config=__lowercase		)
							model.to(__lowercase		)
							model.eval()
							snake_case_						       =	model(__lowercase			,      attention_mask=__lowercase			,      token_type_ids=__lowercase			,      labels=__lowercase		)
							self.parent.assertEqual(result.logits.shape			,      (self.batch_size, self.seq_length, self.num_labels)		)






						def 			snake_case__		(		self				:	List[str]			,      __lowercase				:	str			,      __lowercase				:	List[str]			,      __lowercase				:	Optional[int]			,      __lowercase				:	Optional[Any]			,      __lowercase				:	Optional[Any]			,      __lowercase				:	Optional[Any]			,      __lowercase				:	Union[str, Any]		):






							"""simple docstring"""






							snake_case_						       =	DebertaForQuestionAnswering(config=__lowercase		)
							model.to(__lowercase		)
							model.eval()
							snake_case_						       =	model(
							    __lowercase			,      attention_mask=__lowercase			,      token_type_ids=__lowercase			,      start_positions=__lowercase			,      end_positions=__lowercase			,      )
							self.parent.assertEqual(result.start_logits.shape			,      (self.batch_size, self.seq_length)		)
							self.parent.assertEqual(result.end_logits.shape			,      (self.batch_size, self.seq_length)		)






						def 			snake_case__		(		self				:	Tuple		):






							"""simple docstring"""






							snake_case_						       =	self.prepare_config_and_inputs()
							(
							    (
							    snake_case_
							)       , (
							    snake_case_
							)       , (
							    snake_case_
							)       , (
							    snake_case_
							)       , (
							    snake_case_
							)       , (
							    snake_case_
							)       , (
							    snake_case_
							)       , 
							)						       =	config_and_inputs
							snake_case_						       =	{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
							return config, inputs_dict







@require_torch
class 						UpperCAmelCase  (							UpperCAmelCase__			,  UpperCAmelCase__			,  unittest.TestCase     ):







						'''simple docstring'''





						lowerCAmelCase_							=  (
						    (
						        DebertaModel,
						        DebertaForMaskedLM,
						        DebertaForSequenceClassification,
						        DebertaForTokenClassification,
						        DebertaForQuestionAnswering,
						    )
						    if is_torch_available()
						    else ()
						)
						lowerCAmelCase_							=  (
						    {
						        '''feature-extraction''': DebertaModel,
						        '''fill-mask''': DebertaForMaskedLM,
						        '''question-answering''': DebertaForQuestionAnswering,
						        '''text-classification''': DebertaForSequenceClassification,
						        '''token-classification''': DebertaForTokenClassification,
						        '''zero-shot''': DebertaForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)

						lowerCAmelCase_							=  True
						lowerCAmelCase_							=  False
						lowerCAmelCase_							=  False
						lowerCAmelCase_							=  False
						lowerCAmelCase_							=  False






						def 			snake_case__		(		self				:	int		):






							"""simple docstring"""






							snake_case_						       =	DebertaModelTester(self		)
							snake_case_						       =	ConfigTester(self			,      config_class=__lowercase			,      hidden_size=37		)






						def 			snake_case__		(		self				:	Optional[int]		):






							"""simple docstring"""






							self.config_tester.run_common_tests()






						def 			snake_case__		(		self				:	Dict		):






							"""simple docstring"""






							snake_case_						       =	self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_deberta_model(*__lowercase		)






						def 			snake_case__		(		self				:	str		):






							"""simple docstring"""






							snake_case_						       =	self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_deberta_for_sequence_classification(*__lowercase		)






						def 			snake_case__		(		self				:	str		):






							"""simple docstring"""






							snake_case_						       =	self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_deberta_for_masked_lm(*__lowercase		)






						def 			snake_case__		(		self				:	Optional[Any]		):






							"""simple docstring"""






							snake_case_						       =	self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_deberta_for_question_answering(*__lowercase		)






						def 			snake_case__		(		self				:	Optional[Any]		):






							"""simple docstring"""






							snake_case_						       =	self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_deberta_for_token_classification(*__lowercase		)






						@slow
						def 			snake_case__		(		self				:	str		):






							"""simple docstring"""






							for model_name in DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								snake_case_						       =	DebertaModel.from_pretrained(__lowercase		)
								self.assertIsNotNone(__lowercase		)







@require_torch
@require_sentencepiece
@require_tokenizers
class 						UpperCAmelCase  (							unittest.TestCase     ):







						'''simple docstring'''





						@unittest.skip(reason="Model not available yet"		)
						def 			snake_case__		(		self				:	str		):






							"""simple docstring"""






							pass






						@slow
						def 			snake_case__		(		self				:	Optional[int]		):






							"""simple docstring"""






							snake_case_						       =	DebertaModel.from_pretrained("microsoft/deberta-base"		)

							snake_case_						       =	torch.tensor([[0, 3_14_14, 2_32, 3_28, 7_40, 11_40, 1_26_95, 69, 4_60_78, 15_88, 2]]		)
							snake_case_						       =	torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]		)
							with torch.no_grad():
								snake_case_						       =	model(__lowercase			,      attention_mask=__lowercase		)[0]
							# compare the actual values for a slice.
							snake_case_						       =	torch.tensor(
							    [[[-0.5986, -0.8055, -0.8462], [1.4484, -0.9348, -0.8059], [0.3123, 0.0032, -1.4131]]]		)
							self.assertTrue(torch.allclose(output[:, 1:4, 1:4]			,      __lowercase			,      atol=1E-4		)			,      f"{output[:, 1:4, 1:4]}"		)
