

'''simple docstring'''
import unittest

from transformers import DebertaConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         DebertaForMaskedLM,
         DebertaForQuestionAnswering,
         DebertaForSequenceClassification,
         DebertaForTokenClassification,
         DebertaModel,
     )
     from transformers.models.deberta.modeling_deberta import DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST

class 						__magic_name__   (							__SCREAMING_SNAKE_CASE ):







 def __init__(       self  , snake_case_  , snake_case_=13  , snake_case_=7  , snake_case_=True  , snake_case_=True  , snake_case_=True  , snake_case_=True  , snake_case_=99  , snake_case_=32  , snake_case_=5  , snake_case_=4  , snake_case_=37  , snake_case_="gelu"  , snake_case_=0.1  , snake_case_=0.1  , snake_case_=5_12  , snake_case_=16  , snake_case_=2  , snake_case_=0.02  , snake_case_=False  , snake_case_=True  , snake_case_="None"  , snake_case_=3  , snake_case_=4  , snake_case_=None  , ):
   lowercase									=parent
   lowercase									=batch_size
   lowercase									=seq_length
   lowercase									=is_training
   lowercase									=use_input_mask
   lowercase									=use_token_type_ids
   lowercase									=use_labels
   lowercase									=vocab_size
   lowercase									=hidden_size
   lowercase									=num_hidden_layers
   lowercase									=num_attention_heads
   lowercase									=intermediate_size
   lowercase									=hidden_act
   lowercase									=hidden_dropout_prob
   lowercase									=attention_probs_dropout_prob
   lowercase									=max_position_embeddings
   lowercase									=type_vocab_size
   lowercase									=type_sequence_label_size
   lowercase									=initializer_range
   lowercase									=num_labels
   lowercase									=num_choices
   lowercase									=relative_attention
   lowercase									=position_biased_input
   lowercase									=pos_att_type
   lowercase									=scope







 def 						_A(       self   ):
   lowercase									=ids_tensor([self.batch_size, self.seq_length]  , self.vocab_size   )

   lowercase									=None
   if self.use_input_mask:
     lowercase									=ids_tensor([self.batch_size, self.seq_length]  , vocab_size=2   )

   lowercase									=None
   if self.use_token_type_ids:
     lowercase									=ids_tensor([self.batch_size, self.seq_length]  , self.type_vocab_size   )

   lowercase									=None
   lowercase									=None
   lowercase									=None
   if self.use_labels:
     lowercase									=ids_tensor([self.batch_size]  , self.type_sequence_label_size   )
     lowercase									=ids_tensor([self.batch_size, self.seq_length]  , self.num_labels   )
     lowercase									=ids_tensor([self.batch_size]  , self.num_choices   )

   lowercase									=self.get_config()

   return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







 def 						_A(       self   ):
   return DebertaConfig(
       vocab_size=self.vocab_size  , hidden_size=self.hidden_size  , num_hidden_layers=self.num_hidden_layers  , num_attention_heads=self.num_attention_heads  , intermediate_size=self.intermediate_size  , hidden_act=self.hidden_act  , hidden_dropout_prob=self.hidden_dropout_prob  , attention_probs_dropout_prob=self.attention_probs_dropout_prob  , max_position_embeddings=self.max_position_embeddings  , type_vocab_size=self.type_vocab_size  , initializer_range=self.initializer_range  , relative_attention=self.relative_attention  , position_biased_input=self.position_biased_input  , pos_att_type=self.pos_att_type  , )







 def 						_A(       self   ):
   lowercase									=self.get_config()
   lowercase									=3_00
   return config







 def 						_A(       self  , snake_case_   ):
   self.parent.assertListEqual(list(result.loss.size()   )  , []   )







 def 						_A(       self  , snake_case_  , snake_case_  , snake_case_  , snake_case_  , snake_case_  , snake_case_  , snake_case_   ):
   lowercase									=DebertaModel(config=snake_case_   )
   model.to(snake_case_   )
   model.eval()
   lowercase									=model(snake_case_  , attention_mask=snake_case_  , token_type_ids=snake_case_   )[0]
   lowercase									=model(snake_case_  , token_type_ids=snake_case_   )[0]
   lowercase									=model(snake_case_   )[0]

   self.parent.assertListEqual(list(sequence_output.size()   )  , [self.batch_size, self.seq_length, self.hidden_size]   )







 def 						_A(       self  , snake_case_  , snake_case_  , snake_case_  , snake_case_  , snake_case_  , snake_case_  , snake_case_   ):
   lowercase									=DebertaForMaskedLM(config=snake_case_   )
   model.to(snake_case_   )
   model.eval()
   lowercase									=model(snake_case_  , attention_mask=snake_case_  , token_type_ids=snake_case_  , labels=snake_case_   )

   self.parent.assertEqual(result.logits.shape  , (self.batch_size, self.seq_length, self.vocab_size)   )







 def 						_A(       self  , snake_case_  , snake_case_  , snake_case_  , snake_case_  , snake_case_  , snake_case_  , snake_case_   ):
   lowercase									=self.num_labels
   lowercase									=DebertaForSequenceClassification(snake_case_   )
   model.to(snake_case_   )
   model.eval()
   lowercase									=model(snake_case_  , attention_mask=snake_case_  , token_type_ids=snake_case_  , labels=snake_case_   )
   self.parent.assertListEqual(list(result.logits.size()   )  , [self.batch_size, self.num_labels]   )
   self.check_loss_output(snake_case_   )







 def 						_A(       self  , snake_case_  , snake_case_  , snake_case_  , snake_case_  , snake_case_  , snake_case_  , snake_case_   ):
   lowercase									=self.num_labels
   lowercase									=DebertaForTokenClassification(config=snake_case_   )
   model.to(snake_case_   )
   model.eval()
   lowercase									=model(snake_case_  , attention_mask=snake_case_  , token_type_ids=snake_case_  , labels=snake_case_   )
   self.parent.assertEqual(result.logits.shape  , (self.batch_size, self.seq_length, self.num_labels)   )







 def 						_A(       self  , snake_case_  , snake_case_  , snake_case_  , snake_case_  , snake_case_  , snake_case_  , snake_case_   ):
   lowercase									=DebertaForQuestionAnswering(config=snake_case_   )
   model.to(snake_case_   )
   model.eval()
   lowercase									=model(
       snake_case_  , attention_mask=snake_case_  , token_type_ids=snake_case_  , start_positions=snake_case_  , end_positions=snake_case_  , )
   self.parent.assertEqual(result.start_logits.shape  , (self.batch_size, self.seq_length)   )
   self.parent.assertEqual(result.end_logits.shape  , (self.batch_size, self.seq_length)   )







 def 						_A(       self   ):
   lowercase									=self.prepare_config_and_inputs()
   (
       (
       lowercase
   )							, (
       lowercase
   )							, (
       lowercase
   )							, (
       lowercase
   )							, (
       lowercase
   )							, (
       lowercase
   )							, (
       lowercase
   )							, 
   )									=config_and_inputs
   lowercase									={'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
   return config, inputs_dict

@require_torch
class 						__magic_name__   (							__SCREAMING_SNAKE_CASE ,				__SCREAMING_SNAKE_CASE ,				unittest.TestCase ):
 UpperCamelCase__     =       (
     (
         DebertaModel,
         DebertaForMaskedLM,
         DebertaForSequenceClassification,
         DebertaForTokenClassification,
         DebertaForQuestionAnswering,
     )
     if is_torch_available()
     else ()
 )
 UpperCamelCase__     =       (
     {
         'feature-extraction': DebertaModel,
         'fill-mask': DebertaForMaskedLM,
         'question-answering': DebertaForQuestionAnswering,
         'text-classification': DebertaForSequenceClassification,
         'token-classification': DebertaForTokenClassification,
         'zero-shot': DebertaForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )

 UpperCamelCase__     =       True
 UpperCamelCase__     =       False
 UpperCamelCase__     =       False
 UpperCamelCase__     =       False
 UpperCamelCase__     =       False







 def 						_A(       self   ):
   lowercase									=DebertaModelTester(self   )
   lowercase									=ConfigTester(self  , config_class=snake_case_  , hidden_size=37   )







 def 						_A(       self   ):
   self.config_tester.run_common_tests()







 def 						_A(       self   ):
   lowercase									=self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_deberta_model(*snake_case_   )







 def 						_A(       self   ):
   lowercase									=self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_deberta_for_sequence_classification(*snake_case_   )







 def 						_A(       self   ):
   lowercase									=self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_deberta_for_masked_lm(*snake_case_   )







 def 						_A(       self   ):
   lowercase									=self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_deberta_for_question_answering(*snake_case_   )







 def 						_A(       self   ):
   lowercase									=self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_deberta_for_token_classification(*snake_case_   )







 @slow
 def 						_A(       self   ):
   for model_name in DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
     lowercase									=DebertaModel.from_pretrained(snake_case_   )
     self.assertIsNotNone(snake_case_   )

@require_torch
@require_sentencepiece
@require_tokenizers
class 						__magic_name__   (							unittest.TestCase ):







 @unittest.skip(reason='''Model not available yet'''   )
 def 						_A(       self   ):
   pass







 @slow
 def 						_A(       self   ):
   lowercase									=DebertaModel.from_pretrained('''microsoft/deberta-base'''   )

   lowercase									=torch.tensor([[0, 3_14_14, 2_32, 3_28, 7_40, 11_40, 1_26_95, 69, 4_60_78, 15_88, 2]]   )
   lowercase									=torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]   )
   with torch.no_grad():
     lowercase									=model(snake_case_  , attention_mask=snake_case_   )[0]
   # compare the actual values for a slice.
   lowercase									=torch.tensor(
       [[[-0.59_86, -0.80_55, -0.84_62], [1.44_84, -0.93_48, -0.80_59], [0.31_23, 0.00_32, -1.41_31]]]   )
   self.assertTrue(torch.allclose(output[:, 1:4, 1:4]  , snake_case_  , atol=1E-4   )  , f'{output[:, 1:4, 1:4]}'   )





"""simple docstring"""




import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation


_lowerCAmelCase							:   Any				=				logging.get_logger(__name__)

_lowerCAmelCase							:   Any				=				{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

_lowerCAmelCase							:   List[str]				=				{
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/tokenizer.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/tokenizer.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/tokenizer.json",
    },
}

_lowerCAmelCase							:   List[str]				=				{
    "gpt2": 10_24,
    "gpt2-medium": 10_24,
    "gpt2-large": 10_24,
    "gpt2-xl": 10_24,
    "distilgpt2": 10_24,
}






class       UpperCAmelCase_    (							_UpperCamelCase		):
      __SCREAMING_SNAKE_CASE		:  Optional[Any]						=					VOCAB_FILES_NAMES
      __SCREAMING_SNAKE_CASE		:  int						=					PRETRAINED_VOCAB_FILES_MAP
      __SCREAMING_SNAKE_CASE		:  Optional[int]						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __SCREAMING_SNAKE_CASE		:  Optional[int]						=					['input_ids', 'attention_mask']
      __SCREAMING_SNAKE_CASE		:  Dict						=					GPTaTokenizer



      def __init__(	self	:      Optional[Any]	,      A	:      Optional[int]=None	,      A	:      List[Any]=None	,      A	:      Optional[int]=None	,      A	:      Optional[Any]="<|endoftext|>"	,      A	:      Union[str, Any]="<|endoftext|>"	,      A	:      Optional[Any]="<|endoftext|>"	,      A	:      Optional[Any]=False	,      **A	:      str	,      ):
           super().__init__(
               A	,      A	,      tokenizer_file=A	,      unk_token=A	,      bos_token=A	,      eos_token=A	,      add_prefix_space=A	,      **A	,      )

           _UpperCAmelCase     :      List[Any]														=       kwargs.pop("add_bos_token"	,      A		)

           _UpperCAmelCase     :      List[str]														=       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
           if pre_tok_state.get("add_prefix_space"	,      A		) != add_prefix_space:
                _UpperCAmelCase     :      Dict														=       getattr(A	,      pre_tok_state.pop("type"		)		)
                _UpperCAmelCase     :      Optional[Any]														=       add_prefix_space
                _UpperCAmelCase     :      Optional[Any]														=       pre_tok_class(**A		)

           _UpperCAmelCase     :      List[Any]														=       add_prefix_space



      def 		snake_case_   (	self	:      List[Any]	,      *A	:      List[Any]	,      **A	:      List[str]		):
           _UpperCAmelCase     :      Union[str, Any]														=       kwargs.get("is_split_into_words"	,      A		)
           assert self.add_prefix_space or not is_split_into_words, (
               f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
               "to use it with pretokenized inputs."
           )

           return super()._batch_encode_plus(*A	,      **A		)



      def 		snake_case_   (	self	:      Optional[Any]	,      *A	:      List[Any]	,      **A	:      int		):
           _UpperCAmelCase     :      Union[str, Any]														=       kwargs.get("is_split_into_words"	,      A		)

           assert self.add_prefix_space or not is_split_into_words, (
               f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
               "to use it with pretokenized inputs."
           )

           return super()._encode_plus(*A	,      **A		)



      def 		snake_case_   (	self	:      Optional[int]	,      A	:      str	,      A	:      Optional[str] = None		):
           _UpperCAmelCase     :      List[Any]														=       self._tokenizer.model.save(A	,      name=A		)
           return tuple(A		)



      def 		snake_case_   (	self	:      int	,      A	:      "Conversation"		):
           _UpperCAmelCase     :      List[str]														=       []
           for is_user, text in conversation.iter_texts():
                input_ids.extend(self.encode(A	,      add_special_tokens=A		) + [self.eos_token_id]		)

           if len(A		) > self.model_max_length:
                _UpperCAmelCase     :      Optional[int]														=       input_ids[-self.model_max_length :]
           return input_ids
