




"""simple docstring"""






import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XGLMTokenizer, XGLMTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCAmelCase					: Dict        =get_tests_dir("""fixtures/test_sentencepiece.model""")





@require_sentencepiece
@require_tokenizers
class 				_A		( __A						,					unittest.TestCase			):
      snake_case__    :     Tuple													=			XGLMTokenizer
      snake_case__    :     str													=			XGLMTokenizerFast
      snake_case__    :     List[Any]													=			True
      snake_case__    :     List[str]													=			True

      def        A__ (		self		):







            """simple docstring"""



            super().setUp()

            # We have a SentencePiece fixture for testing
            lowercase														=			XGLMTokenizer(__lowerCAmelCase	, keep_accents=__lowerCAmelCase		)
            tokenizer.save_pretrained(self.tmpdirname		)

      def        A__ (		self		):







            """simple docstring"""



            lowercase														=			'<pad>'
            lowercase														=			1

            self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCAmelCase		)	, __lowerCAmelCase		)
            self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCAmelCase		)	, __lowerCAmelCase		)

      def        A__ (		self		):







            """simple docstring"""



            lowercase														=			list(self.get_tokenizer().get_vocab().keys()		)

            self.assertEqual(vocab_keys[0]	, """<s>"""		)
            self.assertEqual(vocab_keys[1]	, """<pad>"""		)
            self.assertEqual(len(__lowerCAmelCase		)	, 1008		)

      def        A__ (		self		):







            """simple docstring"""



            self.assertEqual(self.get_tokenizer().vocab_size	, 1008		)

      def        A__ (		self		):







            """simple docstring"""



            lowercase														=			XGLMTokenizer(__lowerCAmelCase	, keep_accents=__lowerCAmelCase		)

            lowercase														=			tokenizer.tokenize("""This is a test"""		)
            self.assertListEqual(__lowerCAmelCase	, ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]		)

            self.assertListEqual(
                tokenizer.convert_tokens_to_ids(__lowerCAmelCase		)	, [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]	, )

            lowercase														=			tokenizer.tokenize("""I was born in 92000, and this is falsé."""		)
            self.assertListEqual(
                __lowerCAmelCase	, [
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """9""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """é""",
                    """.""",
                ]	, )
            lowercase														=			tokenizer.convert_tokens_to_ids(__lowerCAmelCase		)
            self.assertListEqual(
                __lowerCAmelCase	, [
                    value + tokenizer.fairseq_offset
                    for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
                ]	, )

            lowercase														=			tokenizer.convert_ids_to_tokens(__lowerCAmelCase		)
            self.assertListEqual(
                __lowerCAmelCase	, [
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """<unk>""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """<unk>""",
                    """.""",
                ]	, )

      @cached_property
      def        A__ (		self		):







            """simple docstring"""



            return XGLMTokenizer.from_pretrained("""facebook/xglm-564M"""		)

      def        A__ (		self		):







            """simple docstring"""



            with tempfile.NamedTemporaryFile() as f:
                  shutil.copyfile(__lowerCAmelCase	, f.name		)
                  lowercase														=			XGLMTokenizer(f.name	, keep_accents=__lowerCAmelCase		)
                  lowercase														=			pickle.dumps(__lowerCAmelCase		)
            pickle.loads(__lowerCAmelCase		)

      def        A__ (		self		):







            """simple docstring"""



            if not self.test_rust_tokenizer:
                  return

            lowercase														=			self.get_tokenizer()
            lowercase														=			self.get_rust_tokenizer()

            lowercase														=			'I was born in 92000, and this is falsé.'

            lowercase														=			tokenizer.tokenize(__lowerCAmelCase		)
            lowercase														=			rust_tokenizer.tokenize(__lowerCAmelCase		)
            self.assertListEqual(__lowerCAmelCase	, __lowerCAmelCase		)

            lowercase														=			tokenizer.encode(__lowerCAmelCase	, add_special_tokens=__lowerCAmelCase		)
            lowercase														=			rust_tokenizer.encode(__lowerCAmelCase	, add_special_tokens=__lowerCAmelCase		)
            self.assertListEqual(__lowerCAmelCase	, __lowerCAmelCase		)

            lowercase														=			self.get_rust_tokenizer()
            lowercase														=			tokenizer.encode(__lowerCAmelCase		)
            lowercase														=			rust_tokenizer.encode(__lowerCAmelCase		)
            self.assertListEqual(__lowerCAmelCase	, __lowerCAmelCase		)

      @slow
      def        A__ (		self		):







            """simple docstring"""



            lowercase														=			'Hello World!'
            lowercase														=			[2, 3_1227, 4447, 35]

            self.assertListEqual(__lowerCAmelCase	, self.big_tokenizer.encode(__lowerCAmelCase		)		)

      @slow
      def        A__ (		self		):







            """simple docstring"""



            lowercase														=			(
                'This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'
                ' add words that should not exsist and be tokenized to unk, such as saoneuhaoesuth'
            )
            # fmt: off
            lowercase														=			[2, 1018, 67, 11, 1988, 2617, 5631, 278, 11, 3407, 48, 7_1630, 2_8085, 4, 3234, 157, 13, 6, 5, 6, 4, 3526, 768, 15, 659, 57, 298, 3983, 864, 129, 21, 6, 5, 1_3675, 377, 652, 7580, 1_0341, 155, 2817, 422, 1666, 7, 1674, 53, 113, 20_2277, 1_7892, 33, 60, 87, 4, 3234, 157, 61, 2667, 5_2376, 19, 88, 23, 735]
            # fmt: on

            self.assertListEqual(__lowerCAmelCase	, self.big_tokenizer.encode(__lowerCAmelCase		)		)







      @slow
      def        A__ (		self		):







            """simple docstring"""



            lowercase														=			{
                'input_ids': [[2, 10_8825, 1163, 15, 8_8010, 473, 1_5898, 157, 1_3672, 1857, 312, 8, 23_8021, 1163, 53, 1_3672, 1857, 312, 8, 5_3283, 18_2396, 8, 1_8566, 16, 3_6733, 4101, 8, 230, 24_4017, 12_2553, 7, 15, 13_2597, 4, 293, 1_2511, 7610, 4, 3414, 13_2597, 9, 4, 3_2361, 362, 4, 734, 2_8512, 3_2569, 18, 4, 3_2361, 2_6096, 1_4982, 73, 1_8715, 2_1433, 23_5261, 15, 492, 1_2427, 16, 53, 1_8715, 2_1433, 6_5454, 15, 2_3659, 563, 16, 278, 597, 2843, 595, 7931, 18_2396, 6_4186, 22, 886, 595, 13_2981, 53, 2_5540, 3449, 4_3982, 3_9901, 5951, 878, 330, 4, 2_7694, 8_0269, 312, 53, 6517, 1_1780, 611, 2_0408, 5], [2, 6, 13_2597, 67, 4_2897, 33, 592, 8, 16_3729, 2_5540, 361, 13_6997, 10_9514, 17_3230, 7, 501, 60, 10_2913, 196, 5631, 235, 6_3243, 473, 6, 23_1757, 74, 5277, 7905, 53, 3095, 3_7317, 22, 454, 18_3874, 5], [2, 268, 3_1298, 4_6530, 6, 13_2935, 4_3831, 7, 597, 32, 24, 3688, 9865, 5]],
                'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]
            }  # noqa: E501
            # fmt: on

            self.tokenizer_integration_test_util(
                expected_encoding=__lowerCAmelCase	, model_name="""facebook/xglm-564M"""	, padding=__lowerCAmelCase	, )




"""simple docstring"""






import unittest

from transformers import DebertaConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           DebertaForMaskedLM,
           DebertaForQuestionAnswering,
           DebertaForSequenceClassification,
           DebertaForTokenClassification,
           DebertaModel,
       )
       from transformers.models.deberta.modeling_deberta import DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST





class 				_A		( lowerCAmelCase			):

      def __init__(		self	, __lowerCAmelCase	, __lowerCAmelCase=13	, __lowerCAmelCase=7	, __lowerCAmelCase=True	, __lowerCAmelCase=True	, __lowerCAmelCase=True	, __lowerCAmelCase=True	, __lowerCAmelCase=99	, __lowerCAmelCase=32	, __lowerCAmelCase=5	, __lowerCAmelCase=4	, __lowerCAmelCase=37	, __lowerCAmelCase="gelu"	, __lowerCAmelCase=0.1	, __lowerCAmelCase=0.1	, __lowerCAmelCase=512	, __lowerCAmelCase=16	, __lowerCAmelCase=2	, __lowerCAmelCase=0.0_2	, __lowerCAmelCase=False	, __lowerCAmelCase=True	, __lowerCAmelCase="None"	, __lowerCAmelCase=3	, __lowerCAmelCase=4	, __lowerCAmelCase=None	, ):







            """simple docstring"""



            lowercase														=			parent
            lowercase														=			batch_size
            lowercase														=			seq_length
            lowercase														=			is_training
            lowercase														=			use_input_mask
            lowercase														=			use_token_type_ids
            lowercase														=			use_labels
            lowercase														=			vocab_size
            lowercase														=			hidden_size
            lowercase														=			num_hidden_layers
            lowercase														=			num_attention_heads
            lowercase														=			intermediate_size
            lowercase														=			hidden_act
            lowercase														=			hidden_dropout_prob
            lowercase														=			attention_probs_dropout_prob
            lowercase														=			max_position_embeddings
            lowercase														=			type_vocab_size
            lowercase														=			type_sequence_label_size
            lowercase														=			initializer_range
            lowercase														=			num_labels
            lowercase														=			num_choices
            lowercase														=			relative_attention
            lowercase														=			position_biased_input
            lowercase														=			pos_att_type
            lowercase														=			scope

      def        A__ (		self		):







            """simple docstring"""



            lowercase														=			ids_tensor([self.batch_size, self.seq_length]	, self.vocab_size		)

            lowercase														=			None
            if self.use_input_mask:
                  lowercase														=			ids_tensor([self.batch_size, self.seq_length]	, vocab_size=2		)

            lowercase														=			None
            if self.use_token_type_ids:
                  lowercase														=			ids_tensor([self.batch_size, self.seq_length]	, self.type_vocab_size		)

            lowercase														=			None
            lowercase														=			None
            lowercase														=			None
            if self.use_labels:
                  lowercase														=			ids_tensor([self.batch_size]	, self.type_sequence_label_size		)
                  lowercase														=			ids_tensor([self.batch_size, self.seq_length]	, self.num_labels		)
                  lowercase														=			ids_tensor([self.batch_size]	, self.num_choices		)

            lowercase														=			self.get_config()

            return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

      def        A__ (		self		):







            """simple docstring"""



            return DebertaConfig(
                vocab_size=self.vocab_size	, hidden_size=self.hidden_size	, num_hidden_layers=self.num_hidden_layers	, num_attention_heads=self.num_attention_heads	, intermediate_size=self.intermediate_size	, hidden_act=self.hidden_act	, hidden_dropout_prob=self.hidden_dropout_prob	, attention_probs_dropout_prob=self.attention_probs_dropout_prob	, max_position_embeddings=self.max_position_embeddings	, type_vocab_size=self.type_vocab_size	, initializer_range=self.initializer_range	, relative_attention=self.relative_attention	, position_biased_input=self.position_biased_input	, pos_att_type=self.pos_att_type	, )

      def        A__ (		self		):







            """simple docstring"""



            lowercase														=			self.get_config()
            lowercase														=			300
            return config

      def        A__ (		self	, __lowerCAmelCase		):







            """simple docstring"""



            self.parent.assertListEqual(list(result.loss.size()		)	, []		)

      def        A__ (		self	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase		):







            """simple docstring"""



            lowercase														=			DebertaModel(config=__lowerCAmelCase		)
            model.to(__lowerCAmelCase		)
            model.eval()
            lowercase														=			model(__lowerCAmelCase	, attention_mask=__lowerCAmelCase	, token_type_ids=__lowerCAmelCase		)[0]
            lowercase														=			model(__lowerCAmelCase	, token_type_ids=__lowerCAmelCase		)[0]
            lowercase														=			model(__lowerCAmelCase		)[0]

            self.parent.assertListEqual(list(sequence_output.size()		)	, [self.batch_size, self.seq_length, self.hidden_size]		)

      def        A__ (		self	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase		):







            """simple docstring"""



            lowercase														=			DebertaForMaskedLM(config=__lowerCAmelCase		)
            model.to(__lowerCAmelCase		)
            model.eval()
            lowercase														=			model(__lowerCAmelCase	, attention_mask=__lowerCAmelCase	, token_type_ids=__lowerCAmelCase	, labels=__lowerCAmelCase		)

            self.parent.assertEqual(result.logits.shape	, (self.batch_size, self.seq_length, self.vocab_size)		)

      def        A__ (		self	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase		):







            """simple docstring"""



            lowercase														=			self.num_labels
            lowercase														=			DebertaForSequenceClassification(__lowerCAmelCase		)
            model.to(__lowerCAmelCase		)
            model.eval()
            lowercase														=			model(__lowerCAmelCase	, attention_mask=__lowerCAmelCase	, token_type_ids=__lowerCAmelCase	, labels=__lowerCAmelCase		)
            self.parent.assertListEqual(list(result.logits.size()		)	, [self.batch_size, self.num_labels]		)
            self.check_loss_output(__lowerCAmelCase		)

      def        A__ (		self	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase		):







            """simple docstring"""



            lowercase														=			self.num_labels
            lowercase														=			DebertaForTokenClassification(config=__lowerCAmelCase		)
            model.to(__lowerCAmelCase		)
            model.eval()
            lowercase														=			model(__lowerCAmelCase	, attention_mask=__lowerCAmelCase	, token_type_ids=__lowerCAmelCase	, labels=__lowerCAmelCase		)
            self.parent.assertEqual(result.logits.shape	, (self.batch_size, self.seq_length, self.num_labels)		)

      def        A__ (		self	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase	, __lowerCAmelCase		):







            """simple docstring"""



            lowercase														=			DebertaForQuestionAnswering(config=__lowerCAmelCase		)
            model.to(__lowerCAmelCase		)
            model.eval()
            lowercase														=			model(
                __lowerCAmelCase	, attention_mask=__lowerCAmelCase	, token_type_ids=__lowerCAmelCase	, start_positions=__lowerCAmelCase	, end_positions=__lowerCAmelCase	, )
            self.parent.assertEqual(result.start_logits.shape	, (self.batch_size, self.seq_length)		)
            self.parent.assertEqual(result.end_logits.shape	, (self.batch_size, self.seq_length)		)







      def        A__ (		self		):







            """simple docstring"""



            lowercase														=			self.prepare_config_and_inputs()
            (
                (
                lowercase
            )							,							(
                lowercase
            )							,							(
                lowercase
            )							,							(
                lowercase
            )							,							(
                lowercase
            )							,							(
                lowercase
            )							,							(
                lowercase
            )							,							
            )														=			config_and_inputs
            lowercase														=			{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
            return config, inputs_dict






@require_torch
class 				_A		( lowerCAmelCase						,					lowerCAmelCase						,					unittest.TestCase			):
      snake_case__    :     Any													=			(
          (
              DebertaModel,
              DebertaForMaskedLM,
              DebertaForSequenceClassification,
              DebertaForTokenClassification,
              DebertaForQuestionAnswering,
          )
          if is_torch_available()
          else ()
      )
      snake_case__    :     int													=			(
          {
              'feature-extraction': DebertaModel,
              'fill-mask': DebertaForMaskedLM,
              'question-answering': DebertaForQuestionAnswering,
              'text-classification': DebertaForSequenceClassification,
              'token-classification': DebertaForTokenClassification,
              'zero-shot': DebertaForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )

      snake_case__    :     Optional[Any]													=			True
      snake_case__    :     Optional[Any]													=			False
      snake_case__    :     Optional[int]													=			False
      snake_case__    :     Any													=			False
      snake_case__    :     List[Any]													=			False

      def        A__ (		self		):







            """simple docstring"""



            lowercase														=			DebertaModelTester(self		)
            lowercase														=			ConfigTester(self	, config_class=__lowerCAmelCase	, hidden_size=37		)

      def        A__ (		self		):







            """simple docstring"""



            self.config_tester.run_common_tests()

      def        A__ (		self		):







            """simple docstring"""



            lowercase														=			self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_deberta_model(*__lowerCAmelCase		)

      def        A__ (		self		):







            """simple docstring"""



            lowercase														=			self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_deberta_for_sequence_classification(*__lowerCAmelCase		)

      def        A__ (		self		):







            """simple docstring"""



            lowercase														=			self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_deberta_for_masked_lm(*__lowerCAmelCase		)

      def        A__ (		self		):







            """simple docstring"""



            lowercase														=			self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_deberta_for_question_answering(*__lowerCAmelCase		)

      def        A__ (		self		):







            """simple docstring"""



            lowercase														=			self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_deberta_for_token_classification(*__lowerCAmelCase		)







      @slow
      def        A__ (		self		):







            """simple docstring"""



            for model_name in DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                  lowercase														=			DebertaModel.from_pretrained(__lowerCAmelCase		)
                  self.assertIsNotNone(__lowerCAmelCase		)






@require_torch
@require_sentencepiece
@require_tokenizers
class 				_A		( unittest.TestCase			):

      @unittest.skip(reason="""Model not available yet"""		)
      def        A__ (		self		):







            """simple docstring"""



            pass







      @slow
      def        A__ (		self		):







            """simple docstring"""



            lowercase														=			DebertaModel.from_pretrained("""microsoft/deberta-base"""		)

            lowercase														=			torch.tensor([[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]]		)
            lowercase														=			torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]		)
            with torch.no_grad():
                  lowercase														=			model(__lowerCAmelCase	, attention_mask=__lowerCAmelCase		)[0]
            # compare the actual values for a slice.
            lowercase														=			torch.tensor(
                [[[-0.5_9_8_6, -0.8_0_5_5, -0.8_4_6_2], [1.4_4_8_4, -0.9_3_4_8, -0.8_0_5_9], [0.3_1_2_3, 0.0_0_3_2, -1.4_1_3_1]]]		)
            self.assertTrue(torch.allclose(output[:, 1:4, 1:4]	, __lowerCAmelCase	, atol=1E-4		)	, f'{output[:, 1:4, 1:4]}'		)



