

import unittest

from transformers import DebertaConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        DebertaForMaskedLM,
        DebertaForQuestionAnswering,
        DebertaForSequenceClassification,
        DebertaForTokenClassification,
        DebertaModel,
    )
    from transformers.models.deberta.modeling_deberta import DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST

class      lowerCAmelCase							(      lowercase_    ):
 def __init__(   self						:   List[Any]    ,  UpperCAmelCase						:   Optional[int]    ,  UpperCAmelCase						:   int=13    ,  UpperCAmelCase						:   Dict=7    ,  UpperCAmelCase						:   Tuple=True    ,  UpperCAmelCase						:   Dict=True    ,  UpperCAmelCase						:   Optional[int]=True    ,  UpperCAmelCase						:   Dict=True    ,  UpperCAmelCase						:   Any=99    ,  UpperCAmelCase						:   Optional[Any]=32    ,  UpperCAmelCase						:   Tuple=5    ,  UpperCAmelCase						:   Any=4    ,  UpperCAmelCase						:   Tuple=37    ,  UpperCAmelCase						:   Optional[Any]="gelu"    ,  UpperCAmelCase						:   List[str]=0.1    ,  UpperCAmelCase						:   List[str]=0.1    ,  UpperCAmelCase						:   Dict=512    ,  UpperCAmelCase						:   List[str]=16    ,  UpperCAmelCase						:   Any=2    ,  UpperCAmelCase						:   str=0.0_2    ,  UpperCAmelCase						:   Any=False    ,  UpperCAmelCase						:   Tuple=True    ,  UpperCAmelCase						:   str="None"    ,  UpperCAmelCase						:   Optional[Any]=3    ,  UpperCAmelCase						:   int=4    ,  UpperCAmelCase						:   List[str]=None    ,  )    ->			Dict:
     lowerCamelCase__					:   Dict      				=	parent
     lowerCamelCase__					:   Union[str, Any]      				=	batch_size
     lowerCamelCase__					:   str      				=	seq_length
     lowerCamelCase__					:   Any      				=	is_training
     lowerCamelCase__					:   Any      				=	use_input_mask
     lowerCamelCase__					:   Tuple      				=	use_token_type_ids
     lowerCamelCase__					:   List[str]      				=	use_labels
     lowerCamelCase__					:   Any      				=	vocab_size
     lowerCamelCase__					:   Optional[int]      				=	hidden_size
     lowerCamelCase__					:   Dict      				=	num_hidden_layers
     lowerCamelCase__					:   Dict      				=	num_attention_heads
     lowerCamelCase__					:   Tuple      				=	intermediate_size
     lowerCamelCase__					:   Optional[int]      				=	hidden_act
     lowerCamelCase__					:   Dict      				=	hidden_dropout_prob
     lowerCamelCase__					:   Any      				=	attention_probs_dropout_prob
     lowerCamelCase__					:   str      				=	max_position_embeddings
     lowerCamelCase__					:   str      				=	type_vocab_size
     lowerCamelCase__					:   Any      				=	type_sequence_label_size
     lowerCamelCase__					:   List[Any]      				=	initializer_range
     lowerCamelCase__					:   List[Any]      				=	num_labels
     lowerCamelCase__					:   str      				=	num_choices
     lowerCamelCase__					:   Tuple      				=	relative_attention
     lowerCamelCase__					:   Dict      				=	position_biased_input
     lowerCamelCase__					:   Optional[Any]      				=	pos_att_type
     lowerCamelCase__					:   Union[str, Any]      				=	scope
 def   A_			(   self						:   Dict			)    ->			int:
     lowerCamelCase__					:   Any      				=	ids_tensor([self.batch_size, self.seq_length]    ,  self.vocab_size			)

     lowerCamelCase__					:   Optional[Any]      				=	None
     if self.use_input_mask:
         lowerCamelCase__					:   int      				=	ids_tensor([self.batch_size, self.seq_length]    ,  vocab_size=2			)

     lowerCamelCase__					:   List[str]      				=	None
     if self.use_token_type_ids:
         lowerCamelCase__					:   Dict      				=	ids_tensor([self.batch_size, self.seq_length]    ,  self.type_vocab_size			)

     lowerCamelCase__					:   int      				=	None
     lowerCamelCase__					:   Optional[int]      				=	None
     lowerCamelCase__					:   Dict      				=	None
     if self.use_labels:
         lowerCamelCase__					:   Dict      				=	ids_tensor([self.batch_size]    ,  self.type_sequence_label_size			)
         lowerCamelCase__					:   Tuple      				=	ids_tensor([self.batch_size, self.seq_length]    ,  self.num_labels			)
         lowerCamelCase__					:   Dict      				=	ids_tensor([self.batch_size]    ,  self.num_choices			)

     lowerCamelCase__					:   Optional[Any]      				=	self.get_config()

     return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
 def   A_			(   self						:   Dict			)    ->			Dict:
     return DebertaConfig(
         vocab_size=self.vocab_size    ,  hidden_size=self.hidden_size    ,  num_hidden_layers=self.num_hidden_layers    ,  num_attention_heads=self.num_attention_heads    ,  intermediate_size=self.intermediate_size    ,  hidden_act=self.hidden_act    ,  hidden_dropout_prob=self.hidden_dropout_prob    ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,  max_position_embeddings=self.max_position_embeddings    ,  type_vocab_size=self.type_vocab_size    ,  initializer_range=self.initializer_range    ,  relative_attention=self.relative_attention    ,  position_biased_input=self.position_biased_input    ,  pos_att_type=self.pos_att_type    ,  )
 def   A_			(   self						:   List[Any]			)    ->			List[str]:
     lowerCamelCase__					:   List[Any]      				=	self.get_config()
     lowerCamelCase__					:   Any      				=	300
     return config
 def   A_			(   self						:   Dict    ,  UpperCAmelCase						:   Tuple			)    ->			Dict:
     self.parent.assertListEqual(list(result.loss.size()			)    ,  []			)
 def   A_			(   self						:   Tuple    ,  UpperCAmelCase						:   Any    ,  UpperCAmelCase						:   Union[str, Any]    ,  UpperCAmelCase						:   Optional[int]    ,  UpperCAmelCase						:   Optional[int]    ,  UpperCAmelCase						:   List[Any]    ,  UpperCAmelCase						:   Tuple    ,  UpperCAmelCase						:   Dict			)    ->			Dict:
     lowerCamelCase__					:   Union[str, Any]      				=	DebertaModel(config=lowerCamelCase_			)
     model.to(lowerCamelCase_			)
     model.eval()
     lowerCamelCase__					:   Any      				=	model(lowerCamelCase_    ,  attention_mask=lowerCamelCase_    ,  token_type_ids=lowerCamelCase_			)[0]
     lowerCamelCase__					:   Tuple      				=	model(lowerCamelCase_    ,  token_type_ids=lowerCamelCase_			)[0]
     lowerCamelCase__					:   Optional[int]      				=	model(lowerCamelCase_			)[0]

     self.parent.assertListEqual(list(sequence_output.size()			)    ,  [self.batch_size, self.seq_length, self.hidden_size]			)
 def   A_			(   self						:   str    ,  UpperCAmelCase						:   Union[str, Any]    ,  UpperCAmelCase						:   Optional[int]    ,  UpperCAmelCase						:   List[str]    ,  UpperCAmelCase						:   Union[str, Any]    ,  UpperCAmelCase						:   Union[str, Any]    ,  UpperCAmelCase						:   Optional[Any]    ,  UpperCAmelCase						:   List[str]			)    ->			Tuple:
     lowerCamelCase__					:   Any      				=	DebertaForMaskedLM(config=lowerCamelCase_			)
     model.to(lowerCamelCase_			)
     model.eval()
     lowerCamelCase__					:   List[Any]      				=	model(lowerCamelCase_    ,  attention_mask=lowerCamelCase_    ,  token_type_ids=lowerCamelCase_    ,  labels=lowerCamelCase_			)

     self.parent.assertEqual(result.logits.shape    ,  (self.batch_size, self.seq_length, self.vocab_size)			)
 def   A_			(   self						:   List[Any]    ,  UpperCAmelCase						:   Any    ,  UpperCAmelCase						:   Any    ,  UpperCAmelCase						:   Optional[Any]    ,  UpperCAmelCase						:   int    ,  UpperCAmelCase						:   Optional[int]    ,  UpperCAmelCase						:   Tuple    ,  UpperCAmelCase						:   List[Any]			)    ->			Optional[Any]:
     lowerCamelCase__					:   str      				=	self.num_labels
     lowerCamelCase__					:   Optional[Any]      				=	DebertaForSequenceClassification(lowerCamelCase_			)
     model.to(lowerCamelCase_			)
     model.eval()
     lowerCamelCase__					:   Union[str, Any]      				=	model(lowerCamelCase_    ,  attention_mask=lowerCamelCase_    ,  token_type_ids=lowerCamelCase_    ,  labels=lowerCamelCase_			)
     self.parent.assertListEqual(list(result.logits.size()			)    ,  [self.batch_size, self.num_labels]			)
     self.check_loss_output(lowerCamelCase_			)
 def   A_			(   self						:   str    ,  UpperCAmelCase						:   str    ,  UpperCAmelCase						:   List[Any]    ,  UpperCAmelCase						:   Dict    ,  UpperCAmelCase						:   Union[str, Any]    ,  UpperCAmelCase						:   Union[str, Any]    ,  UpperCAmelCase						:   Any    ,  UpperCAmelCase						:   Optional[int]			)    ->			List[Any]:
     lowerCamelCase__					:   Optional[int]      				=	self.num_labels
     lowerCamelCase__					:   Tuple      				=	DebertaForTokenClassification(config=lowerCamelCase_			)
     model.to(lowerCamelCase_			)
     model.eval()
     lowerCamelCase__					:   Optional[Any]      				=	model(lowerCamelCase_    ,  attention_mask=lowerCamelCase_    ,  token_type_ids=lowerCamelCase_    ,  labels=lowerCamelCase_			)
     self.parent.assertEqual(result.logits.shape    ,  (self.batch_size, self.seq_length, self.num_labels)			)
 def   A_			(   self						:   str    ,  UpperCAmelCase						:   Union[str, Any]    ,  UpperCAmelCase						:   str    ,  UpperCAmelCase						:   Union[str, Any]    ,  UpperCAmelCase						:   int    ,  UpperCAmelCase						:   int    ,  UpperCAmelCase						:   Optional[int]    ,  UpperCAmelCase						:   Union[str, Any]			)    ->			Tuple:
     lowerCamelCase__					:   Any      				=	DebertaForQuestionAnswering(config=lowerCamelCase_			)
     model.to(lowerCamelCase_			)
     model.eval()
     lowerCamelCase__					:   int      				=	model(
         lowerCamelCase_    ,  attention_mask=lowerCamelCase_    ,  token_type_ids=lowerCamelCase_    ,  start_positions=lowerCamelCase_    ,  end_positions=lowerCamelCase_    ,  )
     self.parent.assertEqual(result.start_logits.shape    ,  (self.batch_size, self.seq_length)			)
     self.parent.assertEqual(result.end_logits.shape    ,  (self.batch_size, self.seq_length)			)







 def   A_			(   self						:   Optional[int]			)    ->			Optional[int]:
     lowerCamelCase__					:   int      				=	self.prepare_config_and_inputs()
     (
         lowerCamelCase__       
     )					:   List[Any]      				=	config_and_inputs
     lowerCamelCase__					:   List[Any]      				=	{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
     return config, inputs_dict


@require_torch
class      lowerCAmelCase							(      lowercase_,						lowercase_,						unittest.TestCase    ):
 UpperCAmelCase__									=	(
     (
         DebertaModel,
         DebertaForMaskedLM,
         DebertaForSequenceClassification,
         DebertaForTokenClassification,
         DebertaForQuestionAnswering,
     )
     if is_torch_available()
     else ()
 )
 UpperCAmelCase__									=	(
     {
         """feature-extraction""": DebertaModel,
         """fill-mask""": DebertaForMaskedLM,
         """question-answering""": DebertaForQuestionAnswering,
         """text-classification""": DebertaForSequenceClassification,
         """token-classification""": DebertaForTokenClassification,
         """zero-shot""": DebertaForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )

 UpperCAmelCase__									=	True
 UpperCAmelCase__									=	False
 UpperCAmelCase__									=	False
 UpperCAmelCase__									=	False
 UpperCAmelCase__									=	False
 def   A_			(   self						:   int			)    ->			Union[str, Any]:
     lowerCamelCase__					:   Optional[Any]      				=	DebertaModelTester(self			)
     lowerCamelCase__					:   Tuple      				=	ConfigTester(self    ,  config_class=lowerCamelCase_    ,  hidden_size=37			)
 def   A_			(   self						:   List[Any]			)    ->			List[str]:
     self.config_tester.run_common_tests()
 def   A_			(   self						:   Dict			)    ->			int:
     lowerCamelCase__					:   int      				=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_deberta_model(*lowerCamelCase_			)
 def   A_			(   self						:   Union[str, Any]			)    ->			Tuple:
     lowerCamelCase__					:   Optional[int]      				=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_deberta_for_sequence_classification(*lowerCamelCase_			)
 def   A_			(   self						:   int			)    ->			Optional[Any]:
     lowerCamelCase__					:   int      				=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_deberta_for_masked_lm(*lowerCamelCase_			)
 def   A_			(   self						:   Dict			)    ->			Dict:
     lowerCamelCase__					:   int      				=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_deberta_for_question_answering(*lowerCamelCase_			)
 def   A_			(   self						:   Optional[Any]			)    ->			str:
     lowerCamelCase__					:   List[str]      				=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_deberta_for_token_classification(*lowerCamelCase_			)







 @slow
 def   A_			(   self						:   Optional[Any]			)    ->			List[str]:



     for model_name in DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
         lowerCamelCase__					:   Dict      				=	DebertaModel.from_pretrained(lowerCamelCase_			)
         self.assertIsNotNone(lowerCamelCase_			)


@require_torch
@require_sentencepiece
@require_tokenizers
class      lowerCAmelCase							(      unittest.TestCase    ):
 @unittest.skip(reason='Model not available yet'			)
 def   A_			(   self						:   List[Any]			)    ->			Tuple:
     pass







 @slow
 def   A_			(   self						:   Any			)    ->			str:
     lowerCamelCase__					:   Optional[Any]      				=	DebertaModel.from_pretrained('microsoft/deberta-base'			)

     lowerCamelCase__					:   str      				=	torch.tensor([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]			)
     lowerCamelCase__					:   Any      				=	torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]			)
     with torch.no_grad():
         lowerCamelCase__					:   Union[str, Any]      				=	model(lowerCamelCase_    ,  attention_mask=lowerCamelCase_			)[0]
     # compare the actual values for a slice.
     lowerCamelCase__					:   Union[str, Any]      				=	torch.tensor(
         [[[-0.5_9_8_6, -0.8_0_5_5, -0.8_4_6_2], [1.4_4_8_4, -0.9_3_4_8, -0.8_0_5_9], [0.3_1_2_3, 0.0_0_3_2, -1.4_1_3_1]]]			)
     self.assertTrue(torch.allclose(output[:, 1:4, 1:4]    ,  lowerCamelCase_    ,  atol=1e-4			)    ,  F"""{output[:, 1:4, 1:4]}"""			)












from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)


def  SCREAMING_SNAKE_CASE   ( _UpperCAmelCase		,			_UpperCAmelCase				)    ->  Any:
    lowerCamelCase__					:   Any      				=	[]
    for part_id in partition_order:
        lowerCamelCase__					:   List[str]      				=	df.where(F"""SPARK_PARTITION_ID() = {part_id}"""				).collect()
        for row_idx, row in enumerate(_UpperCAmelCase				):
            expected_row_ids_and_row_dicts.append((F"""{part_id}_{row_idx}""", row.asDict())				)
    return expected_row_ids_and_row_dicts


@require_not_windows
@require_dill_gt_0_3_2
def  SCREAMING_SNAKE_CASE   ( )    ->  List[Any]:
    lowerCamelCase__					:   List[str]      				=	pyspark.sql.SparkSession.builder.master('local[*]'				).appName('pyspark'				).getOrCreate()
    lowerCamelCase__					:   Any      				=	spark.range(100				).repartition(1				)
    lowerCamelCase__					:   str      				=	Spark(_UpperCAmelCase				)
    # The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
    # that each partition can hold 2 rows.
    spark_builder._repartition_df_if_needed(max_shard_size=16				)
    # Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
    assert spark_builder.df.rdd.getNumPartitions() == 50


@require_not_windows
@require_dill_gt_0_3_2
def  SCREAMING_SNAKE_CASE   ( )    ->  Tuple:
    lowerCamelCase__					:   Any      				=	pyspark.sql.SparkSession.builder.master('local[*]'				).appName('pyspark'				).getOrCreate()
    lowerCamelCase__					:   Optional[int]      				=	spark.range(10				).repartition(2				)
    lowerCamelCase__					:   Dict      				=	[1, 0]
    lowerCamelCase__					:   str      				=	_generate_iterable_examples(_UpperCAmelCase		,			_UpperCAmelCase				)  # Reverse the partitions.
    lowerCamelCase__					:   Any      				=	_get_expected_row_ids_and_row_dicts_for_partition_order(_UpperCAmelCase		,			_UpperCAmelCase				)

    for i, (row_id, row_dict) in enumerate(generate_fn()				):
        lowerCamelCase__				,							lowerCamelCase__					:   Tuple      				=	expected_row_ids_and_row_dicts[i]
        assert row_id == expected_row_id
        assert row_dict == expected_row_dict


@require_not_windows
@require_dill_gt_0_3_2
def  SCREAMING_SNAKE_CASE   ( )    ->  Optional[Any]:
    lowerCamelCase__					:   Dict      				=	pyspark.sql.SparkSession.builder.master('local[*]'				).appName('pyspark'				).getOrCreate()
    lowerCamelCase__					:   Dict      				=	spark.range(10				).repartition(1				)
    lowerCamelCase__					:   List[str]      				=	SparkExamplesIterable(_UpperCAmelCase				)
    assert it.n_shards == 1
    for i, (row_id, row_dict) in enumerate(_UpperCAmelCase				):
        assert row_id == F"""0_{i}"""
        assert row_dict == {"id": i}


@require_not_windows
@require_dill_gt_0_3_2
def  SCREAMING_SNAKE_CASE   ( )    ->  Any:
    lowerCamelCase__					:   Dict      				=	pyspark.sql.SparkSession.builder.master('local[*]'				).appName('pyspark'				).getOrCreate()
    lowerCamelCase__					:   str      				=	spark.range(30				).repartition(3				)
    # Mock the generator so that shuffle reverses the partition indices.
    with patch('numpy.random.Generator'				) as generator_mock:
        lowerCamelCase__					:   int      				=	lambda _UpperCAmelCase				: x.reverse()
        lowerCamelCase__					:   Any      				=	_get_expected_row_ids_and_row_dicts_for_partition_order(_UpperCAmelCase		,			[2, 1, 0]				)

        lowerCamelCase__					:   Tuple      				=	SparkExamplesIterable(_UpperCAmelCase				).shuffle_data_sources(_UpperCAmelCase				)
        assert shuffled_it.n_shards == 3
        for i, (row_id, row_dict) in enumerate(_UpperCAmelCase				):
            lowerCamelCase__				,							lowerCamelCase__					:   Union[str, Any]      				=	expected_row_ids_and_row_dicts[i]
            assert row_id == expected_row_id
            assert row_dict == expected_row_dict


@require_not_windows
@require_dill_gt_0_3_2
def  SCREAMING_SNAKE_CASE   ( )    ->  Optional[Any]:
    lowerCamelCase__					:   Optional[Any]      				=	pyspark.sql.SparkSession.builder.master('local[*]'				).appName('pyspark'				).getOrCreate()
    lowerCamelCase__					:   Tuple      				=	spark.range(20				).repartition(4				)

    # Partitions 0 and 2
    lowerCamelCase__					:   Optional[Any]      				=	SparkExamplesIterable(_UpperCAmelCase				).shard_data_sources(worker_id=0		,			num_workers=2				)
    assert shard_it_a.n_shards == 2
    lowerCamelCase__					:   List[str]      				=	_get_expected_row_ids_and_row_dicts_for_partition_order(_UpperCAmelCase		,			[0, 2]				)
    for i, (row_id, row_dict) in enumerate(_UpperCAmelCase				):
        lowerCamelCase__				,							lowerCamelCase__					:   int      				=	expected_row_ids_and_row_dicts_a[i]
        assert row_id == expected_row_id
        assert row_dict == expected_row_dict

    # Partitions 1 and 3
    lowerCamelCase__					:   Dict      				=	SparkExamplesIterable(_UpperCAmelCase				).shard_data_sources(worker_id=1		,			num_workers=2				)
    assert shard_it_a.n_shards == 2
    lowerCamelCase__					:   Optional[int]      				=	_get_expected_row_ids_and_row_dicts_for_partition_order(_UpperCAmelCase		,			[1, 3]				)
    for i, (row_id, row_dict) in enumerate(_UpperCAmelCase				):
        lowerCamelCase__				,							lowerCamelCase__					:   Any      				=	expected_row_ids_and_row_dicts_a[i]
        assert row_id == expected_row_id
        assert row_dict == expected_row_dict


@require_not_windows
@require_dill_gt_0_3_2
def  SCREAMING_SNAKE_CASE   ( )    ->  Dict:
    lowerCamelCase__					:   int      				=	pyspark.sql.SparkSession.builder.master('local[*]'				).appName('pyspark'				).getOrCreate()
    lowerCamelCase__					:   str      				=	spark.range(100				).repartition(1				)
    lowerCamelCase__					:   Dict      				=	Spark(_UpperCAmelCase				)
    # Choose a small max_shard_size for maximum partitioning.
    spark_builder._repartition_df_if_needed(max_shard_size=1				)
    # The new number of partitions should not be greater than the number of rows.
    assert spark_builder.df.rdd.getNumPartitions() == 100





