

import unittest

from transformers import DebertaVaConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import (
					    DebertaVaForMaskedLM,
					    DebertaVaForMultipleChoice,
					    DebertaVaForQuestionAnswering,
					    DebertaVaForSequenceClassification,
					    DebertaVaForTokenClassification,
					    DebertaVaModel,
					)
					from transformers.models.deberta_va.modeling_deberta_va import DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST







class 				lowercase       ( lowercase_    ):




				def __init__(      self   ,     snake_case   ,     snake_case=13   ,     snake_case=7   ,     snake_case=True   ,     snake_case=True   ,     snake_case=True   ,     snake_case=True   ,     snake_case=99   ,     snake_case=32   ,     snake_case=5   ,     snake_case=4   ,     snake_case=37   ,     snake_case="gelu"   ,     snake_case=0.1   ,     snake_case=0.1   ,     snake_case=512   ,     snake_case=16   ,     snake_case=2   ,     snake_case=0.02   ,     snake_case=False   ,     snake_case=True   ,     snake_case="None"   ,     snake_case=3   ,     snake_case=4   ,     snake_case=None   ,     ):
											snake_case_           =					parent
											snake_case_           =					batch_size
											snake_case_           =					seq_length
											snake_case_           =					is_training
											snake_case_           =					use_input_mask
											snake_case_           =					use_token_type_ids
											snake_case_           =					use_labels
											snake_case_           =					vocab_size
											snake_case_           =					hidden_size
											snake_case_           =					num_hidden_layers
											snake_case_           =					num_attention_heads
											snake_case_           =					intermediate_size
											snake_case_           =					hidden_act
											snake_case_           =					hidden_dropout_prob
											snake_case_           =					attention_probs_dropout_prob
											snake_case_           =					max_position_embeddings
											snake_case_           =					type_vocab_size
											snake_case_           =					type_sequence_label_size
											snake_case_           =					initializer_range
											snake_case_           =					num_labels
											snake_case_           =					num_choices
											snake_case_           =					relative_attention
											snake_case_           =					position_biased_input
											snake_case_           =					pos_att_type
											snake_case_           =					scope




				def 					a							(      self ):
											snake_case_           =					ids_tensor([self.batch_size, self.seq_length]   ,     self.vocab_size )

											snake_case_           =					None
											if self.use_input_mask:
																		snake_case_           =					ids_tensor([self.batch_size, self.seq_length]   ,     vocab_size=2 )

											snake_case_           =					None
											if self.use_token_type_ids:
																		snake_case_           =					ids_tensor([self.batch_size, self.seq_length]   ,     self.type_vocab_size )

											snake_case_           =					None
											snake_case_           =					None
											snake_case_           =					None
											if self.use_labels:
																		snake_case_           =					ids_tensor([self.batch_size]   ,     self.type_sequence_label_size )
																		snake_case_           =					ids_tensor([self.batch_size, self.seq_length]   ,     self.num_labels )
																		snake_case_           =					ids_tensor([self.batch_size]   ,     self.num_choices )

											snake_case_           =					self.get_config()

											return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




				def 					a							(      self ):
											return DebertaVaConfig(
											    vocab_size=self.vocab_size   ,     hidden_size=self.hidden_size   ,     num_hidden_layers=self.num_hidden_layers   ,     num_attention_heads=self.num_attention_heads   ,     intermediate_size=self.intermediate_size   ,     hidden_act=self.hidden_act   ,     hidden_dropout_prob=self.hidden_dropout_prob   ,     attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,     max_position_embeddings=self.max_position_embeddings   ,     type_vocab_size=self.type_vocab_size   ,     initializer_range=self.initializer_range   ,     relative_attention=self.relative_attention   ,     position_biased_input=self.position_biased_input   ,     pos_att_type=self.pos_att_type   ,     )




				def 					a							(      self   ,     snake_case ):
											self.parent.assertListEqual(list(result.loss.size() )   ,     [] )




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case ):
											snake_case_           =					DebertaVaModel(config=snake_case )
											model.to(snake_case )
											model.eval()
											snake_case_           =					model(snake_case   ,     attention_mask=snake_case   ,     token_type_ids=snake_case )[0]
											snake_case_           =					model(snake_case   ,     token_type_ids=snake_case )[0]
											snake_case_           =					model(snake_case )[0]

											self.parent.assertListEqual(list(sequence_output.size() )   ,     [self.batch_size, self.seq_length, self.hidden_size] )




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case ):
											snake_case_           =					DebertaVaForMaskedLM(config=snake_case )
											model.to(snake_case )
											model.eval()
											snake_case_           =					model(snake_case   ,     attention_mask=snake_case   ,     token_type_ids=snake_case   ,     labels=snake_case )

											self.parent.assertEqual(result.logits.shape   ,     (self.batch_size, self.seq_length, self.vocab_size) )




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case ):
											snake_case_           =					self.num_labels
											snake_case_           =					DebertaVaForSequenceClassification(snake_case )
											model.to(snake_case )
											model.eval()
											snake_case_           =					model(snake_case   ,     attention_mask=snake_case   ,     token_type_ids=snake_case   ,     labels=snake_case )
											self.parent.assertListEqual(list(result.logits.size() )   ,     [self.batch_size, self.num_labels] )
											self.check_loss_output(snake_case )




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case ):
											snake_case_           =					self.num_labels
											snake_case_           =					DebertaVaForTokenClassification(config=snake_case )
											model.to(snake_case )
											model.eval()
											snake_case_           =					model(snake_case   ,     attention_mask=snake_case   ,     token_type_ids=snake_case   ,     labels=snake_case )
											self.parent.assertEqual(result.logits.shape   ,     (self.batch_size, self.seq_length, self.num_labels) )




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case ):
											snake_case_           =					DebertaVaForQuestionAnswering(config=snake_case )
											model.to(snake_case )
											model.eval()
											snake_case_           =					model(
											    snake_case   ,     attention_mask=snake_case   ,     token_type_ids=snake_case   ,     start_positions=snake_case   ,     end_positions=snake_case   ,     )
											self.parent.assertEqual(result.start_logits.shape   ,     (self.batch_size, self.seq_length) )
											self.parent.assertEqual(result.end_logits.shape   ,     (self.batch_size, self.seq_length) )




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case ):
											snake_case_           =					DebertaVaForMultipleChoice(config=snake_case )
											model.to(snake_case )
											model.eval()
											snake_case_           =					input_ids.unsqueeze(1 ).expand(-1   ,     self.num_choices   ,     -1 ).contiguous()
											snake_case_           =					token_type_ids.unsqueeze(1 ).expand(-1   ,     self.num_choices   ,     -1 ).contiguous()
											snake_case_           =					input_mask.unsqueeze(1 ).expand(-1   ,     self.num_choices   ,     -1 ).contiguous()
											snake_case_           =					model(
											    snake_case   ,     attention_mask=snake_case   ,     token_type_ids=snake_case   ,     labels=snake_case   ,     )
											self.parent.assertEqual(result.logits.shape   ,     (self.batch_size, self.num_choices) )




				def 					a							(      self ):
											snake_case_           =					self.prepare_config_and_inputs()
											(
											    (
											    snake_case_
											)       ,					(
											    snake_case_
											)       ,					(
											    snake_case_
											)       ,					(
											    snake_case_
											)       ,					(
											    snake_case_
											)       ,					(
											    snake_case_
											)       ,					(
											    snake_case_
											)       ,					
											)           =					config_and_inputs
											snake_case_           =					{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
											return config, inputs_dict







@require_torch
class 				lowercase       ( lowercase_			,						lowercase_			,						unittest.TestCase    ):
				__SCREAMING_SNAKE_CASE      :				List[Any]     =    (
				    (
				        DebertaVaModel,
				        DebertaVaForMaskedLM,
				        DebertaVaForSequenceClassification,
				        DebertaVaForTokenClassification,
				        DebertaVaForQuestionAnswering,
				        DebertaVaForMultipleChoice,
				    )
				    if is_torch_available()
				    else ()
				)
				__SCREAMING_SNAKE_CASE      :				Union[str, Any]     =    (
				    {
				        '''feature-extraction''': DebertaVaModel,
				        '''fill-mask''': DebertaVaForMaskedLM,
				        '''question-answering''': DebertaVaForQuestionAnswering,
				        '''text-classification''': DebertaVaForSequenceClassification,
				        '''token-classification''': DebertaVaForTokenClassification,
				        '''zero-shot''': DebertaVaForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)

				__SCREAMING_SNAKE_CASE      :				Dict     =    True
				__SCREAMING_SNAKE_CASE      :				List[Any]     =    False
				__SCREAMING_SNAKE_CASE      :				Optional[int]     =    False
				__SCREAMING_SNAKE_CASE      :				Dict     =    False
				__SCREAMING_SNAKE_CASE      :				Optional[Any]     =    False




				def 					a							(      self ):
											snake_case_           =					DebertaVaModelTester(self )
											snake_case_           =					ConfigTester(self   ,     config_class=snake_case   ,     hidden_size=37 )




				def 					a							(      self ):
											self.config_tester.run_common_tests()




				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_deberta_model(*snake_case )




				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_deberta_for_sequence_classification(*snake_case )




				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_deberta_for_masked_lm(*snake_case )




				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_deberta_for_question_answering(*snake_case )




				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_deberta_for_token_classification(*snake_case )




				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_deberta_for_multiple_choice(*snake_case )




				@slow
				def 					a							(      self ):
											for model_name in DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																		snake_case_           =					DebertaVaModel.from_pretrained(snake_case )
																		self.assertIsNotNone(snake_case )







@require_torch
@require_sentencepiece
@require_tokenizers
class 				lowercase       ( unittest.TestCase    ):




				@unittest.skip(reason='Model not available yet' )
				def 					a							(      self ):
											pass




				@slow
				def 					a							(      self ):
											snake_case_           =					DebertaVaModel.from_pretrained('microsoft/deberta-v2-xlarge' )

											snake_case_           =					torch.tensor([[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]] )
											snake_case_           =					torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]] )
											with torch.no_grad():
																		snake_case_           =					model(snake_case   ,     attention_mask=snake_case )[0]
											# compare the actual values for a slice.
											snake_case_           =					torch.tensor(
											    [[[0.23_56, 0.19_48, 0.03_69], [-0.10_63, 0.35_86, -0.51_52], [-0.63_99, -0.02_59, -0.25_25]]] )
											self.assertTrue(torch.allclose(output[:, 1:4, 1:4]   ,     snake_case   ,     atol=1e-4 )   ,     F'''{output[:, 1:4, 1:4]}''' )







import os
from typing import BinaryIO, Optional, Union

import numpy as np
import pyarrow.parquet as pq

from .. import Audio, Dataset, Features, Image, NamedSplit, Value, config
from ..features.features import FeatureType, _visit
from ..formatting import query_table
from ..packaged_modules import _PACKAGED_DATASETS_MODULES
from ..packaged_modules.parquet.parquet import Parquet
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader





def       __lowerCamelCase			(				UpperCamelCase__   ):







							'''simple docstring'''






							snake_case_           =					np.inf

							def set_batch_size(UpperCamelCase__   ) -> None:
														nonlocal batch_size
														if isinstance(UpperCamelCase__     ,							UpperCamelCase__   ):
																					snake_case_           =					min(UpperCamelCase__     ,							config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS   )
														elif isinstance(UpperCamelCase__     ,							UpperCamelCase__   ):
																					snake_case_           =					min(UpperCamelCase__     ,							config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS   )
														elif isinstance(UpperCamelCase__     ,							UpperCamelCase__   ) and feature.dtype == "binary":
																					snake_case_           =					min(UpperCamelCase__     ,							config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS   )

							_visit(UpperCamelCase__     ,							UpperCamelCase__   )

							return None if batch_size is np.inf else batch_size







class 				lowercase       ( lowercase_    ):




				def __init__(      self   ,     snake_case   ,     snake_case = None   ,     snake_case = None   ,     snake_case = None   ,     snake_case = False   ,     snake_case = False   ,     snake_case = None   ,     **snake_case   ,     ):
											super().__init__(
											    snake_case   ,     split=snake_case   ,     features=snake_case   ,     cache_dir=snake_case   ,     keep_in_memory=snake_case   ,     streaming=snake_case   ,     num_proc=snake_case   ,     **snake_case   ,     )
											snake_case_           =					path_or_paths if isinstance(snake_case   ,     snake_case ) else {self.split: path_or_paths}
											snake_case_           =					_PACKAGED_DATASETS_MODULES['parquet'][1]
											snake_case_           =					Parquet(
											    cache_dir=snake_case   ,     data_files=snake_case   ,     features=snake_case   ,     hash=snake_case   ,     **snake_case   ,     )




				def 					a							(      self ):
											# Build iterable dataset
											if self.streaming:
																		snake_case_           =					self.builder.as_streaming_dataset(split=self.split )
											# Build regular (map-style) dataset
											else:
																		snake_case_           =					None
																		snake_case_           =					None
																		snake_case_           =					None
																		snake_case_           =					None

																		self.builder.download_and_prepare(
																		    download_config=snake_case   ,     download_mode=snake_case   ,     verification_mode=snake_case   ,     base_path=snake_case   ,     num_proc=self.num_proc   ,     )
																		snake_case_           =					self.builder.as_dataset(
																		    split=self.split   ,     verification_mode=snake_case   ,     in_memory=self.keep_in_memory )
											return dataset







class 				lowercase       :




				def __init__(      self   ,     snake_case   ,     snake_case   ,     snake_case = None   ,     **snake_case   ,     ):
											snake_case_           =					dataset
											snake_case_           =					path_or_buf
											snake_case_           =					batch_size or get_writer_batch_size(dataset.features )
											snake_case_           =					parquet_writer_kwargs




				def 					a							(      self ):
											snake_case_           =					self.batch_size if self.batch_size else config.DEFAULT_MAX_BATCH_SIZE

											if isinstance(self.path_or_buf   ,     (str, bytes, os.PathLike) ):
																		with open(self.path_or_buf   ,     'wb+' ) as buffer:
																									snake_case_           =					self._write(file_obj=snake_case   ,     batch_size=snake_case   ,     **self.parquet_writer_kwargs )
											else:
																		snake_case_           =					self._write(file_obj=self.path_or_buf   ,     batch_size=snake_case   ,     **self.parquet_writer_kwargs )
											return written




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     **snake_case ):
											snake_case_           =					0
											snake_case_           =					parquet_writer_kwargs.pop('path_or_buf'   ,     snake_case )
											snake_case_           =					self.dataset.features.arrow_schema

											snake_case_           =					pq.ParquetWriter(snake_case   ,     schema=snake_case   ,     **snake_case )

											for offset in logging.tqdm(
											    range(0   ,     len(self.dataset )   ,     snake_case )   ,     unit='ba'   ,     disable=not logging.is_progress_bar_enabled()   ,     desc='Creating parquet from Arrow format'   ,     ):
																		snake_case_           =					query_table(
																		    table=self.dataset._data   ,     key=slice(snake_case   ,     offset + batch_size )   ,     indices=self.dataset._indices if self.dataset._indices is not None else None   ,     )
																		writer.write_table(snake_case )
																		written += batch.nbytes
											writer.close()
											return written




