
import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


lowercase   =	datasets.utils.logging.get_logger(__name__)





@dataclass
class    __A(					datasets.BuilderConfig							):
					SCREAMING_SNAKE_CASE           =     None
					SCREAMING_SNAKE_CASE           =     "utf-8"
					SCREAMING_SNAKE_CASE           =     None
					SCREAMING_SNAKE_CASE           =     None
					SCREAMING_SNAKE_CASE           =     True  # deprecated
					SCREAMING_SNAKE_CASE           =     None  # deprecated
					SCREAMING_SNAKE_CASE           =     1_0 << 2_0  # 10MB
					SCREAMING_SNAKE_CASE           =     None





class    __A(					datasets.ArrowBasedBuilder							):
					SCREAMING_SNAKE_CASE           =     JsonConfig

					def  lowercase__   (						self       :  Dict ):
												if self.config.block_size is not None:
																			logger.warning("""The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead""" )
																			lowerCamelCase_					=       self.config.block_size
												if self.config.use_threads is not True:
																			logger.warning(
																			    """The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore.""" )
												if self.config.newlines_in_values is not None:
																			raise ValueError("""The JSON loader parameter `newlines_in_values` is no longer supported""" )
												return datasets.DatasetInfo(features=self.config.features )

					def  lowercase__   (						self       :  Optional[int] ,       __UpperCamelCase       :  int ):
												if not self.config.data_files:
																			raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}''' )
												lowerCamelCase_					=       dl_manager.download_and_extract(self.config.data_files )
												if isinstance(__UpperCamelCase ,       (str, list, tuple) ):
																			lowerCamelCase_					=       data_files
																			if isinstance(__UpperCamelCase ,       __UpperCamelCase ):
																										lowerCamelCase_					=       [files]
																			lowerCamelCase_					=       [dl_manager.iter_files(__UpperCamelCase ) for file in files]
																			return [datasets.SplitGenerator(name=datasets.Split.TRAIN ,       gen_kwargs={"""files""": files} )]
												lowerCamelCase_					=       []
												for split_name, files in data_files.items():
																			if isinstance(__UpperCamelCase ,       __UpperCamelCase ):
																										lowerCamelCase_					=       [files]
																			lowerCamelCase_					=       [dl_manager.iter_files(__UpperCamelCase ) for file in files]
																			splits.append(datasets.SplitGenerator(name=__UpperCamelCase ,       gen_kwargs={"""files""": files} ) )
												return splits

					def  lowercase__   (						self       :  Any ,       __UpperCamelCase       :  pa.Table ):
												if self.config.features is not None:
																			# adding missing columns
																			for column_name in set(self.config.features ) - set(pa_table.column_names ):
																										lowerCamelCase_					=       self.config.features.arrow_schema.field(__UpperCamelCase ).type
																										lowerCamelCase_					=       pa_table.append_column(__UpperCamelCase ,       pa.array([None] * len(__UpperCamelCase ) ,       type=__UpperCamelCase ) )
																			# more expensive cast to support nested structures with keys in a different order
																			# allows str <-> int/float or str to Audio for example
																			lowerCamelCase_					=       table_cast(__UpperCamelCase ,       self.config.features.arrow_schema )
												return pa_table



					def  lowercase__   (						self       :  Union[str, Any] ,       __UpperCamelCase       :  Optional[Any] ):
												for file_idx, file in enumerate(itertools.chain.from_iterable(__UpperCamelCase ) ):
																			# If the file is one json object and if we need to look at the list of items in one specific field
																			if self.config.field is not None:
																										with open(__UpperCamelCase ,       encoding=self.config.encoding ,       errors=self.config.encoding_errors ) as f:
																																	lowerCamelCase_					=       json.load(__UpperCamelCase )

																										# We keep only the field we are interested in
																										lowerCamelCase_					=       dataset[self.config.field]

																										# We accept two format: a list of dicts or a dict of lists
																										if isinstance(__UpperCamelCase ,       (list, tuple) ):
																																	lowerCamelCase_					=       set().union(*[row.keys() for row in dataset] )
																																	lowerCamelCase_					=       {col: [row.get(__UpperCamelCase ) for row in dataset] for col in keys}
																										else:
																																	lowerCamelCase_					=       dataset
																										lowerCamelCase_					=       pa.Table.from_pydict(__UpperCamelCase )
																										yield file_idx, self._cast_table(__UpperCamelCase )

																			# If the file has one json object per line
																			else:
																										with open(__UpperCamelCase ,       """rb""" ) as f:
																																	lowerCamelCase_					=       0
																																	# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																																	# Set a default minimum value of 16kB if the chunk size is really small
																																	lowerCamelCase_					=       max(self.config.chunksize // 3_2 ,       1_6 << 1_0 )
																																	lowerCamelCase_					=       (
																																	    self.config.encoding_errors if self.config.encoding_errors is not None else """strict"""
																																	)
																																	while True:
																																								lowerCamelCase_					=       f.read(self.config.chunksize )
																																								if not batch:
																																															break
																																								# Finish current line
																																								try:
																																															batch += f.readline()
																																								except (AttributeError, io.UnsupportedOperation):
																																															batch += readline(__UpperCamelCase )
																																								# PyArrow only accepts utf-8 encoded bytes
																																								if self.config.encoding != "utf-8":
																																															lowerCamelCase_					=       batch.decode(self.config.encoding ,       errors=__UpperCamelCase ).encode("""utf-8""" )
																																								try:
																																															while True:
																																																						try:
																																																													lowerCamelCase_					=       paj.read_json(
																																																													    io.BytesIO(__UpperCamelCase ) ,       read_options=paj.ReadOptions(block_size=__UpperCamelCase ) )
																																																													break
																																																						except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																																													if (
																																																													    isinstance(__UpperCamelCase ,       pa.ArrowInvalid )
																																																													    and "straddling" not in str(__UpperCamelCase )
																																																													    or block_size > len(__UpperCamelCase )
																																																													):
																																																																				raise
																																																													else:
																																																																				# Increase the block size in case it was too small.
																																																																				# The block size will be reset for the next file.
																																																																				logger.debug(
																																																																				    F'''Batch of {len(__UpperCamelCase )} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.''' )
																																																																				block_size *= 2
																																								except pa.ArrowInvalid as e:
																																															try:
																																																						with open(
																																																						    __UpperCamelCase ,       encoding=self.config.encoding ,       errors=self.config.encoding_errors ) as f:
																																																													lowerCamelCase_					=       json.load(__UpperCamelCase )
																																															except json.JSONDecodeError:
																																																						logger.error(F'''Failed to read file \'{file}\' with error {type(__UpperCamelCase )}: {e}''' )
																																																						raise e
																																															# If possible, parse the file as a list of json objects and exit the loop
																																															if isinstance(__UpperCamelCase ,       __UpperCamelCase ):  # list is the only sequence type supported in JSON
																																																						try:
																																																													lowerCamelCase_					=       set().union(*[row.keys() for row in dataset] )
																																																													lowerCamelCase_					=       {col: [row.get(__UpperCamelCase ) for row in dataset] for col in keys}
																																																													lowerCamelCase_					=       pa.Table.from_pydict(__UpperCamelCase )
																																																						except (pa.ArrowInvalid, AttributeError) as e:
																																																													logger.error(F'''Failed to read file \'{file}\' with error {type(__UpperCamelCase )}: {e}''' )
																																																													raise ValueError(F'''Not able to read records in the JSON file at {file}.''' ) from None
																																																						yield file_idx, self._cast_table(__UpperCamelCase )
																																																						break
																																															else:
																																																						logger.error(F'''Failed to read file \'{file}\' with error {type(__UpperCamelCase )}: {e}''' )
																																																						raise ValueError(
																																																						    F'''Not able to read records in the JSON file at {file}. '''
																																																						    F'''You should probably indicate the field of the JSON file containing your records. '''
																																																						    F'''This JSON file contain the following fields: {str(list(dataset.keys() ) )}. '''
																																																						    F'''Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. ''' ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																								yield (file_idx, batch_idx), self._cast_table(__UpperCamelCase )
																																								batch_idx += 1







from __future__ import annotations

import unittest

from transformers import DebertaVaConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers import (
		    TFDebertaVaForMaskedLM,
		    TFDebertaVaForQuestionAnswering,
		    TFDebertaVaForSequenceClassification,
		    TFDebertaVaForTokenClassification,
		    TFDebertaVaModel,
		)





class    __A:

					def __init__(						self       :  List[Any] ,       __UpperCamelCase       :  Tuple ,       __UpperCamelCase       :  Any=1_3 ,       __UpperCamelCase       :  Optional[int]=7 ,       __UpperCamelCase       :  Any=True ,       __UpperCamelCase       :  Dict=True ,       __UpperCamelCase       :  Optional[Any]=True ,       __UpperCamelCase       :  Dict=True ,       __UpperCamelCase       :  Union[str, Any]=9_9 ,       __UpperCamelCase       :  List[str]=3_2 ,       __UpperCamelCase       :  int=2 ,       __UpperCamelCase       :  Optional[Any]=4 ,       __UpperCamelCase       :  Optional[int]=3_7 ,       __UpperCamelCase       :  Optional[Any]="gelu" ,       __UpperCamelCase       :  int=0.1 ,       __UpperCamelCase       :  str=0.1 ,       __UpperCamelCase       :  Tuple=5_1_2 ,       __UpperCamelCase       :  Any=1_6 ,       __UpperCamelCase       :  Dict=2 ,       __UpperCamelCase       :  Optional[Any]=0.02 ,       __UpperCamelCase       :  Optional[Any]=False ,       __UpperCamelCase       :  Dict=True ,       __UpperCamelCase       :  Any="None" ,       __UpperCamelCase       :  List[Any]=3 ,       __UpperCamelCase       :  Optional[int]=4 ,       __UpperCamelCase       :  Dict=None ,       ):
												lowerCamelCase_					=       parent
												lowerCamelCase_					=       batch_size
												lowerCamelCase_					=       seq_length
												lowerCamelCase_					=       is_training
												lowerCamelCase_					=       use_input_mask
												lowerCamelCase_					=       use_token_type_ids
												lowerCamelCase_					=       use_labels
												lowerCamelCase_					=       vocab_size
												lowerCamelCase_					=       hidden_size
												lowerCamelCase_					=       num_hidden_layers
												lowerCamelCase_					=       num_attention_heads
												lowerCamelCase_					=       intermediate_size
												lowerCamelCase_					=       hidden_act
												lowerCamelCase_					=       hidden_dropout_prob
												lowerCamelCase_					=       attention_probs_dropout_prob
												lowerCamelCase_					=       max_position_embeddings
												lowerCamelCase_					=       type_vocab_size
												lowerCamelCase_					=       type_sequence_label_size
												lowerCamelCase_					=       initializer_range
												lowerCamelCase_					=       num_labels
												lowerCamelCase_					=       num_choices
												lowerCamelCase_					=       relative_attention
												lowerCamelCase_					=       position_biased_input
												lowerCamelCase_					=       pos_att_type
												lowerCamelCase_					=       scope

					def  lowercase__   (						self       :  Optional[Any] ):
												lowerCamelCase_					=       ids_tensor([self.batch_size, self.seq_length] ,       self.vocab_size )

												lowerCamelCase_					=       None
												if self.use_input_mask:
																			lowerCamelCase_					=       random_attention_mask([self.batch_size, self.seq_length] )

												lowerCamelCase_					=       None
												if self.use_token_type_ids:
																			lowerCamelCase_					=       ids_tensor([self.batch_size, self.seq_length] ,       self.type_vocab_size )

												lowerCamelCase_					=       None
												lowerCamelCase_					=       None
												lowerCamelCase_					=       None
												if self.use_labels:
																			lowerCamelCase_					=       ids_tensor([self.batch_size] ,       self.type_sequence_label_size )
																			lowerCamelCase_					=       ids_tensor([self.batch_size, self.seq_length] ,       self.num_labels )

												lowerCamelCase_					=       DebertaVaConfig(
												    vocab_size=self.vocab_size ,       hidden_size=self.hidden_size ,       num_hidden_layers=self.num_hidden_layers ,       num_attention_heads=self.num_attention_heads ,       intermediate_size=self.intermediate_size ,       hidden_act=self.hidden_act ,       hidden_dropout_prob=self.hidden_dropout_prob ,       attention_probs_dropout_prob=self.attention_probs_dropout_prob ,       max_position_embeddings=self.max_position_embeddings ,       type_vocab_size=self.type_vocab_size ,       relative_attention=self.relative_attention ,       position_biased_input=self.position_biased_input ,       initializer_range=self.initializer_range ,       return_dict=__UpperCamelCase ,       )

												return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

					def  lowercase__   (						self       :  str ,       __UpperCamelCase       :  Union[str, Any] ,       __UpperCamelCase       :  Optional[int] ,       __UpperCamelCase       :  Optional[Any] ,       __UpperCamelCase       :  Optional[int] ,       __UpperCamelCase       :  Any ,       __UpperCamelCase       :  int ,       __UpperCamelCase       :  List[str] ):
												lowerCamelCase_					=       TFDebertaVaModel(config=__UpperCamelCase )
												lowerCamelCase_					=       {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}

												lowerCamelCase_					=       [input_ids, input_mask]
												lowerCamelCase_					=       model(__UpperCamelCase )

												lowerCamelCase_					=       model(__UpperCamelCase )

												self.parent.assertEqual(result.last_hidden_state.shape ,       (self.batch_size, self.seq_length, self.hidden_size) )

					def  lowercase__   (						self       :  Optional[int] ,       __UpperCamelCase       :  List[str] ,       __UpperCamelCase       :  str ,       __UpperCamelCase       :  Dict ,       __UpperCamelCase       :  str ,       __UpperCamelCase       :  List[str] ,       __UpperCamelCase       :  Tuple ,       __UpperCamelCase       :  int ):
												lowerCamelCase_					=       TFDebertaVaForMaskedLM(config=__UpperCamelCase )
												lowerCamelCase_					=       {
												    """input_ids""": input_ids,
												    """attention_mask""": input_mask,
												    """token_type_ids""": token_type_ids,
												}
												lowerCamelCase_					=       model(__UpperCamelCase )
												self.parent.assertEqual(result.logits.shape ,       (self.batch_size, self.seq_length, self.vocab_size) )

					def  lowercase__   (						self       :  List[Any] ,       __UpperCamelCase       :  Union[str, Any] ,       __UpperCamelCase       :  str ,       __UpperCamelCase       :  int ,       __UpperCamelCase       :  Optional[int] ,       __UpperCamelCase       :  List[Any] ,       __UpperCamelCase       :  str ,       __UpperCamelCase       :  List[Any] ):
												lowerCamelCase_					=       self.num_labels
												lowerCamelCase_					=       TFDebertaVaForSequenceClassification(config=__UpperCamelCase )
												lowerCamelCase_					=       {
												    """input_ids""": input_ids,
												    """attention_mask""": input_mask,
												    """token_type_ids""": token_type_ids,
												}

												lowerCamelCase_					=       model(__UpperCamelCase )
												self.parent.assertEqual(result.logits.shape ,       (self.batch_size, self.num_labels) )

					def  lowercase__   (						self       :  int ,       __UpperCamelCase       :  Tuple ,       __UpperCamelCase       :  Any ,       __UpperCamelCase       :  str ,       __UpperCamelCase       :  Optional[int] ,       __UpperCamelCase       :  List[str] ,       __UpperCamelCase       :  List[Any] ,       __UpperCamelCase       :  List[Any] ):
												lowerCamelCase_					=       self.num_labels
												lowerCamelCase_					=       TFDebertaVaForTokenClassification(config=__UpperCamelCase )
												lowerCamelCase_					=       {
												    """input_ids""": input_ids,
												    """attention_mask""": input_mask,
												    """token_type_ids""": token_type_ids,
												}
												lowerCamelCase_					=       model(__UpperCamelCase )
												self.parent.assertEqual(result.logits.shape ,       (self.batch_size, self.seq_length, self.num_labels) )

					def  lowercase__   (						self       :  List[Any] ,       __UpperCamelCase       :  Optional[int] ,       __UpperCamelCase       :  Optional[Any] ,       __UpperCamelCase       :  Optional[int] ,       __UpperCamelCase       :  str ,       __UpperCamelCase       :  Optional[Any] ,       __UpperCamelCase       :  List[str] ,       __UpperCamelCase       :  int ):
												lowerCamelCase_					=       TFDebertaVaForQuestionAnswering(config=__UpperCamelCase )
												lowerCamelCase_					=       {
												    """input_ids""": input_ids,
												    """attention_mask""": input_mask,
												    """token_type_ids""": token_type_ids,
												}

												lowerCamelCase_					=       model(__UpperCamelCase )
												self.parent.assertEqual(result.start_logits.shape ,       (self.batch_size, self.seq_length) )
												self.parent.assertEqual(result.end_logits.shape ,       (self.batch_size, self.seq_length) )



					def  lowercase__   (						self       :  Dict ):
												lowerCamelCase_					=       self.prepare_config_and_inputs()
												(
												    (
												    lowerCamelCase_
												)	, (
												    lowerCamelCase_
												)	, (
												    lowerCamelCase_
												)	, (
												    lowerCamelCase_
												)	, (
												    lowerCamelCase_
												)	, (
												    lowerCamelCase_
												)	, (
												    lowerCamelCase_
												)	, 
												)					=       config_and_inputs
												lowerCamelCase_					=       {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
												return config, inputs_dict





@require_tf
class    __A(					UpperCAmelCase  ,		UpperCAmelCase  ,		unittest.TestCase							):
					SCREAMING_SNAKE_CASE           =     (
					    (
					        TFDebertaVaModel,
					        TFDebertaVaForMaskedLM,
					        TFDebertaVaForQuestionAnswering,
					        TFDebertaVaForSequenceClassification,
					        TFDebertaVaForTokenClassification,
					    )
					    if is_tf_available()
					    else ()
					)
					SCREAMING_SNAKE_CASE           =     (
					    {
					        '''feature-extraction''': TFDebertaVaModel,
					        '''fill-mask''': TFDebertaVaForMaskedLM,
					        '''question-answering''': TFDebertaVaForQuestionAnswering,
					        '''text-classification''': TFDebertaVaForSequenceClassification,
					        '''token-classification''': TFDebertaVaForTokenClassification,
					        '''zero-shot''': TFDebertaVaForSequenceClassification,
					    }
					    if is_tf_available()
					    else {}
					)

					SCREAMING_SNAKE_CASE           =     False
					SCREAMING_SNAKE_CASE           =     False

					def  lowercase__   (						self       :  Dict ):
												lowerCamelCase_					=       TFDebertaVaModelTester(self )
												lowerCamelCase_					=       ConfigTester(self ,       config_class=__UpperCamelCase ,       hidden_size=3_7 )

					def  lowercase__   (						self       :  Dict ):
												self.config_tester.run_common_tests()

					def  lowercase__   (						self       :  Optional[Any] ):
												lowerCamelCase_					=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_model(*__UpperCamelCase )

					def  lowercase__   (						self       :  Dict ):
												lowerCamelCase_					=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_masked_lm(*__UpperCamelCase )

					def  lowercase__   (						self       :  List[Any] ):
												lowerCamelCase_					=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_question_answering(*__UpperCamelCase )

					def  lowercase__   (						self       :  Tuple ):
												lowerCamelCase_					=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_sequence_classification(*__UpperCamelCase )

					def  lowercase__   (						self       :  Any ):
												lowerCamelCase_					=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_token_classification(*__UpperCamelCase )



					@slow
					def  lowercase__   (						self       :  int ):
												lowerCamelCase_					=       TFDebertaVaModel.from_pretrained("""kamalkraj/deberta-v2-xlarge""" )
												self.assertIsNotNone(__UpperCamelCase )





@require_tf
class    __A(					unittest.TestCase							):

					@unittest.skip(reason="""Model not available yet""" )
					def  lowercase__   (						self       :  Optional[int] ):
												pass



					@slow
					def  lowercase__   (						self       :  Any ):
												lowerCamelCase_					=       TFDebertaVaModel.from_pretrained("""kamalkraj/deberta-v2-xlarge""" )
												lowerCamelCase_					=       tf.constant([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]] )
												lowerCamelCase_					=       tf.constant([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]] )
												lowerCamelCase_					=       model(__UpperCamelCase ,       attention_mask=__UpperCamelCase )[0]

												lowerCamelCase_					=       tf.constant(
												    [[[0.2356, 0.1948, 0.0369], [-0.1063, 0.3586, -0.5152], [-0.6399, -0.0259, -0.2525]]] )
												tf.debugging.assert_near(output[:, 1:4, 1:4] ,       __UpperCamelCase ,       atol=1E-4 )





