
# Lint as: python3
import os
import re
import urllib.parse
from pathlib import Path
from typing import Callable, List, Optional, Union
from zipfile import ZipFile

from ..utils.file_utils import cached_path, hf_github_url
from ..utils.logging import get_logger
from ..utils.version import Version


lowercase   =	get_logger(__name__)





class    __A:
					SCREAMING_SNAKE_CASE           =     '''dummy_data'''
					SCREAMING_SNAKE_CASE           =     '''datasets'''
					SCREAMING_SNAKE_CASE           =     False

					def __init__(						self       :  Union[str, Any] ,       __UpperCamelCase       :  str ,       __UpperCamelCase       :  str ,       __UpperCamelCase       :  Union[Version, str] ,       __UpperCamelCase       :  Optional[str] = None ,       __UpperCamelCase       :  bool = False ,       __UpperCamelCase       :  bool = True ,       __UpperCamelCase       :  Optional[List[Callable]] = None ,       ):
												lowerCamelCase_					=       0
												lowerCamelCase_					=       dataset_name
												lowerCamelCase_					=       cache_dir
												lowerCamelCase_					=       use_local_dummy_data
												lowerCamelCase_					=       config
												# download_callbacks take a single url as input
												lowerCamelCase_					=       download_callbacks or []
												# if False, it doesn't load existing files and it returns the paths of the dummy files relative
												# to the dummy_data zip file root
												lowerCamelCase_					=       load_existing_dummy_data

												# TODO(PVP, QL) might need to make this more general
												lowerCamelCase_					=       str(__UpperCamelCase )
												# to be downloaded
												lowerCamelCase_					=       None
												lowerCamelCase_					=       None

					@property
					def  lowercase__   (						self       :  List[str] ):
												if self._dummy_file is None:
																			lowerCamelCase_					=       self.download_dummy_data()
												return self._dummy_file

					@property
					def  lowercase__   (						self       :  str ):
												if self.config is not None:
																			# structure is dummy / config_name / version_name
																			return os.path.join("""dummy""" ,       self.config.name ,       self.version_name )
												# structure is dummy / version_name
												return os.path.join("""dummy""" ,       self.version_name )

					@property
					def  lowercase__   (						self       :  List[str] ):
												return os.path.join(self.dummy_data_folder ,       """dummy_data.zip""" )

					def  lowercase__   (						self       :  int ):
												lowerCamelCase_					=       (
												    self.local_path_to_dummy_data if self.use_local_dummy_data is True else self.github_path_to_dummy_data
												)

												lowerCamelCase_					=       cached_path(
												    __UpperCamelCase ,       cache_dir=self.cache_dir ,       extract_compressed_file=__UpperCamelCase ,       force_extract=__UpperCamelCase )

												return os.path.join(__UpperCamelCase ,       self.dummy_file_name )

					@property
					def  lowercase__   (						self       :  int ):
												return os.path.join(self.datasets_scripts_dir ,       self.dataset_name ,       self.dummy_zip_file )

					@property
					def  lowercase__   (						self       :  int ):
												if self._bucket_url is None:
																			lowerCamelCase_					=       hf_github_url(self.dataset_name ,       self.dummy_zip_file.replace(os.sep ,       """/""" ) )
												return self._bucket_url

					@property
					def  lowercase__   (						self       :  Optional[Any] ):
												# return full path if its a dir
												if os.path.isdir(self.dummy_file ):
																			return self.dummy_file
												# else cut off path to file -> example `xsum`.
												return "/".join(self.dummy_file.replace(os.sep ,       """/""" ).split("""/""" )[:-1] )

					def  lowercase__   (						self       :  Any ,       __UpperCamelCase       :  Tuple ,       *__UpperCamelCase       :  List[Any] ):
												if self.load_existing_dummy_data:
																			# dummy data is downloaded and tested
																			lowerCamelCase_					=       self.dummy_file
												else:
																			# dummy data cannot be downloaded and only the path to dummy file is returned
																			lowerCamelCase_					=       self.dummy_file_name

												# special case when data_url is a dict
												if isinstance(__UpperCamelCase ,       __UpperCamelCase ):
																			return self.create_dummy_data_dict(__UpperCamelCase ,       __UpperCamelCase )
												elif isinstance(__UpperCamelCase ,       (list, tuple) ):
																			return self.create_dummy_data_list(__UpperCamelCase ,       __UpperCamelCase )
												else:
																			return self.create_dummy_data_single(__UpperCamelCase ,       __UpperCamelCase )

					def  lowercase__   (						self       :  Union[str, Any] ,       __UpperCamelCase       :  Tuple ,       *__UpperCamelCase       :  int ):
												return self.download_and_extract(__UpperCamelCase )

					def  lowercase__   (						self       :  Optional[int] ,       __UpperCamelCase       :  Optional[Any] ,       __UpperCamelCase       :  int ):
												return self.download_and_extract(__UpperCamelCase )

					def  lowercase__   (						self       :  Any ,       __UpperCamelCase       :  List[str] ,       *__UpperCamelCase       :  int ,       **__UpperCamelCase       :  int ):
												return path

					def  lowercase__   (						self       :  Union[str, Any] ):
												return {}

					def  lowercase__   (						self       :  Tuple ,       __UpperCamelCase       :  Union[str, Any] ,       __UpperCamelCase       :  int ):
												lowerCamelCase_					=       {}
												for key, single_urls in data_url.items():
																			for download_callback in self.download_callbacks:
																										if isinstance(__UpperCamelCase ,       __UpperCamelCase ):
																																	for single_url in single_urls:
																																								download_callback(__UpperCamelCase )
																										else:
																																	lowerCamelCase_					=       single_urls
																																	download_callback(__UpperCamelCase )
            # we force the name of each key to be the last file / folder name of the url path
            # if the url has arguments, we need to encode them with urllib.parse.quote_plus
																			if isinstance(__UpperCamelCase ,       __UpperCamelCase ):
																										lowerCamelCase_					=       [os.path.join(__UpperCamelCase ,       urllib.parse.quote_plus(Path(__UpperCamelCase ).name ) ) for x in single_urls]
																			else:
																										lowerCamelCase_					=       single_urls
																										lowerCamelCase_					=       os.path.join(__UpperCamelCase ,       urllib.parse.quote_plus(Path(__UpperCamelCase ).name ) )
																			lowerCamelCase_					=       value

												# make sure that values are unique
												if all(isinstance(__UpperCamelCase ,       __UpperCamelCase ) for i in dummy_data_dict.values() ) and len(set(dummy_data_dict.values() ) ) < len(
												    dummy_data_dict.values() ):
																			# append key to value to make its name unique
																			lowerCamelCase_					=       {key: value + key for key, value in dummy_data_dict.items()}

												return dummy_data_dict

					def  lowercase__   (						self       :  Union[str, Any] ,       __UpperCamelCase       :  List[str] ,       __UpperCamelCase       :  List[str] ):
												lowerCamelCase_					=       []
												# trick: if there are many shards named like `data.txt-000001-of-00300`, only use the first one
												lowerCamelCase_					=       all(bool(re.findall("""[0-9]{3,}-of-[0-9]{3,}""" ,       __UpperCamelCase ) ) for url in data_url )
												lowerCamelCase_					=       all(
												    url.startswith("""https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/pubmed""" ) for url in data_url )
												if data_url and (is_tf_records or is_pubmed_records):
																			lowerCamelCase_					=       [data_url[0]] * len(__UpperCamelCase )
												for single_url in data_url:
																			for download_callback in self.download_callbacks:
																										download_callback(__UpperCamelCase )
																			# we force the name of each key to be the last file / folder name of the url path
																			# if the url has arguments, we need to encode them with urllib.parse.quote_plus
																			lowerCamelCase_					=       os.path.join(__UpperCamelCase ,       urllib.parse.quote_plus(single_url.split("""/""" )[-1] ) )
																			dummy_data_list.append(__UpperCamelCase )
												return dummy_data_list

					def  lowercase__   (						self       :  Optional[int] ,       __UpperCamelCase       :  Optional[int] ,       __UpperCamelCase       :  Optional[int] ):
												for download_callback in self.download_callbacks:
																			download_callback(__UpperCamelCase )
												# we force the name of each key to be the last file / folder name of the url path
												# if the url has arguments, we need to encode them with urllib.parse.quote_plus
												lowerCamelCase_					=       os.path.join(__UpperCamelCase ,       urllib.parse.quote_plus(data_url.split("""/""" )[-1] ) )
												if os.path.exists(__UpperCamelCase ) or not self.load_existing_dummy_data:
																			return value
												else:
																			# Backward compatibility, maybe deprecate at one point.
																			# For many datasets with single url calls to dl_manager.download_and_extract,
																			# the dummy_data.zip file is actually the zipped downloaded file
																			# while now we expected the dummy_data.zip file to be a directory containing
																			# the downloaded file.
																			return path_to_dummy_data

					def  lowercase__   (						self       :  Union[str, Any] ):
												pass

					def  lowercase__   (						self       :  Tuple ):
												pass

					def  lowercase__   (						self       :  Union[str, Any] ,       __UpperCamelCase       :  Any ):
												def _iter_archive_members(__UpperCamelCase       :  Any ):
																			# this preserves the order of the members inside the ZIP archive
																			lowerCamelCase_					=       Path(self.dummy_file ).parent
																			lowerCamelCase_					=       path.relative_to(__UpperCamelCase )
																			with ZipFile(self.local_path_to_dummy_data ) as zip_file:
																										lowerCamelCase_					=       zip_file.namelist()
																			for member in members:
																										if member.startswith(relative_path.as_posix() ):
																																	yield dummy_parent_path.joinpath(__UpperCamelCase )

												lowerCamelCase_					=       Path(__UpperCamelCase )
												lowerCamelCase_					=       _iter_archive_members(__UpperCamelCase ) if self.use_local_dummy_data else path.rglob("""*""" )
												for file_path in file_paths:
																			if file_path.is_file() and not file_path.name.startswith((""".""", """__""") ):
																										yield file_path.relative_to(__UpperCamelCase ).as_posix(), file_path.open("""rb""" )



					def  lowercase__   (						self       :  str ,       __UpperCamelCase       :  Union[str, Any] ):
												if not isinstance(__UpperCamelCase ,       __UpperCamelCase ):
																			lowerCamelCase_					=       [paths]
												for path in paths:
																			if os.path.isfile(__UpperCamelCase ):
																										if os.path.basename(__UpperCamelCase ).startswith((""".""", """__""") ):
																																	return
																										yield path
																			else:
																										for dirpath, dirnames, filenames in os.walk(__UpperCamelCase ):
																																	if os.path.basename(__UpperCamelCase ).startswith((""".""", """__""") ):
																																								continue
																																	dirnames.sort()
																																	for filename in sorted(__UpperCamelCase ):
																																								if filename.startswith((""".""", """__""") ):
																																															continue
																																								yield os.path.join(__UpperCamelCase ,       __UpperCamelCase )







from __future__ import annotations

import unittest

from transformers import DebertaVaConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers import (
		    TFDebertaVaForMaskedLM,
		    TFDebertaVaForQuestionAnswering,
		    TFDebertaVaForSequenceClassification,
		    TFDebertaVaForTokenClassification,
		    TFDebertaVaModel,
		)





class    __A:

					def __init__(						self       :  List[Any] ,       __UpperCamelCase       :  Tuple ,       __UpperCamelCase       :  Any=1_3 ,       __UpperCamelCase       :  Optional[int]=7 ,       __UpperCamelCase       :  Any=True ,       __UpperCamelCase       :  Dict=True ,       __UpperCamelCase       :  Optional[Any]=True ,       __UpperCamelCase       :  Dict=True ,       __UpperCamelCase       :  Union[str, Any]=9_9 ,       __UpperCamelCase       :  List[str]=3_2 ,       __UpperCamelCase       :  int=2 ,       __UpperCamelCase       :  Optional[Any]=4 ,       __UpperCamelCase       :  Optional[int]=3_7 ,       __UpperCamelCase       :  Optional[Any]="gelu" ,       __UpperCamelCase       :  int=0.1 ,       __UpperCamelCase       :  str=0.1 ,       __UpperCamelCase       :  Tuple=5_1_2 ,       __UpperCamelCase       :  Any=1_6 ,       __UpperCamelCase       :  Dict=2 ,       __UpperCamelCase       :  Optional[Any]=0.02 ,       __UpperCamelCase       :  Optional[Any]=False ,       __UpperCamelCase       :  Dict=True ,       __UpperCamelCase       :  Any="None" ,       __UpperCamelCase       :  List[Any]=3 ,       __UpperCamelCase       :  Optional[int]=4 ,       __UpperCamelCase       :  Dict=None ,       ):
												lowerCamelCase_					=       parent
												lowerCamelCase_					=       batch_size
												lowerCamelCase_					=       seq_length
												lowerCamelCase_					=       is_training
												lowerCamelCase_					=       use_input_mask
												lowerCamelCase_					=       use_token_type_ids
												lowerCamelCase_					=       use_labels
												lowerCamelCase_					=       vocab_size
												lowerCamelCase_					=       hidden_size
												lowerCamelCase_					=       num_hidden_layers
												lowerCamelCase_					=       num_attention_heads
												lowerCamelCase_					=       intermediate_size
												lowerCamelCase_					=       hidden_act
												lowerCamelCase_					=       hidden_dropout_prob
												lowerCamelCase_					=       attention_probs_dropout_prob
												lowerCamelCase_					=       max_position_embeddings
												lowerCamelCase_					=       type_vocab_size
												lowerCamelCase_					=       type_sequence_label_size
												lowerCamelCase_					=       initializer_range
												lowerCamelCase_					=       num_labels
												lowerCamelCase_					=       num_choices
												lowerCamelCase_					=       relative_attention
												lowerCamelCase_					=       position_biased_input
												lowerCamelCase_					=       pos_att_type
												lowerCamelCase_					=       scope

					def  lowercase__   (						self       :  Optional[Any] ):
												lowerCamelCase_					=       ids_tensor([self.batch_size, self.seq_length] ,       self.vocab_size )

												lowerCamelCase_					=       None
												if self.use_input_mask:
																			lowerCamelCase_					=       random_attention_mask([self.batch_size, self.seq_length] )

												lowerCamelCase_					=       None
												if self.use_token_type_ids:
																			lowerCamelCase_					=       ids_tensor([self.batch_size, self.seq_length] ,       self.type_vocab_size )

												lowerCamelCase_					=       None
												lowerCamelCase_					=       None
												lowerCamelCase_					=       None
												if self.use_labels:
																			lowerCamelCase_					=       ids_tensor([self.batch_size] ,       self.type_sequence_label_size )
																			lowerCamelCase_					=       ids_tensor([self.batch_size, self.seq_length] ,       self.num_labels )

												lowerCamelCase_					=       DebertaVaConfig(
												    vocab_size=self.vocab_size ,       hidden_size=self.hidden_size ,       num_hidden_layers=self.num_hidden_layers ,       num_attention_heads=self.num_attention_heads ,       intermediate_size=self.intermediate_size ,       hidden_act=self.hidden_act ,       hidden_dropout_prob=self.hidden_dropout_prob ,       attention_probs_dropout_prob=self.attention_probs_dropout_prob ,       max_position_embeddings=self.max_position_embeddings ,       type_vocab_size=self.type_vocab_size ,       relative_attention=self.relative_attention ,       position_biased_input=self.position_biased_input ,       initializer_range=self.initializer_range ,       return_dict=__UpperCamelCase ,       )

												return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

					def  lowercase__   (						self       :  str ,       __UpperCamelCase       :  Union[str, Any] ,       __UpperCamelCase       :  Optional[int] ,       __UpperCamelCase       :  Optional[Any] ,       __UpperCamelCase       :  Optional[int] ,       __UpperCamelCase       :  Any ,       __UpperCamelCase       :  int ,       __UpperCamelCase       :  List[str] ):
												lowerCamelCase_					=       TFDebertaVaModel(config=__UpperCamelCase )
												lowerCamelCase_					=       {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}

												lowerCamelCase_					=       [input_ids, input_mask]
												lowerCamelCase_					=       model(__UpperCamelCase )

												lowerCamelCase_					=       model(__UpperCamelCase )

												self.parent.assertEqual(result.last_hidden_state.shape ,       (self.batch_size, self.seq_length, self.hidden_size) )

					def  lowercase__   (						self       :  Optional[int] ,       __UpperCamelCase       :  List[str] ,       __UpperCamelCase       :  str ,       __UpperCamelCase       :  Dict ,       __UpperCamelCase       :  str ,       __UpperCamelCase       :  List[str] ,       __UpperCamelCase       :  Tuple ,       __UpperCamelCase       :  int ):
												lowerCamelCase_					=       TFDebertaVaForMaskedLM(config=__UpperCamelCase )
												lowerCamelCase_					=       {
												    """input_ids""": input_ids,
												    """attention_mask""": input_mask,
												    """token_type_ids""": token_type_ids,
												}
												lowerCamelCase_					=       model(__UpperCamelCase )
												self.parent.assertEqual(result.logits.shape ,       (self.batch_size, self.seq_length, self.vocab_size) )

					def  lowercase__   (						self       :  List[Any] ,       __UpperCamelCase       :  Union[str, Any] ,       __UpperCamelCase       :  str ,       __UpperCamelCase       :  int ,       __UpperCamelCase       :  Optional[int] ,       __UpperCamelCase       :  List[Any] ,       __UpperCamelCase       :  str ,       __UpperCamelCase       :  List[Any] ):
												lowerCamelCase_					=       self.num_labels
												lowerCamelCase_					=       TFDebertaVaForSequenceClassification(config=__UpperCamelCase )
												lowerCamelCase_					=       {
												    """input_ids""": input_ids,
												    """attention_mask""": input_mask,
												    """token_type_ids""": token_type_ids,
												}

												lowerCamelCase_					=       model(__UpperCamelCase )
												self.parent.assertEqual(result.logits.shape ,       (self.batch_size, self.num_labels) )

					def  lowercase__   (						self       :  int ,       __UpperCamelCase       :  Tuple ,       __UpperCamelCase       :  Any ,       __UpperCamelCase       :  str ,       __UpperCamelCase       :  Optional[int] ,       __UpperCamelCase       :  List[str] ,       __UpperCamelCase       :  List[Any] ,       __UpperCamelCase       :  List[Any] ):
												lowerCamelCase_					=       self.num_labels
												lowerCamelCase_					=       TFDebertaVaForTokenClassification(config=__UpperCamelCase )
												lowerCamelCase_					=       {
												    """input_ids""": input_ids,
												    """attention_mask""": input_mask,
												    """token_type_ids""": token_type_ids,
												}
												lowerCamelCase_					=       model(__UpperCamelCase )
												self.parent.assertEqual(result.logits.shape ,       (self.batch_size, self.seq_length, self.num_labels) )

					def  lowercase__   (						self       :  List[Any] ,       __UpperCamelCase       :  Optional[int] ,       __UpperCamelCase       :  Optional[Any] ,       __UpperCamelCase       :  Optional[int] ,       __UpperCamelCase       :  str ,       __UpperCamelCase       :  Optional[Any] ,       __UpperCamelCase       :  List[str] ,       __UpperCamelCase       :  int ):
												lowerCamelCase_					=       TFDebertaVaForQuestionAnswering(config=__UpperCamelCase )
												lowerCamelCase_					=       {
												    """input_ids""": input_ids,
												    """attention_mask""": input_mask,
												    """token_type_ids""": token_type_ids,
												}

												lowerCamelCase_					=       model(__UpperCamelCase )
												self.parent.assertEqual(result.start_logits.shape ,       (self.batch_size, self.seq_length) )
												self.parent.assertEqual(result.end_logits.shape ,       (self.batch_size, self.seq_length) )



					def  lowercase__   (						self       :  Dict ):
												lowerCamelCase_					=       self.prepare_config_and_inputs()
												(
												    (
												    lowerCamelCase_
												)	, (
												    lowerCamelCase_
												)	, (
												    lowerCamelCase_
												)	, (
												    lowerCamelCase_
												)	, (
												    lowerCamelCase_
												)	, (
												    lowerCamelCase_
												)	, (
												    lowerCamelCase_
												)	, 
												)					=       config_and_inputs
												lowerCamelCase_					=       {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
												return config, inputs_dict





@require_tf
class    __A(					UpperCAmelCase  ,		UpperCAmelCase  ,		unittest.TestCase							):
					SCREAMING_SNAKE_CASE           =     (
					    (
					        TFDebertaVaModel,
					        TFDebertaVaForMaskedLM,
					        TFDebertaVaForQuestionAnswering,
					        TFDebertaVaForSequenceClassification,
					        TFDebertaVaForTokenClassification,
					    )
					    if is_tf_available()
					    else ()
					)
					SCREAMING_SNAKE_CASE           =     (
					    {
					        '''feature-extraction''': TFDebertaVaModel,
					        '''fill-mask''': TFDebertaVaForMaskedLM,
					        '''question-answering''': TFDebertaVaForQuestionAnswering,
					        '''text-classification''': TFDebertaVaForSequenceClassification,
					        '''token-classification''': TFDebertaVaForTokenClassification,
					        '''zero-shot''': TFDebertaVaForSequenceClassification,
					    }
					    if is_tf_available()
					    else {}
					)

					SCREAMING_SNAKE_CASE           =     False
					SCREAMING_SNAKE_CASE           =     False

					def  lowercase__   (						self       :  Dict ):
												lowerCamelCase_					=       TFDebertaVaModelTester(self )
												lowerCamelCase_					=       ConfigTester(self ,       config_class=__UpperCamelCase ,       hidden_size=3_7 )

					def  lowercase__   (						self       :  Dict ):
												self.config_tester.run_common_tests()

					def  lowercase__   (						self       :  Optional[Any] ):
												lowerCamelCase_					=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_model(*__UpperCamelCase )

					def  lowercase__   (						self       :  Dict ):
												lowerCamelCase_					=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_masked_lm(*__UpperCamelCase )

					def  lowercase__   (						self       :  List[Any] ):
												lowerCamelCase_					=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_question_answering(*__UpperCamelCase )

					def  lowercase__   (						self       :  Tuple ):
												lowerCamelCase_					=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_sequence_classification(*__UpperCamelCase )

					def  lowercase__   (						self       :  Any ):
												lowerCamelCase_					=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_token_classification(*__UpperCamelCase )



					@slow
					def  lowercase__   (						self       :  int ):
												lowerCamelCase_					=       TFDebertaVaModel.from_pretrained("""kamalkraj/deberta-v2-xlarge""" )
												self.assertIsNotNone(__UpperCamelCase )





@require_tf
class    __A(					unittest.TestCase							):

					@unittest.skip(reason="""Model not available yet""" )
					def  lowercase__   (						self       :  Optional[int] ):
												pass



					@slow
					def  lowercase__   (						self       :  Any ):
												lowerCamelCase_					=       TFDebertaVaModel.from_pretrained("""kamalkraj/deberta-v2-xlarge""" )
												lowerCamelCase_					=       tf.constant([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]] )
												lowerCamelCase_					=       tf.constant([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]] )
												lowerCamelCase_					=       model(__UpperCamelCase ,       attention_mask=__UpperCamelCase )[0]

												lowerCamelCase_					=       tf.constant(
												    [[[0.2356, 0.1948, 0.0369], [-0.1063, 0.3586, -0.5152], [-0.6399, -0.0259, -0.2525]]] )
												tf.debugging.assert_near(output[:, 1:4, 1:4] ,       __UpperCamelCase ,       atol=1E-4 )





