






"""simple docstring"""






from __future__ import annotations

import unittest

from transformers import DebertaVaConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import (
      TFDebertaVaForMaskedLM,
      TFDebertaVaForQuestionAnswering,
      TFDebertaVaForSequenceClassification,
      TFDebertaVaForTokenClassification,
      TFDebertaVaModel,
  )




class 							lowerCAmelCase__       :






      '''simple docstring'''

      def __init__(		self  :						Tuple		,			lowercase_  :						Any		,			lowercase_  :						Tuple=13		,			lowercase_  :						List[str]=7		,			lowercase_  :						Union[str, Any]=True		,			lowercase_  :						Optional[int]=True		,			lowercase_  :						Optional[int]=True		,			lowercase_  :						Any=True		,			lowercase_  :						Optional[int]=99		,			lowercase_  :						Optional[int]=32		,			lowercase_  :						List[Any]=2		,			lowercase_  :						Any=4		,			lowercase_  :						Tuple=37		,			lowercase_  :						Any="gelu"		,			lowercase_  :						str=0.1		,			lowercase_  :						int=0.1		,			lowercase_  :						int=512		,			lowercase_  :						Optional[Any]=16		,			lowercase_  :						Union[str, Any]=2		,			lowercase_  :						Optional[int]=0.02		,			lowercase_  :						List[str]=False		,			lowercase_  :						Union[str, Any]=True		,			lowercase_  :						Any="None"		,			lowercase_  :						List[Any]=3		,			lowercase_  :						List[Any]=4		,			lowercase_  :						List[Any]=None		,			):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Dict							=	parent
          SCREAMING_SNAKE_CASE_   :						Any							=	batch_size
          SCREAMING_SNAKE_CASE_   :						Dict							=	seq_length
          SCREAMING_SNAKE_CASE_   :						List[Any]							=	is_training
          SCREAMING_SNAKE_CASE_   :						int							=	use_input_mask
          SCREAMING_SNAKE_CASE_   :						Dict							=	use_token_type_ids
          SCREAMING_SNAKE_CASE_   :						Dict							=	use_labels
          SCREAMING_SNAKE_CASE_   :						List[Any]							=	vocab_size
          SCREAMING_SNAKE_CASE_   :						List[Any]							=	hidden_size
          SCREAMING_SNAKE_CASE_   :						int							=	num_hidden_layers
          SCREAMING_SNAKE_CASE_   :						List[Any]							=	num_attention_heads
          SCREAMING_SNAKE_CASE_   :						int							=	intermediate_size
          SCREAMING_SNAKE_CASE_   :						Dict							=	hidden_act
          SCREAMING_SNAKE_CASE_   :						str							=	hidden_dropout_prob
          SCREAMING_SNAKE_CASE_   :						Tuple							=	attention_probs_dropout_prob
          SCREAMING_SNAKE_CASE_   :						str							=	max_position_embeddings
          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	type_vocab_size
          SCREAMING_SNAKE_CASE_   :						Any							=	type_sequence_label_size
          SCREAMING_SNAKE_CASE_   :						str							=	initializer_range
          SCREAMING_SNAKE_CASE_   :						Optional[int]							=	num_labels
          SCREAMING_SNAKE_CASE_   :						Optional[int]							=	num_choices
          SCREAMING_SNAKE_CASE_   :						Optional[int]							=	relative_attention
          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	position_biased_input
          SCREAMING_SNAKE_CASE_   :						Dict							=	pos_att_type
          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	scope
      def _SCREAMING_SNAKE_CASE				(		self  :						Tuple):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						int							=	ids_tensor([self.batch_size, self.seq_length]		,			self.vocab_size)

          SCREAMING_SNAKE_CASE_   :						str							=	None
          if self.use_input_mask:
              SCREAMING_SNAKE_CASE_   :						Optional[int]							=	random_attention_mask([self.batch_size, self.seq_length])

          SCREAMING_SNAKE_CASE_   :						Tuple							=	None
          if self.use_token_type_ids:
              SCREAMING_SNAKE_CASE_   :						List[Any]							=	ids_tensor([self.batch_size, self.seq_length]		,			self.type_vocab_size)

          SCREAMING_SNAKE_CASE_   :						Any							=	None
          SCREAMING_SNAKE_CASE_   :						Dict							=	None
          SCREAMING_SNAKE_CASE_   :						Optional[int]							=	None
          if self.use_labels:
              SCREAMING_SNAKE_CASE_   :						int							=	ids_tensor([self.batch_size]		,			self.type_sequence_label_size)
              SCREAMING_SNAKE_CASE_   :						str							=	ids_tensor([self.batch_size, self.seq_length]		,			self.num_labels)

          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	DebertaVaConfig(
              vocab_size=self.vocab_size		,			hidden_size=self.hidden_size		,			num_hidden_layers=self.num_hidden_layers		,			num_attention_heads=self.num_attention_heads		,			intermediate_size=self.intermediate_size		,			hidden_act=self.hidden_act		,			hidden_dropout_prob=self.hidden_dropout_prob		,			attention_probs_dropout_prob=self.attention_probs_dropout_prob		,			max_position_embeddings=self.max_position_embeddings		,			type_vocab_size=self.type_vocab_size		,			relative_attention=self.relative_attention		,			position_biased_input=self.position_biased_input		,			initializer_range=self.initializer_range		,			return_dict=lowercase_		,			)

          return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
      def _SCREAMING_SNAKE_CASE				(		self  :						Union[str, Any]		,			lowercase_  :						Union[str, Any]		,			lowercase_  :						List[Any]		,			lowercase_  :						Union[str, Any]		,			lowercase_  :						int		,			lowercase_  :						Union[str, Any]		,			lowercase_  :						Union[str, Any]		,			lowercase_  :						List[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Tuple							=	TFDebertaVaModel(config=lowercase_)
          SCREAMING_SNAKE_CASE_   :						Optional[int]							=	{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}

          SCREAMING_SNAKE_CASE_   :						int							=	[input_ids, input_mask]
          SCREAMING_SNAKE_CASE_   :						Dict							=	model(lowercase_)

          SCREAMING_SNAKE_CASE_   :						str							=	model(lowercase_)

          self.parent.assertEqual(result.last_hidden_state.shape		,			(self.batch_size, self.seq_length, self.hidden_size))
      def _SCREAMING_SNAKE_CASE				(		self  :						int		,			lowercase_  :						Tuple		,			lowercase_  :						Dict		,			lowercase_  :						Tuple		,			lowercase_  :						int		,			lowercase_  :						Dict		,			lowercase_  :						int		,			lowercase_  :						List[str]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	TFDebertaVaForMaskedLM(config=lowercase_)
          SCREAMING_SNAKE_CASE_   :						Any							=	{
              '''input_ids''': input_ids,
              '''attention_mask''': input_mask,
              '''token_type_ids''': token_type_ids,
          }
          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	model(lowercase_)
          self.parent.assertEqual(result.logits.shape		,			(self.batch_size, self.seq_length, self.vocab_size))
      def _SCREAMING_SNAKE_CASE				(		self  :						List[Any]		,			lowercase_  :						Optional[int]		,			lowercase_  :						int		,			lowercase_  :						str		,			lowercase_  :						Union[str, Any]		,			lowercase_  :						Union[str, Any]		,			lowercase_  :						Dict		,			lowercase_  :						str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						List[str]							=	self.num_labels
          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	TFDebertaVaForSequenceClassification(config=lowercase_)
          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	{
              '''input_ids''': input_ids,
              '''attention_mask''': input_mask,
              '''token_type_ids''': token_type_ids,
          }

          SCREAMING_SNAKE_CASE_   :						Any							=	model(lowercase_)
          self.parent.assertEqual(result.logits.shape		,			(self.batch_size, self.num_labels))
      def _SCREAMING_SNAKE_CASE				(		self  :						int		,			lowercase_  :						List[Any]		,			lowercase_  :						int		,			lowercase_  :						Tuple		,			lowercase_  :						Any		,			lowercase_  :						List[str]		,			lowercase_  :						str		,			lowercase_  :						List[str]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Optional[int]							=	self.num_labels
          SCREAMING_SNAKE_CASE_   :						List[str]							=	TFDebertaVaForTokenClassification(config=lowercase_)
          SCREAMING_SNAKE_CASE_   :						int							=	{
              '''input_ids''': input_ids,
              '''attention_mask''': input_mask,
              '''token_type_ids''': token_type_ids,
          }
          SCREAMING_SNAKE_CASE_   :						List[Any]							=	model(lowercase_)
          self.parent.assertEqual(result.logits.shape		,			(self.batch_size, self.seq_length, self.num_labels))
      def _SCREAMING_SNAKE_CASE				(		self  :						Dict		,			lowercase_  :						Optional[Any]		,			lowercase_  :						List[str]		,			lowercase_  :						List[str]		,			lowercase_  :						Optional[int]		,			lowercase_  :						List[Any]		,			lowercase_  :						Optional[int]		,			lowercase_  :						int):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Dict							=	TFDebertaVaForQuestionAnswering(config=lowercase_)
          SCREAMING_SNAKE_CASE_   :						Tuple							=	{
              '''input_ids''': input_ids,
              '''attention_mask''': input_mask,
              '''token_type_ids''': token_type_ids,
          }

          SCREAMING_SNAKE_CASE_   :						int							=	model(lowercase_)
          self.parent.assertEqual(result.start_logits.shape		,			(self.batch_size, self.seq_length))
          self.parent.assertEqual(result.end_logits.shape		,			(self.batch_size, self.seq_length))



      def _SCREAMING_SNAKE_CASE				(		self  :						List[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Tuple							=	self.prepare_config_and_inputs()
          (
              (
              SCREAMING_SNAKE_CASE_
          )		,    (
              SCREAMING_SNAKE_CASE_
          )		,    (
              SCREAMING_SNAKE_CASE_
          )		,    (
              SCREAMING_SNAKE_CASE_
          )		,    (
              SCREAMING_SNAKE_CASE_
          )		,    (
              SCREAMING_SNAKE_CASE_
          )		,    (
              SCREAMING_SNAKE_CASE_
          )		,    
          )   :						Optional[Any]							=	config_and_inputs
          SCREAMING_SNAKE_CASE_   :						Union[str, Any]							=	{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
          return config, inputs_dict




@require_tf
class 							lowerCAmelCase__       (    UpperCAmelCase__       ,	UpperCAmelCase__       ,	unittest.TestCase				):






      '''simple docstring'''

      __UpperCamelCase							=						(
          (
              TFDebertaVaModel,
              TFDebertaVaForMaskedLM,
              TFDebertaVaForQuestionAnswering,
              TFDebertaVaForSequenceClassification,
              TFDebertaVaForTokenClassification,
          )
          if is_tf_available()
          else ()
      )
      __UpperCamelCase							=						(
          {
              "feature-extraction": TFDebertaVaModel,
              "fill-mask": TFDebertaVaForMaskedLM,
              "question-answering": TFDebertaVaForQuestionAnswering,
              "text-classification": TFDebertaVaForSequenceClassification,
              "token-classification": TFDebertaVaForTokenClassification,
              "zero-shot": TFDebertaVaForSequenceClassification,
          }
          if is_tf_available()
          else {}
      )

      __UpperCamelCase							=						False
      __UpperCamelCase							=						False
      def _SCREAMING_SNAKE_CASE				(		self  :						int):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						str							=	TFDebertaVaModelTester(self)
          SCREAMING_SNAKE_CASE_   :						Tuple							=	ConfigTester(self		,			config_class=lowercase_		,			hidden_size=37)
      def _SCREAMING_SNAKE_CASE				(		self  :						int):




          '''simple docstring'''


          self.config_tester.run_common_tests()
      def _SCREAMING_SNAKE_CASE				(		self  :						Dict):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*lowercase_)
      def _SCREAMING_SNAKE_CASE				(		self  :						Any):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						List[Any]							=	self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*lowercase_)
      def _SCREAMING_SNAKE_CASE				(		self  :						Dict):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Tuple							=	self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*lowercase_)
      def _SCREAMING_SNAKE_CASE				(		self  :						Tuple):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*lowercase_)
      def _SCREAMING_SNAKE_CASE				(		self  :						Optional[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Tuple							=	self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*lowercase_)



      @slow
      def _SCREAMING_SNAKE_CASE				(		self  :						Dict):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Any							=	TFDebertaVaModel.from_pretrained('''kamalkraj/deberta-v2-xlarge''')
          self.assertIsNotNone(lowercase_)




@require_tf
class 							lowerCAmelCase__       (    unittest.TestCase				):






      '''simple docstring'''

      @unittest.skip(reason='''Model not available yet''')
      def _SCREAMING_SNAKE_CASE				(		self  :						Tuple):




          '''simple docstring'''


          pass



      @slow
      def _SCREAMING_SNAKE_CASE				(		self  :						Optional[int]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_   :						Tuple							=	TFDebertaVaModel.from_pretrained('''kamalkraj/deberta-v2-xlarge''')
          SCREAMING_SNAKE_CASE_   :						Dict							=	tf.constant([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]])
          SCREAMING_SNAKE_CASE_   :						str							=	tf.constant([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])
          SCREAMING_SNAKE_CASE_   :						Optional[int]							=	model(lowercase_		,			attention_mask=lowercase_)[0]

          SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	tf.constant(
              [[[0.23_56, 0.19_48, 0.03_69], [-0.10_63, 0.35_86, -0.51_52], [-0.63_99, -0.02_59, -0.25_25]]])
          tf.debugging.assert_near(output[:, 1:4, 1:4]		,			lowercase_		,			atol=1e-4)










"""simple docstring"""






import multiprocessing
import time

from arguments import PretokenizationArguments
from datasets import load_dataset

from transformers import AutoTokenizer, HfArgumentParser



def       _A    (__a							)				-> Optional[int]:
    """simple docstring"""






    SCREAMING_SNAKE_CASE_   :						Optional[Any]							=	{}
    SCREAMING_SNAKE_CASE_   :						List[Any]							=	tokenizer(example['''content''']					,       truncation=__a							)['''input_ids''']
    SCREAMING_SNAKE_CASE_   :						List[Any]							=	len(example['''content''']							) / len(output['''input_ids''']							)
    return output


UpperCAmelCase_     :				Tuple         =				HfArgumentParser(PretokenizationArguments)
UpperCAmelCase_     :				Tuple         =				parser.parse_args()
if args.num_workers is None:
  UpperCAmelCase_     :				Tuple         =				multiprocessing.cpu_count()
UpperCAmelCase_     :				str         =				AutoTokenizer.from_pretrained(args.tokenizer_dir)

UpperCAmelCase_     :				Tuple         =				time.time()
UpperCAmelCase_     :				List[Any]         =				load_dataset(args.dataset_name, split="""train""")
print(f'''Dataset loaded in {time.time()-t_start:.2f}s''')

UpperCAmelCase_     :				Dict         =				time.time()
UpperCAmelCase_     :				Dict         =				ds.map(
    tokenize,
    num_proc=args.num_workers,
    remove_columns=[
        """repo_name""",
        """path""",
        """copies""",
        """size""",
        """content""",
        """license""",
        """hash""",
        """line_mean""",
        """line_max""",
        """alpha_frac""",
        """autogenerated""",
    ],
)
print(f'''Dataset tokenized in {time.time()-t_start:.2f}s''')

UpperCAmelCase_     :				str         =				time.time()
ds.push_to_hub(args.tokenized_data_repo)





print(f'''Data pushed to the hub in {time.time()-t_start:.2f}s''')


