


'''simple docstring'''
import json
from typing import Dict, List, Optional, Tuple, Union

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding, EncodedInput
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_led import LEDTokenizer


UpperCamelCase__          = logging.get_logger(__name__)


UpperCamelCase__          = {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase__          = {
    '''vocab_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json''',
    },
}

UpperCamelCase__          = {
    '''allenai/led-base-16384''': 1_6_3_8_4,
}




class      lowerCamelCase_  (      __a ):
      lowerCAmelCase__							=	VOCAB_FILES_NAMES
      lowerCAmelCase__							=	PRETRAINED_VOCAB_FILES_MAP
      lowerCAmelCase__							=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowerCAmelCase__							=	LEDTokenizer
      lowerCAmelCase__							=	['input_ids', 'attention_mask']






      def __init__(   self							:       Dict		,		_A							:       Tuple=None		,		_A							:       Union[str, Any]=None		,		_A							:       str=None		,		_A							:       List[Any]="replace"		,		_A							:       Dict="<s>"		,		_A							:       Dict="</s>"		,		_A							:       Tuple="</s>"		,		_A							:       int="<s>"		,		_A							:       Optional[Any]="<unk>"		,		_A							:       str="<pad>"		,		_A							:       int="<mask>"		,		_A							:       int=False		,		_A							:       str=True		,		**_A							:       Optional[int]		,		):


          '''simple docstring'''


          super().__init__(
              _A		,		_A		,		tokenizer_file=_A		,		errors=_A		,		bos_token=_A		,		eos_token=_A		,		sep_token=_A		,		cls_token=_A		,		unk_token=_A		,		pad_token=_A		,		mask_token=_A		,		add_prefix_space=_A		,		trim_offsets=_A		,		**_A		,		)

          UpperCAmelCase__     :						Union[str, Any]						=							json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()					)
          if pre_tok_state.get('''add_prefix_space'''		,		_A					) != add_prefix_space:
              UpperCAmelCase__     :						Dict						=							getattr(_A		,		pre_tok_state.pop('''type'''					)					)
              UpperCAmelCase__     :						int						=							add_prefix_space
              UpperCAmelCase__     :						Dict						=							pre_tok_class(**_A					)

          UpperCAmelCase__     :						Optional[Any]						=							add_prefix_space

          # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
          UpperCAmelCase__     :						Optional[Any]						=							'''post_processor'''
          UpperCAmelCase__     :						Tuple						=							getattr(self.backend_tokenizer		,		_A		,		_A					)
          if tokenizer_component_instance:
              UpperCAmelCase__     :						List[Any]						=							json.loads(tokenizer_component_instance.__getstate__()					)

              # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
              if "sep" in state:
                  UpperCAmelCase__     :						Optional[int]						=							tuple(state['''sep''']					)
              if "cls" in state:
                  UpperCAmelCase__     :						str						=							tuple(state['''cls''']					)

              UpperCAmelCase__     :						Dict						=							False

              if state.get('''add_prefix_space'''		,		_A					) != add_prefix_space:
                  UpperCAmelCase__     :						List[str]						=							add_prefix_space
                  UpperCAmelCase__     :						List[str]						=							True

              if state.get('''trim_offsets'''		,		_A					) != trim_offsets:
                  UpperCAmelCase__     :						List[str]						=							trim_offsets
                  UpperCAmelCase__     :						Optional[Any]						=							True

              if changes_to_apply:
                  UpperCAmelCase__     :						Optional[Any]						=							getattr(_A		,		state.pop('''type'''					)					)
                  UpperCAmelCase__     :						Union[str, Any]						=							component_class(**_A					)
                  setattr(self.backend_tokenizer		,		_A		,		_A					)






      @property
      # Copied from transformers.models.bart.tokenization_bart_fast.BartTokenizerFast.mask_token with BART->LED
      def lowercase_				(   self							:       Any					):


          '''simple docstring'''


          if self._mask_token is None:
              if self.verbose:
                  logger.error('''Using mask_token, but it is not set yet.'''					)
              return None
          return str(self._mask_token					)






      @mask_token.setter
      def lowercase_				(   self							:       List[str]		,		_A							:       Union[str, Any]					):


          '''simple docstring'''


          UpperCAmelCase__     :						str						=							AddedToken(_A		,		lstrip=_A		,		rstrip=_A					) if isinstance(_A		,		_A					) else value
          UpperCAmelCase__     :						Optional[Any]						=							value






      def lowercase_				(   self							:       Union[str, Any]		,		*_A							:       List[str]		,		**_A							:       List[Any]					):


          '''simple docstring'''


          UpperCAmelCase__     :						List[str]						=							kwargs.get('''is_split_into_words'''		,		_A					)

          if is_split_into_words and not self.add_prefix_space:
              raise ValueError(
                  f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
                  '''to use it with pretokenized inputs.'''					)

          return super()._batch_encode_plus(*_A		,		**_A					)






      def lowercase_				(   self							:       str		,		*_A							:       List[str]		,		**_A							:       str					):


          '''simple docstring'''


          UpperCAmelCase__     :						Optional[int]						=							kwargs.get('''is_split_into_words'''		,		_A					)

          if is_split_into_words and not self.add_prefix_space:
              raise ValueError(
                  f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
                  '''to use it with pretokenized inputs.'''					)

          return super()._encode_plus(*_A		,		**_A					)






      def lowercase_				(   self							:       Any		,		_A							:       str		,		_A							:       Optional[str] = None					):


          '''simple docstring'''


          UpperCAmelCase__     :						Tuple						=							self._tokenizer.model.save(_A		,		name=_A					)
          return tuple(_A					)






      def lowercase_				(   self							:       Union[str, Any]		,		_A							:       int		,		_A							:       str=None					):


          '''simple docstring'''


          UpperCAmelCase__     :						List[str]						=							[self.bos_token_id] + token_ids_a + [self.eos_token_id]
          if token_ids_a is None:
              return output

          return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






      def lowercase_				(   self							:       Union[str, Any]		,		_A							:       List[int]		,		_A							:       Optional[List[int]] = None					):


          '''simple docstring'''


          UpperCAmelCase__     :						List[Any]						=							[self.sep_token_id]
          UpperCAmelCase__     :						Union[str, Any]						=							[self.cls_token_id]

          if token_ids_a is None:
              return len(cls + token_ids_a + sep					) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]






      def lowercase_				(   self							:       Dict		,		_A							:       Union[Dict[str, EncodedInput], BatchEncoding]		,		_A							:       Optional[int] = None		,		_A							:       PaddingStrategy = PaddingStrategy.DO_NOT_PAD		,		_A							:       Optional[int] = None		,		_A							:       Optional[bool] = None		,		):


          '''simple docstring'''


          UpperCAmelCase__     :						int						=							super()._pad(
              encoded_inputs=_A		,		max_length=_A		,		padding_strategy=_A		,		pad_to_multiple_of=_A		,		return_attention_mask=_A		,		)

          # Load from model defaults
          if return_attention_mask is None:
              UpperCAmelCase__     :						Tuple						=							'''attention_mask''' in self.model_input_names

          if return_attention_mask and "global_attention_mask" in encoded_inputs:
              UpperCAmelCase__     :						int						=							encoded_inputs[self.model_input_names[0]]
              # `global_attention_mask` need to have the same length as other (sequential) inputs.
              UpperCAmelCase__     :						int						=							len(encoded_inputs['''global_attention_mask''']					) != len(_A					)

              if needs_to_be_padded:
                  UpperCAmelCase__     :						int						=							len(_A					) - len(encoded_inputs['''global_attention_mask''']					)

                  if self.padding_side == "right":
                      # Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
                      UpperCAmelCase__     :						Tuple						=							(
                          encoded_inputs['''global_attention_mask'''] + [-1] * difference
                      )
                  elif self.padding_side == "left":
                      UpperCAmelCase__     :						Optional[Any]						=							[-1] * difference + encoded_inputs[
                          '''global_attention_mask'''
                      ]
                  else:
                      raise ValueError('''Invalid padding strategy:''' + str(self.padding_side					)					)

          return encoded_inputs






'''simple docstring'''


from __future__ import annotations

import unittest

from transformers import DebertaVaConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
	import tensorflow as tf

	from transformers import (
	    TFDebertaVaForMaskedLM,
	    TFDebertaVaForQuestionAnswering,
	    TFDebertaVaForSequenceClassification,
	    TFDebertaVaForTokenClassification,
	    TFDebertaVaModel,
	)
class 						a_				:




	def __init__(       self     :  Tuple       ,		__lowerCAmelCase     :  Optional[int]       ,		__lowerCAmelCase     :  Union[str, Any]=1_3       ,		__lowerCAmelCase     :  int=7       ,		__lowerCAmelCase     :  int=True       ,		__lowerCAmelCase     :  str=True       ,		__lowerCAmelCase     :  str=True       ,		__lowerCAmelCase     :  Optional[Any]=True       ,		__lowerCAmelCase     :  List[Any]=9_9       ,		__lowerCAmelCase     :  List[str]=3_2       ,		__lowerCAmelCase     :  List[Any]=2       ,		__lowerCAmelCase     :  Any=4       ,		__lowerCAmelCase     :  Any=3_7       ,		__lowerCAmelCase     :  Union[str, Any]="gelu"       ,		__lowerCAmelCase     :  Any=0.1       ,		__lowerCAmelCase     :  int=0.1       ,		__lowerCAmelCase     :  Optional[int]=5_1_2       ,		__lowerCAmelCase     :  Optional[Any]=1_6       ,		__lowerCAmelCase     :  Optional[Any]=2       ,		__lowerCAmelCase     :  Dict=0.02       ,		__lowerCAmelCase     :  Dict=False       ,		__lowerCAmelCase     :  int=True       ,		__lowerCAmelCase     :  Tuple="None"       ,		__lowerCAmelCase     :  Optional[int]=3       ,		__lowerCAmelCase     :  Optional[int]=4       ,		__lowerCAmelCase     :  Union[str, Any]=None       ,		):
							__snake_case						 =   parent
							__snake_case						 =   batch_size
							__snake_case						 =   seq_length
							__snake_case						 =   is_training
							__snake_case						 =   use_input_mask
							__snake_case						 =   use_token_type_ids
							__snake_case						 =   use_labels
							__snake_case						 =   vocab_size
							__snake_case						 =   hidden_size
							__snake_case						 =   num_hidden_layers
							__snake_case						 =   num_attention_heads
							__snake_case						 =   intermediate_size
							__snake_case						 =   hidden_act
							__snake_case						 =   hidden_dropout_prob
							__snake_case						 =   attention_probs_dropout_prob
							__snake_case						 =   max_position_embeddings
							__snake_case						 =   type_vocab_size
							__snake_case						 =   type_sequence_label_size
							__snake_case						 =   initializer_range
							__snake_case						 =   num_labels
							__snake_case						 =   num_choices
							__snake_case						 =   relative_attention
							__snake_case						 =   position_biased_input
							__snake_case						 =   pos_att_type
							__snake_case						 =   scope




	def   lowercase__   (       self     :  Dict      ):
							__snake_case						 =   ids_tensor([self.batch_size, self.seq_length]       ,		self.vocab_size      )

							__snake_case						 =   None
							if self.use_input_mask:
													__snake_case						 =   random_attention_mask([self.batch_size, self.seq_length]      )

							__snake_case						 =   None
							if self.use_token_type_ids:
													__snake_case						 =   ids_tensor([self.batch_size, self.seq_length]       ,		self.type_vocab_size      )

							__snake_case						 =   None
							__snake_case						 =   None
							__snake_case						 =   None
							if self.use_labels:
													__snake_case						 =   ids_tensor([self.batch_size]       ,		self.type_sequence_label_size      )
													__snake_case						 =   ids_tensor([self.batch_size, self.seq_length]       ,		self.num_labels      )

							__snake_case						 =   DebertaVaConfig(
							    vocab_size=self.vocab_size       ,		hidden_size=self.hidden_size       ,		num_hidden_layers=self.num_hidden_layers       ,		num_attention_heads=self.num_attention_heads       ,		intermediate_size=self.intermediate_size       ,		hidden_act=self.hidden_act       ,		hidden_dropout_prob=self.hidden_dropout_prob       ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,		max_position_embeddings=self.max_position_embeddings       ,		type_vocab_size=self.type_vocab_size       ,		relative_attention=self.relative_attention       ,		position_biased_input=self.position_biased_input       ,		initializer_range=self.initializer_range       ,		return_dict=__lowerCAmelCase       ,		)

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




	def   lowercase__   (       self     :  Union[str, Any]       ,		__lowerCAmelCase     :  List[str]       ,		__lowerCAmelCase     :  Optional[Any]       ,		__lowerCAmelCase     :  Optional[Any]       ,		__lowerCAmelCase     :  Tuple       ,		__lowerCAmelCase     :  List[str]       ,		__lowerCAmelCase     :  Tuple       ,		__lowerCAmelCase     :  Union[str, Any]      ):
							__snake_case						 =   TFDebertaVaModel(config=__lowerCAmelCase      )
							__snake_case						 =   {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}

							__snake_case						 =   [input_ids, input_mask]
							__snake_case						 =   model(__lowerCAmelCase      )

							__snake_case						 =   model(__lowerCAmelCase      )

							self.parent.assertEqual(result.last_hidden_state.shape       ,		(self.batch_size, self.seq_length, self.hidden_size)      )




	def   lowercase__   (       self     :  Optional[Any]       ,		__lowerCAmelCase     :  Dict       ,		__lowerCAmelCase     :  Optional[Any]       ,		__lowerCAmelCase     :  Optional[int]       ,		__lowerCAmelCase     :  int       ,		__lowerCAmelCase     :  Optional[int]       ,		__lowerCAmelCase     :  List[Any]       ,		__lowerCAmelCase     :  List[Any]      ):
							__snake_case						 =   TFDebertaVaForMaskedLM(config=__lowerCAmelCase      )
							__snake_case						 =   {
							    'input_ids': input_ids,
							    'attention_mask': input_mask,
							    'token_type_ids': token_type_ids,
							}
							__snake_case						 =   model(__lowerCAmelCase      )
							self.parent.assertEqual(result.logits.shape       ,		(self.batch_size, self.seq_length, self.vocab_size)      )




	def   lowercase__   (       self     :  Optional[int]       ,		__lowerCAmelCase     :  str       ,		__lowerCAmelCase     :  List[Any]       ,		__lowerCAmelCase     :  Union[str, Any]       ,		__lowerCAmelCase     :  Optional[int]       ,		__lowerCAmelCase     :  Optional[Any]       ,		__lowerCAmelCase     :  Any       ,		__lowerCAmelCase     :  List[Any]      ):
							__snake_case						 =   self.num_labels
							__snake_case						 =   TFDebertaVaForSequenceClassification(config=__lowerCAmelCase      )
							__snake_case						 =   {
							    'input_ids': input_ids,
							    'attention_mask': input_mask,
							    'token_type_ids': token_type_ids,
							}

							__snake_case						 =   model(__lowerCAmelCase      )
							self.parent.assertEqual(result.logits.shape       ,		(self.batch_size, self.num_labels)      )




	def   lowercase__   (       self     :  Tuple       ,		__lowerCAmelCase     :  Dict       ,		__lowerCAmelCase     :  str       ,		__lowerCAmelCase     :  List[str]       ,		__lowerCAmelCase     :  List[str]       ,		__lowerCAmelCase     :  Tuple       ,		__lowerCAmelCase     :  Tuple       ,		__lowerCAmelCase     :  int      ):
							__snake_case						 =   self.num_labels
							__snake_case						 =   TFDebertaVaForTokenClassification(config=__lowerCAmelCase      )
							__snake_case						 =   {
							    'input_ids': input_ids,
							    'attention_mask': input_mask,
							    'token_type_ids': token_type_ids,
							}
							__snake_case						 =   model(__lowerCAmelCase      )
							self.parent.assertEqual(result.logits.shape       ,		(self.batch_size, self.seq_length, self.num_labels)      )




	def   lowercase__   (       self     :  int       ,		__lowerCAmelCase     :  Optional[Any]       ,		__lowerCAmelCase     :  List[Any]       ,		__lowerCAmelCase     :  Any       ,		__lowerCAmelCase     :  Tuple       ,		__lowerCAmelCase     :  Union[str, Any]       ,		__lowerCAmelCase     :  Optional[int]       ,		__lowerCAmelCase     :  List[Any]      ):
							__snake_case						 =   TFDebertaVaForQuestionAnswering(config=__lowerCAmelCase      )
							__snake_case						 =   {
							    'input_ids': input_ids,
							    'attention_mask': input_mask,
							    'token_type_ids': token_type_ids,
							}

							__snake_case						 =   model(__lowerCAmelCase      )
							self.parent.assertEqual(result.start_logits.shape       ,		(self.batch_size, self.seq_length)      )
							self.parent.assertEqual(result.end_logits.shape       ,		(self.batch_size, self.seq_length)      )




	def   lowercase__   (       self     :  Union[str, Any]      ):
							__snake_case						 =   self.prepare_config_and_inputs()
							(
							    (
							    __snake_case
							) ,       (
							    __snake_case
							) ,       (
							    __snake_case
							) ,       (
							    __snake_case
							) ,       (
							    __snake_case
							) ,       (
							    __snake_case
							) ,       (
							    __snake_case
							) ,       
							)						 =   config_and_inputs
							__snake_case						 =   {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
							return config, inputs_dict





@require_tf
class 						a_				(			UpperCAmelCase__     ,						UpperCAmelCase__     ,						unittest.TestCase						):
	lowercase_       :     List[str]			  =      (
	    (
	        TFDebertaVaModel,
	        TFDebertaVaForMaskedLM,
	        TFDebertaVaForQuestionAnswering,
	        TFDebertaVaForSequenceClassification,
	        TFDebertaVaForTokenClassification,
	    )
	    if is_tf_available()
	    else ()
	)
	lowercase_       :     Union[str, Any]			  =      (
	    {
	        '''feature-extraction''': TFDebertaVaModel,
	        '''fill-mask''': TFDebertaVaForMaskedLM,
	        '''question-answering''': TFDebertaVaForQuestionAnswering,
	        '''text-classification''': TFDebertaVaForSequenceClassification,
	        '''token-classification''': TFDebertaVaForTokenClassification,
	        '''zero-shot''': TFDebertaVaForSequenceClassification,
	    }
	    if is_tf_available()
	    else {}
	)

	lowercase_       :     int			  =      False
	lowercase_       :     List[Any]			  =      False




	def   lowercase__   (       self     :  Any      ):
							__snake_case						 =   TFDebertaVaModelTester(self      )
							__snake_case						 =   ConfigTester(self       ,		config_class=__lowerCAmelCase       ,		hidden_size=3_7      )




	def   lowercase__   (       self     :  Dict      ):
							self.config_tester.run_common_tests()




	def   lowercase__   (       self     :  Optional[int]      ):
							__snake_case						 =   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*__lowerCAmelCase      )




	def   lowercase__   (       self     :  Union[str, Any]      ):
							__snake_case						 =   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*__lowerCAmelCase      )




	def   lowercase__   (       self     :  List[Any]      ):
							__snake_case						 =   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*__lowerCAmelCase      )




	def   lowercase__   (       self     :  Any      ):
							__snake_case						 =   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(*__lowerCAmelCase      )




	def   lowercase__   (       self     :  Dict      ):
							__snake_case						 =   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*__lowerCAmelCase      )




	@slow
	def   lowercase__   (       self     :  List[str]      ):
							__snake_case						 =   TFDebertaVaModel.from_pretrained('kamalkraj/deberta-v2-xlarge'      )
							self.assertIsNotNone(__lowerCAmelCase      )





@require_tf
class 						a_				(			unittest.TestCase						):




	@unittest.skip(reason='Model not available yet'      )
	def   lowercase__   (       self     :  List[str]      ):
							pass




	@slow
	def   lowercase__   (       self     :  int      ):
							__snake_case						 =   TFDebertaVaModel.from_pretrained('kamalkraj/deberta-v2-xlarge'      )
							__snake_case						 =   tf.constant([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]]      )
							__snake_case						 =   tf.constant([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]      )
							__snake_case						 =   model(__lowerCAmelCase       ,		attention_mask=__lowerCAmelCase      )[0]

							__snake_case						 =   tf.constant(
							    [[[0.2356, 0.1948, 0.0369], [-0.1063, 0.3586, -0.5152], [-0.6399, -0.0259, -0.2525]]]      )
							tf.debugging.assert_near(output[:, 1:4, 1:4]       ,		__lowerCAmelCase       ,		atol=1E-4      )

