

import json
import os
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase			       =			logging.get_logger(__name__)

_UpperCamelCase			       =			{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

_UpperCamelCase			       =			{
    '''vocab_file''': {
        '''microsoft/biogpt''': '''https://huggingface.co/microsoft/biogpt/resolve/main/vocab.json''',
    },
    '''merges_file''': {'''microsoft/biogpt''': '''https://huggingface.co/microsoft/biogpt/resolve/main/merges.txt'''},
}

_UpperCamelCase			       =			{
    '''microsoft/biogpt''': 1024,
}
def 						UpperCamelCase_(			snake_case__:  Optional[Any]   )       -> Dict:
 UpperCAmelCase__							=				set()
 UpperCAmelCase__							=				word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)   )
  UpperCAmelCase__							=				char
 return pairs







class     lowercase						(       _UpperCamelCase					):



 '''simple docstring'''


 __SCREAMING_SNAKE_CASE												=				VOCAB_FILES_NAMES
 __SCREAMING_SNAKE_CASE												=				PRETRAINED_VOCAB_FILES_MAP
 __SCREAMING_SNAKE_CASE												=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __SCREAMING_SNAKE_CASE												=				["""input_ids""", """attention_mask"""]




 def __init__(self       ,       __a       ,       __a       ,       __a="<unk>"       ,       __a="<s>"       ,       __a="</s>"       ,       __a="</s>"       ,       __a="<pad>"       ,       **__a       ,       ) ->					Optional[Any]:






  """simple docstring"""





  super().__init__(
      bos_token=__a       ,       eos_token=__a       ,       sep_token=__a       ,       unk_token=__a       ,       pad_token=__a       ,       **__a       ,       )

  try:
   import sacremoses
  except ImportError:
   raise ImportError(
       'You need to install sacremoses to use BioGptTokenizer. '
       'See https://pypi.org/project/sacremoses/ for installation.' )

  UpperCAmelCase__							=				'en'
  UpperCAmelCase__							=				sacremoses
  # cache of sm.MosesTokenizer instance
  UpperCAmelCase__							=				{}
  UpperCAmelCase__							=				{}
  with open(__a       ,       encoding='utf-8' ) as vocab_handle:
   UpperCAmelCase__							=				json.load(__a )
  UpperCAmelCase__							=				{v: k for k, v in self.encoder.items()}
  with open(__a       ,       encoding='utf-8' ) as merges_handle:
   UpperCAmelCase__							=				merges_handle.read().split('\n' )[:-1]
  UpperCAmelCase__							=				[tuple(merge.split()[:2] ) for merge in merges]
  UpperCAmelCase__							=				dict(zip(__a       ,       range(len(__a ) ) ) )
  UpperCAmelCase__							=				{}




 @property
 def   UpperCamelCase__							(self ) ->					str:






  """simple docstring"""





  return len(self.encoder )




 def   UpperCamelCase__							(self ) ->					Optional[Any]:






  """simple docstring"""





  return dict(self.encoder       ,       **self.added_tokens_encoder )




 def   UpperCamelCase__							(self       ,       __a       ,       __a ) ->					Optional[int]:






  """simple docstring"""





  if lang not in self.cache_moses_tokenizer:
   UpperCAmelCase__							=				self.sm.MosesTokenizer(lang=__a )
   UpperCAmelCase__							=				moses_tokenizer
  return self.cache_moses_tokenizer[lang].tokenize(
      __a       ,       aggressive_dash_splits=__a       ,       return_str=__a       ,       escape=__a )




 def   UpperCamelCase__							(self       ,       __a       ,       __a ) ->					Any:






  """simple docstring"""





  if lang not in self.cache_moses_detokenizer:
   UpperCAmelCase__							=				self.sm.MosesDetokenizer(lang=__a )
   UpperCAmelCase__							=				moses_detokenizer
  return self.cache_moses_detokenizer[lang].detokenize(__a )




 def   UpperCamelCase__							(self       ,       __a ) ->					Tuple:






  """simple docstring"""





  UpperCAmelCase__							=				tuple(token[:-1] ) + (token[-1] + '</w>',)
  if token in self.cache:
   return self.cache[token]
  UpperCAmelCase__							=				get_pairs(__a )

  if not pairs:
   return token + "</w>"

  while True:
   UpperCAmelCase__							=				min(__a       ,       key=lambda __a : self.bpe_ranks.get(__a       ,       float('inf' ) ) )
   if bigram not in self.bpe_ranks:
    break
   UpperCAmelCase__				,    UpperCAmelCase__							=				bigram
   UpperCAmelCase__							=				[]
   UpperCAmelCase__							=				0
   while i < len(__a ):
    try:
     UpperCAmelCase__							=				word.index(__a       ,       __a )
    except ValueError:
     new_word.extend(word[i:] )
     break
    else:
     new_word.extend(word[i:j] )
     UpperCAmelCase__							=				j

    if word[i] == first and i < len(__a ) - 1 and word[i + 1] == second:
     new_word.append(first + second )
     i += 2
    else:
     new_word.append(word[i] )
     i += 1
   UpperCAmelCase__							=				tuple(__a )
   UpperCAmelCase__							=				new_word
   if len(__a ) == 1:
    break
   else:
    UpperCAmelCase__							=				get_pairs(__a )
  UpperCAmelCase__							=				' '.join(__a )
  if word == "\n  </w>":
   UpperCAmelCase__							=				'\n</w>'
  UpperCAmelCase__							=				word
  return word




 def   UpperCamelCase__							(self       ,       __a       ,       __a=False ) ->					Dict:






  """simple docstring"""





  if bypass_tokenizer:
   UpperCAmelCase__							=				text.split()
  else:
   UpperCAmelCase__							=				self.moses_tokenize(__a       ,       self.lang )

  UpperCAmelCase__							=				[]
  for token in text:
   if token:
    split_tokens.extend(list(self.bpe(__a ).split(' ' ) ) )

  return split_tokens




 def   UpperCamelCase__							(self       ,       __a ) ->					Optional[int]:






  """simple docstring"""





  return self.encoder.get(__a       ,       self.encoder.get(self.unk_token ) )




 def   UpperCamelCase__							(self       ,       __a ) ->					Any:






  """simple docstring"""





  return self.decoder.get(__a       ,       self.unk_token )




 def   UpperCamelCase__							(self       ,       __a ) ->					str:






  """simple docstring"""





  UpperCAmelCase__							=				[t.replace(' '       ,       '' ).replace('</w>'       ,       ' ' ) for t in tokens]
  UpperCAmelCase__							=				''.join(__a ).split()
  # detokenize
  UpperCAmelCase__							=				self.moses_detokenize(__a       ,       self.lang )
  return text




 def   UpperCamelCase__							(self       ,       __a       ,       __a = None ) ->					List[int]:






  """simple docstring"""





  if token_ids_a is None:
   return [self.sep_token_id] + token_ids_a
  UpperCAmelCase__							=				[self.sep_token_id]
  return sep + token_ids_a + sep + token_ids_a




 def   UpperCamelCase__							(self       ,       __a       ,       __a = None       ,       __a = False ) ->					List[int]:






  """simple docstring"""





  if already_has_special_tokens:
   return super().get_special_tokens_mask(
       token_ids_a=__a       ,       token_ids_a=__a       ,       already_has_special_tokens=__a )
  # no bos used in fairseq
  if token_ids_a is not None:
   return [1] + ([0] * len(__a )) + [1] + ([0] * len(__a ))
  return [1] + ([0] * len(__a ))




 def   UpperCamelCase__							(self       ,       __a       ,       __a = None ) ->					List[int]:






  """simple docstring"""





  UpperCAmelCase__							=				[self.sep_token_id]

  # no bos used in fairseq
  if token_ids_a is None:
   return len(token_ids_a + sep ) * [0]
  return len(token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]




 def   UpperCamelCase__							(self       ,       __a       ,       __a = None ) ->					Tuple[str]:






  """simple docstring"""





  if not os.path.isdir(__a ):
   logger.error(F"Vocabulary path ({save_directory}) should be a directory" )
   return
  UpperCAmelCase__							=				os.path.join(
      __a       ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] )
  UpperCAmelCase__							=				os.path.join(
      __a       ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'] )

  with open(__a       ,       'w'       ,       encoding='utf-8' ) as f:
   f.write(json.dumps(self.encoder       ,       indent=2       ,       sort_keys=__a       ,       ensure_ascii=__a ) + '\n' )

  UpperCAmelCase__							=				0
  with open(__a       ,       'w'       ,       encoding='utf-8' ) as writer:
   for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,       key=lambda __a : kv[1] ):
    if index != token_index:
     logger.warning(
         F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
         ' Please check that the tokenizer is not corrupted!' )
     UpperCAmelCase__							=				token_index
    writer.write(' '.join(__a ) + '\n' )
    index += 1

  return vocab_file, merge_file




 def __getstate__(self ) ->					str:






  """simple docstring"""





  UpperCAmelCase__							=				self.__dict__.copy()
  UpperCAmelCase__							=				None
  return state




 def __setstate__(self       ,       __a ) ->					Union[str, Any]:






  """simple docstring"""





  UpperCAmelCase__							=				d

  try:
   import sacremoses
  except ImportError:
   raise ImportError(
       'You need to install sacremoses to use XLMTokenizer. '
       'See https://pypi.org/project/sacremoses/ for installation.' )

  UpperCAmelCase__							=				sacremoses







from __future__ import annotations

import unittest

from transformers import DebertaConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import tensorflow as tf

							from transformers import (
							    TFDebertaForMaskedLM,
							    TFDebertaForQuestionAnswering,
							    TFDebertaForSequenceClassification,
							    TFDebertaForTokenClassification,
							    TFDebertaModel,
							)

class   A__							:




						def __init__(				self    :					Dict ,			_a    :					List[Any] ,			_a    :					int=13 ,			_a    :					List[str]=7 ,			_a    :					Optional[Any]=True ,			_a    :					Any=True ,			_a    :					int=True ,			_a    :					Dict=True ,			_a    :					Union[str, Any]=99 ,			_a    :					str=32 ,			_a    :					Dict=2 ,			_a    :					Dict=4 ,			_a    :					str=37 ,			_a    :					Dict="gelu" ,			_a    :					Any=0.1 ,			_a    :					int=0.1 ,			_a    :					List[str]=512 ,			_a    :					Optional[int]=16 ,			_a    :					Tuple=2 ,			_a    :					List[str]=0.02 ,			_a    :					Optional[int]=3 ,			_a    :					List[Any]=4 ,			_a    :					Union[str, Any]=None ,			)							->   Tuple:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =parent
										_SCREAMING_SNAKE_CASE	     =13
										_SCREAMING_SNAKE_CASE	     =7
										_SCREAMING_SNAKE_CASE	     =True
										_SCREAMING_SNAKE_CASE	     =True
										_SCREAMING_SNAKE_CASE	     =True
										_SCREAMING_SNAKE_CASE	     =True
										_SCREAMING_SNAKE_CASE	     =99
										_SCREAMING_SNAKE_CASE	     =32
										_SCREAMING_SNAKE_CASE	     =2
										_SCREAMING_SNAKE_CASE	     =4
										_SCREAMING_SNAKE_CASE	     =37
										_SCREAMING_SNAKE_CASE	     ='''gelu'''
										_SCREAMING_SNAKE_CASE	     =0.1
										_SCREAMING_SNAKE_CASE	     =0.1
										_SCREAMING_SNAKE_CASE	     =512
										_SCREAMING_SNAKE_CASE	     =16
										_SCREAMING_SNAKE_CASE	     =False
										_SCREAMING_SNAKE_CASE	     =-1
										_SCREAMING_SNAKE_CASE	     =True
										_SCREAMING_SNAKE_CASE	     =2
										_SCREAMING_SNAKE_CASE	     =0.02
										_SCREAMING_SNAKE_CASE	     =3
										_SCREAMING_SNAKE_CASE	     =4
										_SCREAMING_SNAKE_CASE	     =None




						def 							__UpperCamelCase (				self    :					Tuple     )							->   Any:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =ids_tensor([self.batch_size, self.seq_length] ,			self.vocab_size     )

										_SCREAMING_SNAKE_CASE	     =None
										if self.use_input_mask:
														_SCREAMING_SNAKE_CASE	     =random_attention_mask([self.batch_size, self.seq_length]     )

										_SCREAMING_SNAKE_CASE	     =None
										if self.use_token_type_ids:
														_SCREAMING_SNAKE_CASE	     =ids_tensor([self.batch_size, self.seq_length] ,			self.type_vocab_size     )

										_SCREAMING_SNAKE_CASE	     =None
										_SCREAMING_SNAKE_CASE	     =None
										_SCREAMING_SNAKE_CASE	     =None
										if self.use_labels:
														_SCREAMING_SNAKE_CASE	     =ids_tensor([self.batch_size] ,			self.type_sequence_label_size     )
														_SCREAMING_SNAKE_CASE	     =ids_tensor([self.batch_size, self.seq_length] ,			self.num_labels     )

										_SCREAMING_SNAKE_CASE	     =DebertaConfig(
										    vocab_size=self.vocab_size ,			hidden_size=self.hidden_size ,			num_hidden_layers=self.num_hidden_layers ,			num_attention_heads=self.num_attention_heads ,			intermediate_size=self.intermediate_size ,			hidden_act=self.hidden_act ,			hidden_dropout_prob=self.hidden_dropout_prob ,			attention_probs_dropout_prob=self.attention_probs_dropout_prob ,			max_position_embeddings=self.max_position_embeddings ,			type_vocab_size=self.type_vocab_size ,			relative_attention=self.relative_attention ,			max_relative_positions=self.max_relative_positions ,			position_biased_input=self.position_biased_input ,			initializer_range=self.initializer_range ,			return_dict=_a ,			)

										return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




						def 							__UpperCamelCase (				self    :					Optional[int] ,			_a    :					Optional[Any] ,			_a    :					Dict ,			_a    :					List[str] ,			_a    :					Dict ,			_a    :					List[Any] ,			_a    :					Dict ,			_a    :					Tuple     )							->   Optional[int]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =TFDebertaModel(config=_a     )
										_SCREAMING_SNAKE_CASE	     ={'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}

										_SCREAMING_SNAKE_CASE	     =[input_ids, input_mask]
										_SCREAMING_SNAKE_CASE	     =model(_a     )

										_SCREAMING_SNAKE_CASE	     =model(_a     )

										self.parent.assertEqual(result.last_hidden_state.shape ,			(self.batch_size, self.seq_length, self.hidden_size)     )




						def 							__UpperCamelCase (				self    :					int ,			_a    :					List[Any] ,			_a    :					Tuple ,			_a    :					List[str] ,			_a    :					Optional[int] ,			_a    :					Union[str, Any] ,			_a    :					Optional[Any] ,			_a    :					Optional[int]     )							->   int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =TFDebertaForMaskedLM(config=_a     )
										_SCREAMING_SNAKE_CASE	     ={
										    '''input_ids''': input_ids,
										    '''attention_mask''': input_mask,
										    '''token_type_ids''': token_type_ids,
										}
										_SCREAMING_SNAKE_CASE	     =model(_a     )
										self.parent.assertEqual(result.logits.shape ,			(self.batch_size, self.seq_length, self.vocab_size)     )




						def 							__UpperCamelCase (				self    :					Union[str, Any] ,			_a    :					List[str] ,			_a    :					List[Any] ,			_a    :					Optional[int] ,			_a    :					List[Any] ,			_a    :					Optional[Any] ,			_a    :					Dict ,			_a    :					Optional[int]     )							->   List[str]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.num_labels
										_SCREAMING_SNAKE_CASE	     =TFDebertaForSequenceClassification(config=_a     )
										_SCREAMING_SNAKE_CASE	     ={
										    '''input_ids''': input_ids,
										    '''attention_mask''': input_mask,
										    '''token_type_ids''': token_type_ids,
										}

										_SCREAMING_SNAKE_CASE	     =model(_a     )
										self.parent.assertEqual(result.logits.shape ,			(self.batch_size, self.num_labels)     )




						def 							__UpperCamelCase (				self    :					List[Any] ,			_a    :					Any ,			_a    :					Optional[int] ,			_a    :					Optional[Any] ,			_a    :					Optional[Any] ,			_a    :					Optional[int] ,			_a    :					Optional[int] ,			_a    :					Any     )							->   Optional[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.num_labels
										_SCREAMING_SNAKE_CASE	     =TFDebertaForTokenClassification(config=_a     )
										_SCREAMING_SNAKE_CASE	     ={
										    '''input_ids''': input_ids,
										    '''attention_mask''': input_mask,
										    '''token_type_ids''': token_type_ids,
										}
										_SCREAMING_SNAKE_CASE	     =model(_a     )
										self.parent.assertEqual(result.logits.shape ,			(self.batch_size, self.seq_length, self.num_labels)     )




						def 							__UpperCamelCase (				self    :					List[Any] ,			_a    :					Dict ,			_a    :					str ,			_a    :					List[Any] ,			_a    :					Optional[Any] ,			_a    :					Optional[int] ,			_a    :					Optional[int] ,			_a    :					List[str]     )							->   Any:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =TFDebertaForQuestionAnswering(config=_a     )
										_SCREAMING_SNAKE_CASE	     ={
										    '''input_ids''': input_ids,
										    '''attention_mask''': input_mask,
										    '''token_type_ids''': token_type_ids,
										}

										_SCREAMING_SNAKE_CASE	     =model(_a     )
										self.parent.assertEqual(result.start_logits.shape ,			(self.batch_size, self.seq_length)     )
										self.parent.assertEqual(result.end_logits.shape ,			(self.batch_size, self.seq_length)     )




						def 							__UpperCamelCase (				self    :					str     )							->   List[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.prepare_config_and_inputs()
										(
										    (
										    _SCREAMING_SNAKE_CASE
										)     , (
										    _SCREAMING_SNAKE_CASE
										)     , (
										    _SCREAMING_SNAKE_CASE
										)     , (
										    _SCREAMING_SNAKE_CASE
										)     , (
										    _SCREAMING_SNAKE_CASE
										)     , (
										    _SCREAMING_SNAKE_CASE
										)     , (
										    _SCREAMING_SNAKE_CASE
										)     , 
										)	     =config_and_inputs
										_SCREAMING_SNAKE_CASE	     ={'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
										return config, inputs_dict

@require_tf
class   A__							(						UpperCamelCase__      ,   UpperCamelCase__      ,   unittest.TestCase     ):
						UpperCAmelCase									=  (
						    (
						        TFDebertaModel,
						        TFDebertaForMaskedLM,
						        TFDebertaForQuestionAnswering,
						        TFDebertaForSequenceClassification,
						        TFDebertaForTokenClassification,
						    )
						    if is_tf_available()
						    else ()
						)
						UpperCAmelCase									=  (
						    {
						        "feature-extraction": TFDebertaModel,
						        "fill-mask": TFDebertaForMaskedLM,
						        "question-answering": TFDebertaForQuestionAnswering,
						        "text-classification": TFDebertaForSequenceClassification,
						        "token-classification": TFDebertaForTokenClassification,
						        "zero-shot": TFDebertaForSequenceClassification,
						    }
						    if is_tf_available()
						    else {}
						)

						UpperCAmelCase									=  False
						UpperCAmelCase									=  False




						def 							__UpperCamelCase (				self    :					Optional[int]     )							->   Optional[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =TFDebertaModelTester(self     )
										_SCREAMING_SNAKE_CASE	     =ConfigTester(self ,			config_class=_a ,			hidden_size=37     )




						def 							__UpperCamelCase (				self    :					List[str]     )							->   Any:


										"""simple docstring"""





										self.config_tester.run_common_tests()




						def 							__UpperCamelCase (				self    :					Optional[Any]     )							->   Optional[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*_a     )




						def 							__UpperCamelCase (				self    :					Any     )							->   List[str]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_masked_lm(*_a     )




						def 							__UpperCamelCase (				self    :					List[Any]     )							->   List[str]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_question_answering(*_a     )




						def 							__UpperCamelCase (				self    :					Dict     )							->   Optional[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_sequence_classification(*_a     )




						def 							__UpperCamelCase (				self    :					Optional[Any]     )							->   Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_token_classification(*_a     )




						@slow
						def 							__UpperCamelCase (				self    :					List[Any]     )							->   List[str]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =TFDebertaModel.from_pretrained('''kamalkraj/deberta-base'''     )
										self.assertIsNotNone(_a     )

@require_tf
class   A__							(						unittest.TestCase     ):




						@unittest.skip(reason='''Model not available yet'''     )
						def 							__UpperCamelCase (				self    :					Tuple     )							->   List[str]:


										"""simple docstring"""





										pass




						@slow
						def 							__UpperCamelCase (				self    :					List[Any]     )							->   int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE	     =TFDebertaModel.from_pretrained('''kamalkraj/deberta-base'''     )
										_SCREAMING_SNAKE_CASE	     =tf.constant([[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]]     )
										_SCREAMING_SNAKE_CASE	     =tf.constant([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]     )
										_SCREAMING_SNAKE_CASE	     =model(_a ,			attention_mask=_a     )[0]

										_SCREAMING_SNAKE_CASE	     =tf.constant(
										    [
										        [
										            [-0.59_85_58_96, -0.80_55_25_66, -0.8_46_21_35],
										            [1.4_48_40_25, -0.93_48_37_94, -0.80_59_30_85],
										            [0.3_12_27_41, 0.00_31_60_59, -1.4_13_13_77],
										        ]
										    ]     )
										tf.debugging.assert_near(output[:, 1:4, 1:4] ,			_a ,			atol=1E-4     )