


'''simple docstring'''



import unittest

from transformers import DebertaVaConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          DebertaVaForMaskedLM,
          DebertaVaForMultipleChoice,
          DebertaVaForQuestionAnswering,
          DebertaVaForSequenceClassification,
          DebertaVaForTokenClassification,
          DebertaVaModel,
      )
      from transformers.models.deberta_va.modeling_deberta_va import DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST





class   UpperCAmelCase_    (_UpperCAmelCase    ):





    """simple docstring"""






    def __init__(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_=13     ,	SCREAMING_SNAKE_CASE_=7     ,	SCREAMING_SNAKE_CASE_=True     ,	SCREAMING_SNAKE_CASE_=True     ,	SCREAMING_SNAKE_CASE_=True     ,	SCREAMING_SNAKE_CASE_=True     ,	SCREAMING_SNAKE_CASE_=99     ,	SCREAMING_SNAKE_CASE_=32     ,	SCREAMING_SNAKE_CASE_=5     ,	SCREAMING_SNAKE_CASE_=4     ,	SCREAMING_SNAKE_CASE_=37     ,	SCREAMING_SNAKE_CASE_="gelu"     ,	SCREAMING_SNAKE_CASE_=0.1     ,	SCREAMING_SNAKE_CASE_=0.1     ,	SCREAMING_SNAKE_CASE_=5_12     ,	SCREAMING_SNAKE_CASE_=16     ,	SCREAMING_SNAKE_CASE_=2     ,	SCREAMING_SNAKE_CASE_=0.0_2     ,	SCREAMING_SNAKE_CASE_=False     ,	SCREAMING_SNAKE_CASE_=True     ,	SCREAMING_SNAKE_CASE_="None"     ,	SCREAMING_SNAKE_CASE_=3     ,	SCREAMING_SNAKE_CASE_=4     ,	SCREAMING_SNAKE_CASE_=None     ,	)							->	Optional[int]:
         __lowerCamelCase  :      List[str]														=						parent
         __lowerCamelCase  :      List[Any]														=						batch_size
         __lowerCamelCase  :      Dict														=						seq_length
         __lowerCamelCase  :      int														=						is_training
         __lowerCamelCase  :      Any														=						use_input_mask
         __lowerCamelCase  :      int														=						use_token_type_ids
         __lowerCamelCase  :      Optional[int]														=						use_labels
         __lowerCamelCase  :      str														=						vocab_size
         __lowerCamelCase  :      List[str]														=						hidden_size
         __lowerCamelCase  :      Any														=						num_hidden_layers
         __lowerCamelCase  :      Dict														=						num_attention_heads
         __lowerCamelCase  :      int														=						intermediate_size
         __lowerCamelCase  :      int														=						hidden_act
         __lowerCamelCase  :      List[Any]														=						hidden_dropout_prob
         __lowerCamelCase  :      Optional[Any]														=						attention_probs_dropout_prob
         __lowerCamelCase  :      int														=						max_position_embeddings
         __lowerCamelCase  :      Dict														=						type_vocab_size
         __lowerCamelCase  :      str														=						type_sequence_label_size
         __lowerCamelCase  :      Any														=						initializer_range
         __lowerCamelCase  :      List[Any]														=						num_labels
         __lowerCamelCase  :      int														=						num_choices
         __lowerCamelCase  :      Union[str, Any]														=						relative_attention
         __lowerCamelCase  :      Optional[int]														=						position_biased_input
         __lowerCamelCase  :      Union[str, Any]														=						pos_att_type
         __lowerCamelCase  :      Optional[int]														=						scope







    def  lowercase_							(				self      )							->	Union[str, Any]:
         __lowerCamelCase  :      List[Any]														=						ids_tensor([self.batch_size, self.seq_length]     ,	self.vocab_size      )

         __lowerCamelCase  :      List[str]														=						None
         if self.use_input_mask:
              __lowerCamelCase  :      str														=						ids_tensor([self.batch_size, self.seq_length]     ,	vocab_size=2      )

         __lowerCamelCase  :      Union[str, Any]														=						None
         if self.use_token_type_ids:
              __lowerCamelCase  :      Tuple														=						ids_tensor([self.batch_size, self.seq_length]     ,	self.type_vocab_size      )

         __lowerCamelCase  :      List[Any]														=						None
         __lowerCamelCase  :      Union[str, Any]														=						None
         __lowerCamelCase  :      List[str]														=						None
         if self.use_labels:
              __lowerCamelCase  :      str														=						ids_tensor([self.batch_size]     ,	self.type_sequence_label_size      )
              __lowerCamelCase  :      List[Any]														=						ids_tensor([self.batch_size, self.seq_length]     ,	self.num_labels      )
              __lowerCamelCase  :      Any														=						ids_tensor([self.batch_size]     ,	self.num_choices      )

         __lowerCamelCase  :      List[str]														=						self.get_config()

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







    def  lowercase_							(				self      )							->	Optional[int]:
         return DebertaVaConfig(
             vocab_size=self.vocab_size     ,	hidden_size=self.hidden_size     ,	num_hidden_layers=self.num_hidden_layers     ,	num_attention_heads=self.num_attention_heads     ,	intermediate_size=self.intermediate_size     ,	hidden_act=self.hidden_act     ,	hidden_dropout_prob=self.hidden_dropout_prob     ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,	max_position_embeddings=self.max_position_embeddings     ,	type_vocab_size=self.type_vocab_size     ,	initializer_range=self.initializer_range     ,	relative_attention=self.relative_attention     ,	position_biased_input=self.position_biased_input     ,	pos_att_type=self.pos_att_type     ,	)







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_      )							->	Any:
         self.parent.assertListEqual(list(result.loss.size()      )     ,	[]      )







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )							->	Any:
         __lowerCamelCase  :      Union[str, Any]														=						DebertaVaModel(config=SCREAMING_SNAKE_CASE_      )
         model.to(SCREAMING_SNAKE_CASE_      )
         model.eval()
         __lowerCamelCase  :      List[Any]														=						model(SCREAMING_SNAKE_CASE_     ,	attention_mask=SCREAMING_SNAKE_CASE_     ,	token_type_ids=SCREAMING_SNAKE_CASE_      )[0]
         __lowerCamelCase  :      Any														=						model(SCREAMING_SNAKE_CASE_     ,	token_type_ids=SCREAMING_SNAKE_CASE_      )[0]
         __lowerCamelCase  :      str														=						model(SCREAMING_SNAKE_CASE_      )[0]

         self.parent.assertListEqual(list(sequence_output.size()      )     ,	[self.batch_size, self.seq_length, self.hidden_size]      )







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )							->	Tuple:
         __lowerCamelCase  :      Tuple														=						DebertaVaForMaskedLM(config=SCREAMING_SNAKE_CASE_      )
         model.to(SCREAMING_SNAKE_CASE_      )
         model.eval()
         __lowerCamelCase  :      List[str]														=						model(SCREAMING_SNAKE_CASE_     ,	attention_mask=SCREAMING_SNAKE_CASE_     ,	token_type_ids=SCREAMING_SNAKE_CASE_     ,	labels=SCREAMING_SNAKE_CASE_      )

         self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.seq_length, self.vocab_size)      )







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )							->	Tuple:
         __lowerCamelCase  :      Union[str, Any]														=						self.num_labels
         __lowerCamelCase  :      Dict														=						DebertaVaForSequenceClassification(SCREAMING_SNAKE_CASE_      )
         model.to(SCREAMING_SNAKE_CASE_      )
         model.eval()
         __lowerCamelCase  :      int														=						model(SCREAMING_SNAKE_CASE_     ,	attention_mask=SCREAMING_SNAKE_CASE_     ,	token_type_ids=SCREAMING_SNAKE_CASE_     ,	labels=SCREAMING_SNAKE_CASE_      )
         self.parent.assertListEqual(list(result.logits.size()      )     ,	[self.batch_size, self.num_labels]      )
         self.check_loss_output(SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )							->	int:
         __lowerCamelCase  :      Dict														=						self.num_labels
         __lowerCamelCase  :      int														=						DebertaVaForTokenClassification(config=SCREAMING_SNAKE_CASE_      )
         model.to(SCREAMING_SNAKE_CASE_      )
         model.eval()
         __lowerCamelCase  :      Dict														=						model(SCREAMING_SNAKE_CASE_     ,	attention_mask=SCREAMING_SNAKE_CASE_     ,	token_type_ids=SCREAMING_SNAKE_CASE_     ,	labels=SCREAMING_SNAKE_CASE_      )
         self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.seq_length, self.num_labels)      )







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )							->	Dict:
         __lowerCamelCase  :      Tuple														=						DebertaVaForQuestionAnswering(config=SCREAMING_SNAKE_CASE_      )
         model.to(SCREAMING_SNAKE_CASE_      )
         model.eval()
         __lowerCamelCase  :      List[str]														=						model(
             SCREAMING_SNAKE_CASE_     ,	attention_mask=SCREAMING_SNAKE_CASE_     ,	token_type_ids=SCREAMING_SNAKE_CASE_     ,	start_positions=SCREAMING_SNAKE_CASE_     ,	end_positions=SCREAMING_SNAKE_CASE_     ,	)
         self.parent.assertEqual(result.start_logits.shape     ,	(self.batch_size, self.seq_length)      )
         self.parent.assertEqual(result.end_logits.shape     ,	(self.batch_size, self.seq_length)      )







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )							->	str:
         __lowerCamelCase  :      List[str]														=						DebertaVaForMultipleChoice(config=SCREAMING_SNAKE_CASE_      )
         model.to(SCREAMING_SNAKE_CASE_      )
         model.eval()
         __lowerCamelCase  :      Dict														=						input_ids.unsqueeze(1      ).expand(-1     ,	self.num_choices     ,	-1      ).contiguous()
         __lowerCamelCase  :      Any														=						token_type_ids.unsqueeze(1      ).expand(-1     ,	self.num_choices     ,	-1      ).contiguous()
         __lowerCamelCase  :      List[Any]														=						input_mask.unsqueeze(1      ).expand(-1     ,	self.num_choices     ,	-1      ).contiguous()
         __lowerCamelCase  :      Optional[Any]														=						model(
             SCREAMING_SNAKE_CASE_     ,	attention_mask=SCREAMING_SNAKE_CASE_     ,	token_type_ids=SCREAMING_SNAKE_CASE_     ,	labels=SCREAMING_SNAKE_CASE_     ,	)
         self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.num_choices)      )







    def  lowercase_							(				self      )							->	Tuple:
         __lowerCamelCase  :      Tuple														=						self.prepare_config_and_inputs()
         (
             (
             __lowerCamelCase
         )							,			(
             __lowerCamelCase
         )							,			(
             __lowerCamelCase
         )							,			(
             __lowerCamelCase
         )							,			(
             __lowerCamelCase
         )							,			(
             __lowerCamelCase
         )							,			(
             __lowerCamelCase
         )							,			
         )  :      Optional[Any]														=						config_and_inputs
         __lowerCamelCase  :      List[str]														=						{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
         return config, inputs_dict







@require_torch
class   UpperCAmelCase_    (_UpperCAmelCase   ,						_UpperCAmelCase   ,						unittest.TestCase    ):





    """simple docstring"""






    lowerCamelCase    :   List[Any]					=     (
        (
            DebertaVaModel,
            DebertaVaForMaskedLM,
            DebertaVaForSequenceClassification,
            DebertaVaForTokenClassification,
            DebertaVaForQuestionAnswering,
            DebertaVaForMultipleChoice,
        )
        if is_torch_available()
        else ()
    )
    lowerCamelCase    :   Any					=     (
        {
            'feature-extraction': DebertaVaModel,
            'fill-mask': DebertaVaForMaskedLM,
            'question-answering': DebertaVaForQuestionAnswering,
            'text-classification': DebertaVaForSequenceClassification,
            'token-classification': DebertaVaForTokenClassification,
            'zero-shot': DebertaVaForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )

    lowerCamelCase    :   Any					=     True
    lowerCamelCase    :   List[Any]					=     False
    lowerCamelCase    :   Union[str, Any]					=     False
    lowerCamelCase    :   Optional[int]					=     False
    lowerCamelCase    :   Dict					=     False







    def  lowercase_							(				self      )							->	str:
         __lowerCamelCase  :      Optional[int]														=						DebertaVaModelTester(self      )
         __lowerCamelCase  :      Union[str, Any]														=						ConfigTester(self     ,	config_class=SCREAMING_SNAKE_CASE_     ,	hidden_size=37      )







    def  lowercase_							(				self      )							->	Optional[Any]:
         self.config_tester.run_common_tests()







    def  lowercase_							(				self      )							->	Any:
         __lowerCamelCase  :      Tuple														=						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_deberta_model(*SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self      )							->	str:
         __lowerCamelCase  :      Tuple														=						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_deberta_for_sequence_classification(*SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self      )							->	Any:
         __lowerCamelCase  :      int														=						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_deberta_for_masked_lm(*SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self      )							->	Any:
         __lowerCamelCase  :      Optional[Any]														=						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_deberta_for_question_answering(*SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self      )							->	int:
         __lowerCamelCase  :      Tuple														=						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_deberta_for_token_classification(*SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self      )							->	Any:
         __lowerCamelCase  :      Optional[Any]														=						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_deberta_for_multiple_choice(*SCREAMING_SNAKE_CASE_      )







    @slow
    def  lowercase_							(				self      )							->	Dict:
         for model_name in DEBERTA_V2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              __lowerCamelCase  :      str														=						DebertaVaModel.from_pretrained(SCREAMING_SNAKE_CASE_      )
              self.assertIsNotNone(SCREAMING_SNAKE_CASE_      )







@require_torch
@require_sentencepiece
@require_tokenizers
class   UpperCAmelCase_    (unittest.TestCase    ):





    """simple docstring"""






    @unittest.skip(reason='Model not available yet'      )
    def  lowercase_							(				self      )							->	int:
         pass







    @slow
    def  lowercase_							(				self      )							->	Union[str, Any]:
         __lowerCamelCase  :      Dict														=						DebertaVaModel.from_pretrained('microsoft/deberta-v2-xlarge'      )

         __lowerCamelCase  :      Optional[Any]														=						torch.tensor([[0, 3_14_14, 2_32, 3_28, 7_40, 11_40, 1_26_95, 69, 4_60_78, 15_88, 2]]      )
         __lowerCamelCase  :      Tuple														=						torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]      )
         with torch.no_grad():
              __lowerCamelCase  :      Tuple														=						model(SCREAMING_SNAKE_CASE_     ,	attention_mask=SCREAMING_SNAKE_CASE_      )[0]
         # compare the actual values for a slice.
         __lowerCamelCase  :      Any														=						torch.tensor(
             [[[0.2_3_5_6, 0.1_9_4_8, 0.0_3_6_9], [-0.1_0_6_3, 0.3_5_8_6, -0.5_1_5_2], [-0.6_3_9_9, -0.0_2_5_9, -0.2_5_2_5]]]      )
         self.assertTrue(torch.allclose(output[:, 1:4, 1:4]     ,	SCREAMING_SNAKE_CASE_     ,	atol=1E-4      )     ,	f'{output[:, 1:4, 1:4]}'      )








import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__magic_name__  							=	logging.get_logger(__name__)

__magic_name__  							=	{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

__magic_name__  							=	{
    '''vocab_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'''},
    '''merges_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'''},
}

__magic_name__  							=	{
    '''ctrl''': 256,
}

__magic_name__  							=	{
    '''Pregnancy''': 168_629,
    '''Christianity''': 7_675,
    '''Explain''': 106_423,
    '''Fitness''': 63_440,
    '''Saving''': 63_163,
    '''Ask''': 27_171,
    '''Ass''': 95_985,
    '''Joke''': 163_509,
    '''Questions''': 45_622,
    '''Thoughts''': 49_605,
    '''Retail''': 52_342,
    '''Feminism''': 164_338,
    '''Writing''': 11_992,
    '''Atheism''': 192_263,
    '''Netflix''': 48_616,
    '''Computing''': 39_639,
    '''Opinion''': 43_213,
    '''Alone''': 44_967,
    '''Funny''': 58_917,
    '''Gaming''': 40_358,
    '''Human''': 4_088,
    '''India''': 1_331,
    '''Joker''': 77_138,
    '''Diet''': 36_206,
    '''Legal''': 11_859,
    '''Norman''': 4_939,
    '''Tip''': 72_689,
    '''Weight''': 52_343,
    '''Movies''': 46_273,
    '''Running''': 23_425,
    '''Science''': 2_090,
    '''Horror''': 37_793,
    '''Confession''': 60_572,
    '''Finance''': 12_250,
    '''Politics''': 16_360,
    '''Scary''': 191_985,
    '''Support''': 12_654,
    '''Technologies''': 32_516,
    '''Teenage''': 66_160,
    '''Event''': 32_769,
    '''Learned''': 67_460,
    '''Notion''': 182_770,
    '''Wikipedia''': 37_583,
    '''Books''': 6_665,
    '''Extract''': 76_050,
    '''Confessions''': 102_701,
    '''Conspiracy''': 75_932,
    '''Links''': 63_674,
    '''Narcissus''': 150_425,
    '''Relationship''': 54_766,
    '''Relationships''': 134_796,
    '''Reviews''': 41_671,
    '''News''': 4_256,
    '''Translation''': 26_820,
    '''multilingual''': 128_406,
}





def 						SCREAMING_SNAKE_CASE__     (   __lowerCAmelCase				):
    snake_case__    =					set()
    snake_case__    =					word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)				)
        snake_case__    =					char

    snake_case__    =					set(__lowerCAmelCase				)
    return pairs





class        _SCREAMING_SNAKE_CASE	(   __UpperCamelCase			):
      _A    :		Tuple     						=  VOCAB_FILES_NAMES
      _A    :		str     						=  PRETRAINED_VOCAB_FILES_MAP
      _A    :		int     						=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _A    :		List[Any]     						=  CONTROL_CODES
      def __init__(      self		,						lowerCamelCase		,						lowerCamelCase		,						lowerCamelCase="<unk>"		,						**lowerCamelCase			):
          super().__init__(unk_token=lowerCamelCase		,						**lowerCamelCase			)

          with open(lowerCamelCase		,						encoding="utf-8"			) as vocab_handle:
              snake_case__    =					json.load(lowerCamelCase			)
          snake_case__    =					{v: k for k, v in self.encoder.items()}
          with open(lowerCamelCase		,						encoding="utf-8"			) as merges_handle:
              snake_case__    =					merges_handle.read().split("\n"			)[1:-1]
          snake_case__    =					[tuple(merge.split()			) for merge in merges]
          snake_case__    =					dict(zip(lowerCamelCase		,						range(len(lowerCamelCase			)			)			)			)
          snake_case__    =					{}
      @property
      def A_      (      self			):
          return len(self.encoder			)
      def A_      (      self			):
          return dict(self.encoder		,						**self.added_tokens_encoder			)
      def A_      (      self		,						lowerCamelCase			):
          if token in self.cache:
              return self.cache[token]
          snake_case__    =					tuple(lowerCamelCase			)
          snake_case__    =					tuple(list(word[:-1]			) + [word[-1] + "</w>"]			)
          snake_case__    =					get_pairs(lowerCamelCase			)

          if not pairs:
              return token

          while True:
              snake_case__    =					min(lowerCamelCase		,						key=lambda lowerCamelCase			: self.bpe_ranks.get(lowerCamelCase		,						float("inf"			)			)			)
              if bigram not in self.bpe_ranks:
                  break
              snake_case__    ,       snake_case__    =					bigram
              snake_case__    =					[]
              snake_case__    =					0
              while i < len(lowerCamelCase			):
                  try:
                      snake_case__    =					word.index(lowerCamelCase		,						lowerCamelCase			)
                  except ValueError:
                      new_word.extend(word[i:]			)
                      break
                  else:
                      new_word.extend(word[i:j]			)
                      snake_case__    =					j

                  if word[i] == first and i < len(lowerCamelCase			) - 1 and word[i + 1] == second:
                      new_word.append(first + second			)
                      i += 2
                  else:
                      new_word.append(word[i]			)
                      i += 1
              snake_case__    =					tuple(lowerCamelCase			)
              snake_case__    =					new_word
              if len(lowerCamelCase			) == 1:
                  break
              else:
                  snake_case__    =					get_pairs(lowerCamelCase			)
          snake_case__    =					"@@ ".join(lowerCamelCase			)
          snake_case__    =					word[:-4]
          snake_case__    =					word
          return word
      def A_      (      self		,						lowerCamelCase			):
          snake_case__    =					[]

          snake_case__    =					re.findall(r"\S+\n?"		,						lowerCamelCase			)

          for token in words:
              split_tokens.extend(list(self.bpe(lowerCamelCase			).split(" "			)			)			)
          return split_tokens
      def A_      (      self		,						lowerCamelCase			):
          return self.encoder.get(lowerCamelCase		,						self.encoder.get(self.unk_token			)			)
      def A_      (      self		,						lowerCamelCase			):
          return self.decoder.get(lowerCamelCase		,						self.unk_token			)
      def A_      (      self		,						lowerCamelCase			):
          snake_case__    =					" ".join(lowerCamelCase			).replace("@@ "		,						""			).strip()
          return out_string



      def A_      (      self		,						lowerCamelCase		,						lowerCamelCase = None			):
          if not os.path.isdir(lowerCamelCase			):
              logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""			)
              return
          snake_case__    =					os.path.join(
              lowerCamelCase		,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]			)
          snake_case__    =					os.path.join(
              lowerCamelCase		,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]			)

          with open(lowerCamelCase		,						"w"		,						encoding="utf-8"			) as f:
              f.write(json.dumps(self.encoder		,						indent=2		,						sort_keys=lowerCamelCase		,						ensure_ascii=lowerCamelCase			) + "\n"			)

          snake_case__    =					0
          with open(lowerCamelCase		,						"w"		,						encoding="utf-8"			) as writer:
              writer.write("#version: 0.2\n"			)
              for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,						key=lambda lowerCamelCase			: kv[1]			):
                  if index != token_index:
                      logger.warning(
                          F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                          " Please check that the tokenizer is not corrupted!"			)
                      snake_case__    =					token_index
                  writer.write(" ".join(lowerCamelCase			) + "\n"			)
                  index += 1

          return vocab_file, merge_file

      # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
      #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
      #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
      #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
      #     return ''.join(tokens_generated_so_far)



