





from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import (
      TFConvBertForMaskedLM,
      TFConvBertForMultipleChoice,
      TFConvBertForQuestionAnswering,
      TFConvBertForSequenceClassification,
      TFConvBertForTokenClassification,
      TFConvBertModel,
  )






class 					lowercase__ :

   '''simple docstring'''


   def __init__(					self,	__magic_name__,	__magic_name__=13,	__magic_name__=7,	__magic_name__=True,	__magic_name__=True,	__magic_name__=True,	__magic_name__=True,	__magic_name__=99,	__magic_name__=32,	__magic_name__=2,	__magic_name__=4,	__magic_name__=37,	__magic_name__="gelu",	__magic_name__=0.1,	__magic_name__=0.1,	__magic_name__=512,	__magic_name__=16,	__magic_name__=2,	__magic_name__=0.02,	__magic_name__=3,	__magic_name__=4,	__magic_name__=None,	)				->   Any:






       """simple docstring"""
       UpperCamelCase__							:       int      =   parent
       UpperCamelCase__							:       Tuple      =   13
       UpperCamelCase__							:       Optional[Any]      =   7
       UpperCamelCase__							:       Dict      =   True
       UpperCamelCase__							:       Optional[int]      =   True
       UpperCamelCase__							:       Tuple      =   True
       UpperCamelCase__							:       Tuple      =   True
       UpperCamelCase__							:       Union[str, Any]      =   99
       UpperCamelCase__							:       List[str]      =   384
       UpperCamelCase__							:       List[str]      =   2
       UpperCamelCase__							:       Union[str, Any]      =   4
       UpperCamelCase__							:       Optional[Any]      =   37
       UpperCamelCase__							:       Tuple      =   '''gelu'''
       UpperCamelCase__							:       int      =   0.1
       UpperCamelCase__							:       Optional[Any]      =   0.1
       UpperCamelCase__							:       Optional[Any]      =   512
       UpperCamelCase__							:       List[str]      =   16
       UpperCamelCase__							:       str      =   2
       UpperCamelCase__							:       List[str]      =   0.02
       UpperCamelCase__							:       Union[str, Any]      =   3
       UpperCamelCase__							:       Any      =   4
       UpperCamelCase__							:       Dict      =   128
       UpperCamelCase__							:       Union[str, Any]      =   2
       UpperCamelCase__							:       Tuple      =   9
       UpperCamelCase__							:       Union[str, Any]      =   1
       UpperCamelCase__							:       Dict      =   None




   def    UpperCamelCase__						(					self							)				->   Tuple:






       """simple docstring"""
       UpperCamelCase__							:       List[Any]      =   ids_tensor([self.batch_size, self.seq_length],	self.vocab_size							)

       UpperCamelCase__							:       Union[str, Any]      =   None
       if self.use_input_mask:
           UpperCamelCase__							:       Optional[Any]      =   random_attention_mask([self.batch_size, self.seq_length]							)

       UpperCamelCase__							:       Union[str, Any]      =   None
       if self.use_token_type_ids:
           UpperCamelCase__							:       str      =   ids_tensor([self.batch_size, self.seq_length],	self.type_vocab_size							)

       UpperCamelCase__							:       List[str]      =   None
       UpperCamelCase__							:       int      =   None
       UpperCamelCase__							:       Tuple      =   None
       if self.use_labels:
           UpperCamelCase__							:       Optional[int]      =   ids_tensor([self.batch_size],	self.type_sequence_label_size							)
           UpperCamelCase__							:       List[str]      =   ids_tensor([self.batch_size, self.seq_length],	self.num_labels							)
           UpperCamelCase__							:       Tuple      =   ids_tensor([self.batch_size],	self.num_choices							)

       UpperCamelCase__							:       Union[str, Any]      =   ConvBertConfig(
           vocab_size=self.vocab_size,	hidden_size=self.hidden_size,	num_hidden_layers=self.num_hidden_layers,	num_attention_heads=self.num_attention_heads,	intermediate_size=self.intermediate_size,	hidden_act=self.hidden_act,	hidden_dropout_prob=self.hidden_dropout_prob,	attention_probs_dropout_prob=self.attention_probs_dropout_prob,	max_position_embeddings=self.max_position_embeddings,	type_vocab_size=self.type_vocab_size,	initializer_range=self.initializer_range,	return_dict=__magic_name__,	)

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




   def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__							)				->   str:






       """simple docstring"""
       UpperCamelCase__							:       int      =   TFConvBertModel(config=__magic_name__							)
       UpperCamelCase__							:       Optional[int]      =   {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}

       UpperCamelCase__							:       Dict      =   [input_ids, input_mask]
       UpperCamelCase__							:       Dict      =   model(__magic_name__							)

       UpperCamelCase__							:       str      =   model(__magic_name__							)

       self.parent.assertEqual(result.last_hidden_state.shape,	(self.batch_size, self.seq_length, self.hidden_size)							)




   def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__							)				->   Optional[int]:






       """simple docstring"""
       UpperCamelCase__							:       Any      =   TFConvBertForMaskedLM(config=__magic_name__							)
       UpperCamelCase__							:       List[str]      =   {
           '''input_ids''': input_ids,
           '''attention_mask''': input_mask,
           '''token_type_ids''': token_type_ids,
       }
       UpperCamelCase__							:       Tuple      =   model(__magic_name__							)
       self.parent.assertEqual(result.logits.shape,	(self.batch_size, self.seq_length, self.vocab_size)							)




   def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__							)				->   str:






       """simple docstring"""
       UpperCamelCase__							:       int      =   self.num_labels
       UpperCamelCase__							:       Optional[Any]      =   TFConvBertForSequenceClassification(config=__magic_name__							)
       UpperCamelCase__							:       int      =   {
           '''input_ids''': input_ids,
           '''attention_mask''': input_mask,
           '''token_type_ids''': token_type_ids,
       }

       UpperCamelCase__							:       Optional[int]      =   model(__magic_name__							)
       self.parent.assertEqual(result.logits.shape,	(self.batch_size, self.num_labels)							)




   def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__							)				->   Union[str, Any]:






       """simple docstring"""
       UpperCamelCase__							:       Union[str, Any]      =   self.num_choices
       UpperCamelCase__							:       Dict      =   TFConvBertForMultipleChoice(config=__magic_name__							)
       UpperCamelCase__							:       Any      =   tf.tile(tf.expand_dims(__magic_name__,	1							),	(1, self.num_choices, 1)							)
       UpperCamelCase__							:       List[Any]      =   tf.tile(tf.expand_dims(__magic_name__,	1							),	(1, self.num_choices, 1)							)
       UpperCamelCase__							:       Any      =   tf.tile(tf.expand_dims(__magic_name__,	1							),	(1, self.num_choices, 1)							)
       UpperCamelCase__							:       int      =   {
           '''input_ids''': multiple_choice_inputs_ids,
           '''attention_mask''': multiple_choice_input_mask,
           '''token_type_ids''': multiple_choice_token_type_ids,
       }
       UpperCamelCase__							:       Any      =   model(__magic_name__							)
       self.parent.assertEqual(result.logits.shape,	(self.batch_size, self.num_choices)							)




   def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__							)				->   Union[str, Any]:






       """simple docstring"""
       UpperCamelCase__							:       Dict      =   self.num_labels
       UpperCamelCase__							:       List[str]      =   TFConvBertForTokenClassification(config=__magic_name__							)
       UpperCamelCase__							:       Union[str, Any]      =   {
           '''input_ids''': input_ids,
           '''attention_mask''': input_mask,
           '''token_type_ids''': token_type_ids,
       }
       UpperCamelCase__							:       Tuple      =   model(__magic_name__							)
       self.parent.assertEqual(result.logits.shape,	(self.batch_size, self.seq_length, self.num_labels)							)




   def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__,	__magic_name__							)				->   int:






       """simple docstring"""
       UpperCamelCase__							:       List[Any]      =   TFConvBertForQuestionAnswering(config=__magic_name__							)
       UpperCamelCase__							:       int      =   {
           '''input_ids''': input_ids,
           '''attention_mask''': input_mask,
           '''token_type_ids''': token_type_ids,
       }

       UpperCamelCase__							:       List[Any]      =   model(__magic_name__							)
       self.parent.assertEqual(result.start_logits.shape,	(self.batch_size, self.seq_length)							)
       self.parent.assertEqual(result.end_logits.shape,	(self.batch_size, self.seq_length)							)




   def    UpperCamelCase__						(					self							)				->   Union[str, Any]:






       """simple docstring"""
       UpperCamelCase__							:       Optional[Any]      =   self.prepare_config_and_inputs()
       (
           (
           UpperCamelCase__
       )     ,(
           UpperCamelCase__
       )     ,(
           UpperCamelCase__
       )     ,(
           UpperCamelCase__
       )     ,(
           UpperCamelCase__
       )     ,(
           UpperCamelCase__
       )     ,(
           UpperCamelCase__
       )     ,
       )							:       List[str]      =   config_and_inputs
       UpperCamelCase__							:       List[Any]      =   {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
       return config, inputs_dict






@require_tf
class 					lowercase__ (			__lowerCamelCase    ,       __lowerCamelCase    ,       unittest.TestCase       ):

   '''simple docstring'''


   a		:							List[str]     	=	(
       (
           TFConvBertModel,
           TFConvBertForMaskedLM,
           TFConvBertForQuestionAnswering,
           TFConvBertForSequenceClassification,
           TFConvBertForTokenClassification,
           TFConvBertForMultipleChoice,
       )
       if is_tf_available()
       else ()
   )
   a		:							Any     	=	(
       {
           "feature-extraction": TFConvBertModel,
           "fill-mask": TFConvBertForMaskedLM,
           "question-answering": TFConvBertForQuestionAnswering,
           "text-classification": TFConvBertForSequenceClassification,
           "token-classification": TFConvBertForTokenClassification,
           "zero-shot": TFConvBertForSequenceClassification,
       }
       if is_tf_available()
       else {}
   )
   a		:							Optional[int]     	=	False
   a		:							Optional[Any]     	=	False
   a		:							Tuple     	=	False




   def    UpperCamelCase__						(					self							)				->   Tuple:






       """simple docstring"""
       UpperCamelCase__							:       Union[str, Any]      =   TFConvBertModelTester(self							)
       UpperCamelCase__							:       Tuple      =   ConfigTester(self,	config_class=__magic_name__,	hidden_size=37							)




   def    UpperCamelCase__						(					self							)				->   Any:






       """simple docstring"""
       self.config_tester.run_common_tests()




   def    UpperCamelCase__						(					self							)				->   Tuple:






       """simple docstring"""
       UpperCamelCase__							:       Tuple      =   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*__magic_name__							)




   def    UpperCamelCase__						(					self							)				->   Union[str, Any]:






       """simple docstring"""
       UpperCamelCase__							:       int      =   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_lm(*__magic_name__							)




   def    UpperCamelCase__						(					self							)				->   Any:






       """simple docstring"""
       UpperCamelCase__							:       Optional[int]      =   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_multiple_choice(*__magic_name__							)




   def    UpperCamelCase__						(					self							)				->   Union[str, Any]:






       """simple docstring"""
       UpperCamelCase__							:       Any      =   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*__magic_name__							)




   def    UpperCamelCase__						(					self							)				->   List[str]:






       """simple docstring"""
       UpperCamelCase__							:       str      =   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*__magic_name__							)




   def    UpperCamelCase__						(					self							)				->   List[str]:






       """simple docstring"""
       UpperCamelCase__							:       List[str]      =   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*__magic_name__							)




   @slow
   def    UpperCamelCase__						(					self							)				->   Optional[int]:






       """simple docstring"""
       UpperCamelCase__     ,UpperCamelCase__							:       Union[str, Any]      =   self.model_tester.prepare_config_and_inputs_for_common()
       UpperCamelCase__							:       List[str]      =   True
       UpperCamelCase__							:       int      =   True

       if hasattr(__magic_name__,	'''use_cache'''							):
           UpperCamelCase__							:       Any      =   True

       UpperCamelCase__							:       str      =   getattr(self.model_tester,	'''encoder_seq_length''',	self.model_tester.seq_length							)
       UpperCamelCase__							:       str      =   getattr(self.model_tester,	'''key_length''',	__magic_name__							)

       for model_class in self.all_model_classes:
           UpperCamelCase__							:       Any      =   self._prepare_for_class(__magic_name__,	__magic_name__							)
           UpperCamelCase__							:       Any      =   model_class(__magic_name__							)
           UpperCamelCase__							:       Optional[int]      =   len(model(__magic_name__							)							)

           with tempfile.TemporaryDirectory() as tmpdirname:
               model.save_pretrained(__magic_name__,	saved_model=__magic_name__							)
               UpperCamelCase__							:       str      =   os.path.join(__magic_name__,	'''saved_model''',	'''1'''							)
               UpperCamelCase__							:       Optional[Any]      =   tf.keras.models.load_model(__magic_name__							)
               UpperCamelCase__							:       Optional[Any]      =   model(__magic_name__							)

               if self.is_encoder_decoder:
                   UpperCamelCase__							:       int      =   outputs['''encoder_hidden_states''']
                   UpperCamelCase__							:       List[Any]      =   outputs['''encoder_attentions''']
               else:
                   UpperCamelCase__							:       str      =   outputs['''hidden_states''']
                   UpperCamelCase__							:       Optional[int]      =   outputs['''attentions''']

               self.assertEqual(len(__magic_name__							),	__magic_name__							)

               UpperCamelCase__							:       Any      =   getattr(
                   self.model_tester,	'''expected_num_hidden_layers''',	self.model_tester.num_hidden_layers + 1							)

               self.assertEqual(len(__magic_name__							),	__magic_name__							)
               self.assertListEqual(
                   list(output_hidden_states[0].shape[-2:]							),	[self.model_tester.seq_length, self.model_tester.hidden_size],	)

               self.assertEqual(len(__magic_name__							),	self.model_tester.num_hidden_layers							)
               self.assertListEqual(
                   list(output_attentions[0].shape[-3:]							),	[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length],	)




   @slow
   def    UpperCamelCase__						(					self							)				->   Optional[Any]:






       """simple docstring"""
       UpperCamelCase__							:       Union[str, Any]      =   TFConvBertModel.from_pretrained('''YituTech/conv-bert-base'''							)
       self.assertIsNotNone(__magic_name__							)




   def    UpperCamelCase__						(					self							)				->   List[Any]:






       """simple docstring"""
       UpperCamelCase__     ,UpperCamelCase__							:       Optional[int]      =   self.model_tester.prepare_config_and_inputs_for_common()
       UpperCamelCase__							:       Tuple      =   True
       UpperCamelCase__							:       List[str]      =   getattr(self.model_tester,	'''decoder_seq_length''',	self.model_tester.seq_length							)
       UpperCamelCase__							:       List[Any]      =   getattr(self.model_tester,	'''encoder_seq_length''',	self.model_tester.seq_length							)
       UpperCamelCase__							:       Dict      =   getattr(self.model_tester,	'''key_length''',	__magic_name__							)
       UpperCamelCase__							:       Union[str, Any]      =   getattr(self.model_tester,	'''key_length''',	__magic_name__							)

       def check_decoder_attentions_output(__magic_name__							):
           UpperCamelCase__							:       Dict      =   len(__magic_name__							)
           self.assertEqual(out_len % 2,	0							)
           UpperCamelCase__							:       List[Any]      =   outputs.decoder_attentions
           self.assertEqual(len(__magic_name__							),	self.model_tester.num_hidden_layers							)
           self.assertListEqual(
               list(decoder_attentions[0].shape[-3:]							),	[self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length],	)

       def check_encoder_attentions_output(__magic_name__							):
           UpperCamelCase__							:       Union[str, Any]      =   [
               t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
           ]
           self.assertEqual(len(__magic_name__							),	self.model_tester.num_hidden_layers							)
           self.assertListEqual(
               list(attentions[0].shape[-3:]							),	[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length],	)

       for model_class in self.all_model_classes:
           UpperCamelCase__							:       List[Any]      =   True
           UpperCamelCase__							:       Tuple      =   False
           UpperCamelCase__							:       int      =   model_class(__magic_name__							)
           UpperCamelCase__							:       str      =   model(self._prepare_for_class(__magic_name__,	__magic_name__							)							)
           UpperCamelCase__							:       Tuple      =   len(__magic_name__							)
           self.assertEqual(config.output_hidden_states,	__magic_name__							)
           check_encoder_attentions_output(__magic_name__							)

           if self.is_encoder_decoder:
               UpperCamelCase__							:       Optional[Any]      =   model_class(__magic_name__							)
               UpperCamelCase__							:       Dict      =   model(self._prepare_for_class(__magic_name__,	__magic_name__							)							)
               self.assertEqual(config.output_hidden_states,	__magic_name__							)
               check_decoder_attentions_output(__magic_name__							)

           # Check that output attentions can also be changed via the config
           del inputs_dict["output_attentions"]
           UpperCamelCase__							:       str      =   True
           UpperCamelCase__							:       Any      =   model_class(__magic_name__							)
           UpperCamelCase__							:       Union[str, Any]      =   model(self._prepare_for_class(__magic_name__,	__magic_name__							)							)
           self.assertEqual(config.output_hidden_states,	__magic_name__							)
           check_encoder_attentions_output(__magic_name__							)

           # Check attention is always last and order is fine
           UpperCamelCase__							:       str      =   True
           UpperCamelCase__							:       Any      =   True
           UpperCamelCase__							:       Optional[int]      =   model_class(__magic_name__							)
           UpperCamelCase__							:       List[str]      =   model(self._prepare_for_class(__magic_name__,	__magic_name__							)							)

           self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1),	len(__magic_name__							)							)
           self.assertEqual(model.config.output_hidden_states,	__magic_name__							)
           check_encoder_attentions_output(__magic_name__							)






@require_tf
class 					lowercase__ (			unittest.TestCase       ):

   '''simple docstring'''


   @slow
   def    UpperCamelCase__						(					self							)				->   int:






       """simple docstring"""
       UpperCamelCase__							:       List[str]      =   TFConvBertModel.from_pretrained('''YituTech/conv-bert-base'''							)
       UpperCamelCase__							:       Any      =   tf.constant([[0, 1, 2, 3, 4, 5]]							)
       UpperCamelCase__							:       Any      =   model(__magic_name__							)[0]

       UpperCamelCase__							:       Optional[int]      =   [1, 6, 768]
       self.assertEqual(output.shape,	__magic_name__							)

       UpperCamelCase__							:       Optional[int]      =   tf.constant(
           [
               [
                   [-0.0347_5493, -0.468_6034, -0.3063_8832],
                   [0.2263_7248, -0.2698_8646, -0.742_3424],
                   [0.1032_4868, -0.4501_3508, -0.5828_0784],
               ]
           ]							)
       tf.debugging.assert_near(output[:, :3, :3],	__magic_name__,	atol=1E-4							)








import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
  if is_torch_available():
    import torch
  if is_tf_available():
    import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


UpperCAmelCase_  							=				logging.get_logger(__name__)

UpperCAmelCase_  							=				{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase_  							=				{
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'Salesforce/codegen-350M-mono': (
            'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'
        ),
    },
}

UpperCAmelCase_  							=				{
    'Salesforce/codegen-350M-mono': 2048,
}






class 					lowercase__ (			__lowerCamelCase       ):

   '''simple docstring'''


   a		:							Union[str, Any]     	=	VOCAB_FILES_NAMES
   a		:							List[str]     	=	PRETRAINED_VOCAB_FILES_MAP
   a		:							Optional[int]     	=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   a		:							List[Any]     	=	["input_ids", "attention_mask"]
   a		:							Any     	=	CodeGenTokenizer




   def __init__(					self,	__magic_name__=None,	__magic_name__=None,	__magic_name__=None,	__magic_name__="<|endoftext|>",	__magic_name__="<|endoftext|>",	__magic_name__="<|endoftext|>",	__magic_name__=False,	**__magic_name__,	)				->   Optional[int]:






       """simple docstring"""
       super().__init__(
           __magic_name__,	__magic_name__,	tokenizer_file=__magic_name__,	unk_token=__magic_name__,	bos_token=__magic_name__,	eos_token=__magic_name__,	add_prefix_space=__magic_name__,	**__magic_name__,	)

       if kwargs.pop('''add_bos_token''',	__magic_name__							):
           UpperCamelCase__							:       List[Any]      =   kwargs.pop('''name_or_path''',	''''''							)
           raise ValueError(
               '''Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'''
               '''Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'''
               f"`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"
               f"`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"
               '''This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'''
               ''' so that the fast tokenizer works correctly.'''							)

       UpperCamelCase__							:       Dict      =   json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
       if pre_tok_state.get('''add_prefix_space''',	__magic_name__							) != add_prefix_space:
           UpperCamelCase__							:       List[str]      =   getattr(__magic_name__,	pre_tok_state.pop('''type'''							)							)
           UpperCamelCase__							:       int      =   add_prefix_space
           UpperCamelCase__							:       List[str]      =   pre_tok_class(**__magic_name__							)

       UpperCamelCase__							:       List[Any]      =   add_prefix_space




   def    UpperCamelCase__						(					self,	*__magic_name__,	**__magic_name__							)				->   BatchEncoding:






       """simple docstring"""
       UpperCamelCase__							:       List[Any]      =   kwargs.get('''is_split_into_words''',	__magic_name__							)
       assert self.add_prefix_space or not is_split_into_words, (
           f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
           "to use it with pretokenized inputs."
       )

       return super()._batch_encode_plus(*__magic_name__,	**__magic_name__							)




   def    UpperCamelCase__						(					self,	*__magic_name__,	**__magic_name__							)				->   BatchEncoding:






       """simple docstring"""
       UpperCamelCase__							:       Dict      =   kwargs.get('''is_split_into_words''',	__magic_name__							)

       assert self.add_prefix_space or not is_split_into_words, (
           f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
           "to use it with pretokenized inputs."
       )

       return super()._encode_plus(*__magic_name__,	**__magic_name__							)




   def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__ = None							)				->   Tuple[str]:






       """simple docstring"""
       UpperCamelCase__							:       List[Any]      =   self._tokenizer.model.save(__magic_name__,	name=__magic_name__							)
       return tuple(__magic_name__							)




   def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__ = False,	__magic_name__ = None,	__magic_name__ = None,	**__magic_name__,	)				->   str:






       """simple docstring"""
       UpperCamelCase__							:       Dict      =   super().decode(
           token_ids=__magic_name__,	skip_special_tokens=__magic_name__,	clean_up_tokenization_spaces=__magic_name__,	**__magic_name__,	)

       if truncate_before_pattern is not None and len(__magic_name__							) > 0:
           UpperCamelCase__							:       str      =   self.truncate(__magic_name__,	__magic_name__							)

       return decoded_text




   def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__							)				->   List[str]:






       """simple docstring"""
       def find_re(__magic_name__,	__magic_name__,	__magic_name__							):
           UpperCamelCase__							:       Dict      =   pattern.search(__magic_name__,	__magic_name__							)
           return m.start() if m else -1

       UpperCamelCase__							:       Union[str, Any]      =   [re.compile(__magic_name__,	re.MULTILINE							) for pattern in truncate_before_pattern]

       UpperCamelCase__							:       List[Any]      =   list(re.finditer('''^print''',	__magic_name__,	re.MULTILINE							)							)

       if len(__magic_name__							) > 1:
           UpperCamelCase__							:       Tuple      =   completion[: prints[1].start()]

       UpperCamelCase__							:       Optional[int]      =   list(re.finditer('''^def''',	__magic_name__,	re.MULTILINE							)							)

       if len(__magic_name__							) > 1:
           UpperCamelCase__							:       Optional[Any]      =   completion[: defs[1].start()]

       UpperCamelCase__							:       str      =   0

       UpperCamelCase__							:       Tuple      =   [
           pos for pos in [find_re(__magic_name__,	__magic_name__,	__magic_name__							) for terminal in terminals] if pos != -1
       ]

       if len(__magic_name__							) > 0:
           return completion[: min(__magic_name__							)]
       else:
           return completion

