






"""simple docstring"""



import unittest

from transformers import MarkupLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     MarkupLMForQuestionAnswering,
     MarkupLMForSequenceClassification,
     MarkupLMForTokenClassification,
     MarkupLMModel,
 )

# TODO check dependencies
from transformers import MarkupLMFeatureExtractor, MarkupLMProcessor, MarkupLMTokenizer




class        lowerCAmelCase__       :
   def __init__(						self	:     Optional[Any]				,       snake_case__	:     Optional[int]				,       snake_case__	:     Optional[int]=1_3				,       snake_case__	:     List[Any]=7				,       snake_case__	:     Any=True				,       snake_case__	:     Optional[int]=True				,       snake_case__	:     Union[str, Any]=True				,       snake_case__	:     List[Any]=True				,       snake_case__	:     Tuple=9_9				,       snake_case__	:     Optional[Any]=3_2				,       snake_case__	:     int=5				,       snake_case__	:     int=4				,       snake_case__	:     Optional[int]=3_7				,       snake_case__	:     Union[str, Any]="gelu"				,       snake_case__	:     int=0.1				,       snake_case__	:     str=0.1				,       snake_case__	:     int=5_1_2				,       snake_case__	:     Optional[Any]=1_6				,       snake_case__	:     str=2				,       snake_case__	:     int=0.02				,       snake_case__	:     Optional[int]=3				,       snake_case__	:     int=None				,       snake_case__	:     Any=2_0				,       snake_case__	:     str=3_0				,       snake_case__	:     str=2				,       snake_case__	:     Optional[int]=2				,       snake_case__	:     Optional[Any]=1_0				,       ):


        '''simple docstring'''
        UpperCAmelCase__		:  Union[str, Any]				=				parent
        UpperCAmelCase__		:  List[Any]				=				batch_size
        UpperCAmelCase__		:  Tuple				=				seq_length
        UpperCAmelCase__		:  Optional[int]				=				is_training
        UpperCAmelCase__		:  List[Any]				=				use_input_mask
        UpperCAmelCase__		:  Optional[int]				=				use_token_type_ids
        UpperCAmelCase__		:  str				=				use_labels
        UpperCAmelCase__		:  int				=				vocab_size
        UpperCAmelCase__		:  List[Any]				=				hidden_size
        UpperCAmelCase__		:  Dict				=				num_hidden_layers
        UpperCAmelCase__		:  Any				=				num_attention_heads
        UpperCAmelCase__		:  Optional[Any]				=				intermediate_size
        UpperCAmelCase__		:  Optional[int]				=				hidden_act
        UpperCAmelCase__		:  Tuple				=				hidden_dropout_prob
        UpperCAmelCase__		:  Any				=				attention_probs_dropout_prob
        UpperCAmelCase__		:  Union[str, Any]				=				max_position_embeddings
        UpperCAmelCase__		:  int				=				type_vocab_size
        UpperCAmelCase__		:  Union[str, Any]				=				type_sequence_label_size
        UpperCAmelCase__		:  Any				=				initializer_range
        UpperCAmelCase__		:  Optional[Any]				=				num_labels
        UpperCAmelCase__		:  Optional[Any]				=				scope
        UpperCAmelCase__		:  List[Any]				=				max_xpath_tag_unit_embeddings
        UpperCAmelCase__		:  Dict				=				max_xpath_subs_unit_embeddings
        UpperCAmelCase__		:  Union[str, Any]				=				tag_pad_id
        UpperCAmelCase__		:  Optional[int]				=				subs_pad_id
        UpperCAmelCase__		:  Optional[int]				=				max_depth
   def 	__a				(						self	:     Optional[int]				):


        '''simple docstring'''
        UpperCAmelCase__		:  Optional[int]				=				ids_tensor([self.batch_size, self.seq_length]				,       self.vocab_size				)

        UpperCAmelCase__		:  str				=				ids_tensor(
            [self.batch_size, self.seq_length, self.max_depth]				,       self.max_xpath_tag_unit_embeddings				)

        UpperCAmelCase__		:  int				=				ids_tensor(
            [self.batch_size, self.seq_length, self.max_depth]				,       self.max_xpath_subs_unit_embeddings				)

        UpperCAmelCase__		:  int				=				None
        if self.use_input_mask:
             UpperCAmelCase__		:  List[Any]				=				ids_tensor([self.batch_size, self.seq_length]				,       vocab_size=2				)

        UpperCAmelCase__		:  Any				=				None
        if self.use_token_type_ids:
             UpperCAmelCase__		:  Optional[int]				=				ids_tensor([self.batch_size, self.seq_length]				,       self.type_vocab_size				)

        UpperCAmelCase__		:  Dict				=				None
        UpperCAmelCase__		:  Optional[int]				=				None
        if self.use_labels:
             UpperCAmelCase__		:  Optional[int]				=				ids_tensor([self.batch_size]				,       self.type_sequence_label_size				)
             UpperCAmelCase__		:  Tuple				=				ids_tensor([self.batch_size, self.seq_length]				,       self.num_labels				)

        UpperCAmelCase__		:  int				=				self.get_config()

        return (
            config,
            input_ids,
            xpath_tags_seq,
            xpath_subs_seq,
            token_type_ids,
            input_mask,
            sequence_labels,
            token_labels,
        )
   def 	__a				(						self	:     Union[str, Any]				):


        '''simple docstring'''
        return MarkupLMConfig(
            vocab_size=self.vocab_size				,       hidden_size=self.hidden_size				,       num_hidden_layers=self.num_hidden_layers				,       num_attention_heads=self.num_attention_heads				,       intermediate_size=self.intermediate_size				,       hidden_act=self.hidden_act				,       hidden_dropout_prob=self.hidden_dropout_prob				,       attention_probs_dropout_prob=self.attention_probs_dropout_prob				,       max_position_embeddings=self.max_position_embeddings				,       type_vocab_size=self.type_vocab_size				,       initializer_range=self.initializer_range				,       max_xpath_tag_unit_embeddings=self.max_xpath_tag_unit_embeddings				,       max_xpath_subs_unit_embeddings=self.max_xpath_subs_unit_embeddings				,       tag_pad_id=self.tag_pad_id				,       subs_pad_id=self.subs_pad_id				,       max_depth=self.max_depth				,       )
   def 	__a				(						self	:     Tuple				,       snake_case__	:     Any				,       snake_case__	:     str				,       snake_case__	:     List[str]				,       snake_case__	:     Any				,       snake_case__	:     Optional[Any]				,       snake_case__	:     int				,       snake_case__	:     List[str]				,       snake_case__	:     str				,       ):


        '''simple docstring'''
        UpperCAmelCase__		:  List[Any]				=				MarkupLMModel(config=snake_case__				)
        model.to(snake_case__				)
        model.eval()
        print("Configs:"				,       model.config.tag_pad_id				,       model.config.subs_pad_id				)
        UpperCAmelCase__		:  Tuple				=				model(snake_case__				,       attention_mask=snake_case__				,       token_type_ids=snake_case__				)
        UpperCAmelCase__		:  List[Any]				=				model(snake_case__				,       token_type_ids=snake_case__				)
        UpperCAmelCase__		:  Union[str, Any]				=				model(snake_case__				)
        self.parent.assertEqual(result.last_hidden_state.shape				,       (self.batch_size, self.seq_length, self.hidden_size)				)
        self.parent.assertEqual(result.pooler_output.shape				,       (self.batch_size, self.hidden_size)				)
   def 	__a				(						self	:     List[Any]				,       snake_case__	:     Any				,       snake_case__	:     List[str]				,       snake_case__	:     str				,       snake_case__	:     List[Any]				,       snake_case__	:     List[Any]				,       snake_case__	:     str				,       snake_case__	:     Dict				,       snake_case__	:     Any				,       ):


        '''simple docstring'''
        UpperCAmelCase__		:  Optional[int]				=				self.num_labels
        UpperCAmelCase__		:  Any				=				MarkupLMForSequenceClassification(snake_case__				)
        model.to(snake_case__				)
        model.eval()
        UpperCAmelCase__		:  Tuple				=				model(
            snake_case__				,       xpath_tags_seq=snake_case__				,       xpath_subs_seq=snake_case__				,       attention_mask=snake_case__				,       token_type_ids=snake_case__				,       labels=snake_case__				,       )
        self.parent.assertEqual(result.logits.shape				,       (self.batch_size, self.num_labels)				)
   def 	__a				(						self	:     List[str]				,       snake_case__	:     List[str]				,       snake_case__	:     Optional[int]				,       snake_case__	:     str				,       snake_case__	:     Optional[int]				,       snake_case__	:     Union[str, Any]				,       snake_case__	:     Dict				,       snake_case__	:     Optional[int]				,       snake_case__	:     Any				,       ):


        '''simple docstring'''
        UpperCAmelCase__		:  Dict				=				self.num_labels
        UpperCAmelCase__		:  List[Any]				=				MarkupLMForTokenClassification(config=snake_case__				)
        model.to(snake_case__				)
        model.eval()
        UpperCAmelCase__		:  Dict				=				model(
            snake_case__				,       xpath_tags_seq=snake_case__				,       xpath_subs_seq=snake_case__				,       attention_mask=snake_case__				,       token_type_ids=snake_case__				,       labels=snake_case__				,       )
        self.parent.assertEqual(result.logits.shape				,       (self.batch_size, self.seq_length, self.num_labels)				)
   def 	__a				(						self	:     Tuple				,       snake_case__	:     str				,       snake_case__	:     Optional[int]				,       snake_case__	:     Optional[int]				,       snake_case__	:     List[str]				,       snake_case__	:     Optional[Any]				,       snake_case__	:     Optional[int]				,       snake_case__	:     Optional[int]				,       snake_case__	:     Any				,       ):


        '''simple docstring'''
        UpperCAmelCase__		:  str				=				MarkupLMForQuestionAnswering(config=snake_case__				)
        model.to(snake_case__				)
        model.eval()
        UpperCAmelCase__		:  Dict				=				model(
            snake_case__				,       xpath_tags_seq=snake_case__				,       xpath_subs_seq=snake_case__				,       attention_mask=snake_case__				,       token_type_ids=snake_case__				,       start_positions=snake_case__				,       end_positions=snake_case__				,       )
        self.parent.assertEqual(result.start_logits.shape				,       (self.batch_size, self.seq_length)				)
        self.parent.assertEqual(result.end_logits.shape				,       (self.batch_size, self.seq_length)				)




   def 	__a				(						self	:     Optional[int]				):


        '''simple docstring'''
        UpperCAmelCase__		:  Dict				=				self.prepare_config_and_inputs()
        (
            (
            UpperCAmelCase__
        )				,							(
            UpperCAmelCase__
        )				,							(
            UpperCAmelCase__
        )				,							(
            UpperCAmelCase__
        )				,							(
            UpperCAmelCase__
        )				,							(
            UpperCAmelCase__
        )				,							(
            UpperCAmelCase__
        )				,							(
            UpperCAmelCase__
        )				,							
        )		:  Any				=				config_and_inputs
        UpperCAmelCase__		:  List[Any]				=				{
            "input_ids": input_ids,
            "xpath_tags_seq": xpath_tags_seq,
            "xpath_subs_seq": xpath_subs_seq,
            "token_type_ids": token_type_ids,
            "attention_mask": input_mask,
        }
        return config, inputs_dict




@require_torch
class        lowerCAmelCase__       (  __magic_name__				,		__magic_name__				,		unittest.TestCase					):
   SCREAMING_SNAKE_CASE_					      =(
       (
           MarkupLMModel,
           MarkupLMForSequenceClassification,
           MarkupLMForTokenClassification,
           MarkupLMForQuestionAnswering,
       )
       if is_torch_available()
       else None
   )
   SCREAMING_SNAKE_CASE_					      =(
       {
           '''feature-extraction''': MarkupLMModel,
           '''question-answering''': MarkupLMForQuestionAnswering,
           '''text-classification''': MarkupLMForSequenceClassification,
           '''token-classification''': MarkupLMForTokenClassification,
           '''zero-shot''': MarkupLMForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
   def 	__a				(						self	:     Union[str, Any]				,       snake_case__	:     Optional[Any]				,       snake_case__	:     Union[str, Any]				,       snake_case__	:     Dict				,       snake_case__	:     List[Any]				,       snake_case__	:     Optional[Any]				):


        '''simple docstring'''
        # ValueError: Nodes must be of type `List[str]` (single pretokenized example), or `List[List[str]]`
        # (batch of pretokenized examples).
        return True
   def 	__a				(						self	:     Tuple				):


        '''simple docstring'''
        UpperCAmelCase__		:  Optional[Any]				=				MarkupLMModelTester(self				)
        UpperCAmelCase__		:  Tuple				=				ConfigTester(self				,       config_class=snake_case__				,       hidden_size=3_7				)
   def 	__a				(						self	:     int				):


        '''simple docstring'''
        self.config_tester.run_common_tests()
   def 	__a				(						self	:     Dict				):


        '''simple docstring'''
        UpperCAmelCase__		:  Dict				=				self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*snake_case__				)
   def 	__a				(						self	:     int				):


        '''simple docstring'''
        UpperCAmelCase__		:  Union[str, Any]				=				self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_sequence_classification(*snake_case__				)
   def 	__a				(						self	:     List[Any]				):


        '''simple docstring'''
        UpperCAmelCase__		:  Optional[int]				=				self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_token_classification(*snake_case__				)




   def 	__a				(						self	:     Any				):


        '''simple docstring'''
        UpperCAmelCase__		:  Tuple				=				self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_question_answering(*snake_case__				)




def      SCREAMING_SNAKE_CASE__    (   )->						Union[str, Any]:






     '''simple docstring'''



     UpperCAmelCase__		:  Dict				=				"\n    <!DOCTYPE html>\n    <html>\n    <head>\n    <title>Page Title</title>\n    </head>\n    <body>\n\n    <h1>This is a Heading</h1>\n    <p>This is a paragraph.</p>\n\n    </body>\n    </html>\n    "

     return html_string




@require_torch
class        lowerCAmelCase__       (  unittest.TestCase					):
   @cached_property
   def 	__a				(						self	:     int				):


        '''simple docstring'''
        # TODO use from_pretrained here
        UpperCAmelCase__		:  List[Any]				=				MarkupLMFeatureExtractor()
        UpperCAmelCase__		:  Dict				=				MarkupLMTokenizer.from_pretrained("microsoft/markuplm-base"				)

        return MarkupLMProcessor(snake_case__				,       snake_case__				)




   @slow
   def 	__a				(						self	:     Dict				):


        '''simple docstring'''
        UpperCAmelCase__		:  Union[str, Any]				=				MarkupLMModel.from_pretrained("microsoft/markuplm-base"				).to(snake_case__				)

        UpperCAmelCase__		:  Dict				=				self.default_processor

        UpperCAmelCase__		:  Any				=				processor(prepare_html_string()				,       return_tensors="pt"				)
        UpperCAmelCase__		:  Optional[int]				=				inputs.to(snake_case__				)

        # forward pass
        with torch.no_grad():
             UpperCAmelCase__		:  List[Any]				=				model(**snake_case__				)

        # verify the last hidden states
        UpperCAmelCase__		:  str				=				torch.Size([1, 1_4, 7_6_8]				)
        self.assertEqual(outputs.last_hidden_state.shape				,       snake_case__				)

        UpperCAmelCase__		:  Union[str, Any]				=				torch.tensor(
            [[0.0675, -0.0052, 0.5001], [-0.2281, 0.0802, 0.2192], [-0.0583, -0.3311, 0.1185]]				).to(snake_case__				)

        self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]				,       snake_case__				,       atol=1e-4				)				)








import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


_lowerCamelCase					:     Optional[Any]					  =  logging.get_logger(__name__)

_lowerCamelCase					:     Dict					  =  {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

_lowerCamelCase					:     List[str]					  =  {
    "vocab_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
    },
    "tokenizer_file": {
        "allenai/longformer-base-4096": (
            "https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json"
        ),
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json"
        ),
    },
}

_lowerCamelCase					:     Dict					  =  {
    "allenai/longformer-base-4096": 4_0_9_6,
    "allenai/longformer-large-4096": 4_0_9_6,
    "allenai/longformer-large-4096-finetuned-triviaqa": 4_0_9_6,
    "allenai/longformer-base-4096-extra.pos.embd.only": 4_0_9_6,
    "allenai/longformer-large-4096-extra.pos.embd.only": 4_0_9_6,
}
class 				__snake_case     (_a						):
				lowerCAmelCase__			=   VOCAB_FILES_NAMES
				lowerCAmelCase__			=   PRETRAINED_VOCAB_FILES_MAP
				lowerCAmelCase__			=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCAmelCase__			=   ["input_ids", "attention_mask"]
				lowerCAmelCase__			=   LongformerTokenizer



				def __init__(					self					:						Optional[Any]						,		_UpperCAmelCase					:						Tuple=None						,		_UpperCAmelCase					:						List[str]=None						,		_UpperCAmelCase					:						List[str]=None						,		_UpperCAmelCase					:						Union[str, Any]="replace"						,		_UpperCAmelCase					:						str="<s>"						,		_UpperCAmelCase					:						str="</s>"						,		_UpperCAmelCase					:						int="</s>"						,		_UpperCAmelCase					:						str="<s>"						,		_UpperCAmelCase					:						Optional[Any]="<unk>"						,		_UpperCAmelCase					:						Optional[Any]="<pad>"						,		_UpperCAmelCase					:						Any="<mask>"						,		_UpperCAmelCase					:						Optional[int]=False						,		_UpperCAmelCase					:						List[str]=True						,		**_UpperCAmelCase					:						Optional[int]						,		)     ->			str:





								'''simple docstring'''




								super().__init__(
								    _UpperCAmelCase						,		_UpperCAmelCase						,		tokenizer_file=_UpperCAmelCase						,		errors=_UpperCAmelCase						,		bos_token=_UpperCAmelCase						,		eos_token=_UpperCAmelCase						,		sep_token=_UpperCAmelCase						,		cls_token=_UpperCAmelCase						,		unk_token=_UpperCAmelCase						,		pad_token=_UpperCAmelCase						,		mask_token=_UpperCAmelCase						,		add_prefix_space=_UpperCAmelCase						,		trim_offsets=_UpperCAmelCase						,		**_UpperCAmelCase						,		)

								_lowerCAmelCase			:			Tuple        =					json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()					)
								if pre_tok_state.get("""add_prefix_space"""						,		_UpperCAmelCase					) != add_prefix_space:
												_lowerCAmelCase			:			str        =					getattr(_UpperCAmelCase						,		pre_tok_state.pop("""type"""					)					)
												_lowerCAmelCase			:			Dict        =					add_prefix_space
												_lowerCAmelCase			:			Union[str, Any]        =					pre_tok_class(**_UpperCAmelCase					)

								_lowerCAmelCase			:			List[Any]        =					add_prefix_space

								_lowerCAmelCase			:			List[str]        =					"""post_processor"""
								_lowerCAmelCase			:			str        =					getattr(self.backend_tokenizer						,		_UpperCAmelCase						,		_UpperCAmelCase					)
								if tokenizer_component_instance:
												_lowerCAmelCase			:			Any        =					json.loads(tokenizer_component_instance.__getstate__()					)

												# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
												if "sep" in state:
																_lowerCAmelCase			:			str        =					tuple(state["""sep"""]					)
												if "cls" in state:
																_lowerCAmelCase			:			int        =					tuple(state["""cls"""]					)

												_lowerCAmelCase			:			Any        =					False

												if state.get("""add_prefix_space"""						,		_UpperCAmelCase					) != add_prefix_space:
																_lowerCAmelCase			:			int        =					add_prefix_space
																_lowerCAmelCase			:			List[str]        =					True

												if state.get("""trim_offsets"""						,		_UpperCAmelCase					) != trim_offsets:
																_lowerCAmelCase			:			List[str]        =					trim_offsets
																_lowerCAmelCase			:			List[str]        =					True

												if changes_to_apply:
																_lowerCAmelCase			:			List[str]        =					getattr(_UpperCAmelCase						,		state.pop("""type"""					)					)
																_lowerCAmelCase			:			Any        =					component_class(**_UpperCAmelCase					)
																setattr(self.backend_tokenizer						,		_UpperCAmelCase						,		_UpperCAmelCase					)



				@property
				def 					SCREAMING_SNAKE_CASE				(					self					:						List[Any]					)     ->			str:





								'''simple docstring'''




								if self._mask_token is None:
												if self.verbose:
																logger.error("""Using mask_token, but it is not set yet."""					)
												return None
								return str(self._mask_token					)



				@mask_token.setter
				def 					SCREAMING_SNAKE_CASE				(					self					:						Any						,		_UpperCAmelCase					:						Any					)     ->			Tuple:





								'''simple docstring'''




								_lowerCAmelCase			:			Tuple        =					AddedToken(_UpperCAmelCase						,		lstrip=_UpperCAmelCase						,		rstrip=_UpperCAmelCase					) if isinstance(_UpperCAmelCase						,		_UpperCAmelCase					) else value
								_lowerCAmelCase			:			List[Any]        =					value



				def 					SCREAMING_SNAKE_CASE				(					self					:						int						,		*_UpperCAmelCase					:						str						,		**_UpperCAmelCase					:						Union[str, Any]					)     ->			BatchEncoding:





								'''simple docstring'''




								_lowerCAmelCase			:			Dict        =					kwargs.get("""is_split_into_words"""						,		_UpperCAmelCase					)
								assert self.add_prefix_space or not is_split_into_words, (
								    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
								    "to use it with pretokenized inputs."
								)

								return super()._batch_encode_plus(*_UpperCAmelCase						,		**_UpperCAmelCase					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						str						,		*_UpperCAmelCase					:						Any						,		**_UpperCAmelCase					:						Optional[Any]					)     ->			BatchEncoding:





								'''simple docstring'''




								_lowerCAmelCase			:			Any        =					kwargs.get("""is_split_into_words"""						,		_UpperCAmelCase					)

								assert self.add_prefix_space or not is_split_into_words, (
								    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
								    "to use it with pretokenized inputs."
								)

								return super()._encode_plus(*_UpperCAmelCase						,		**_UpperCAmelCase					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Dict						,		_UpperCAmelCase					:						str						,		_UpperCAmelCase					:						Optional[str] = None					)     ->			Tuple[str]:





								'''simple docstring'''




								_lowerCAmelCase			:			int        =					self._tokenizer.model.save(_UpperCAmelCase						,		name=_UpperCAmelCase					)
								return tuple(_UpperCAmelCase					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Any						,		_UpperCAmelCase					:						Any						,		_UpperCAmelCase					:						str=None					)     ->			List[Any]:





								'''simple docstring'''




								_lowerCAmelCase			:			Any        =					[self.bos_token_id] + token_ids_a + [self.eos_token_id]
								if token_ids_a is None:
												return output

								return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]



				def 					SCREAMING_SNAKE_CASE				(					self					:						List[Any]						,		_UpperCAmelCase					:						List[int]						,		_UpperCAmelCase					:						Optional[List[int]] = None					)     ->			List[int]:





								'''simple docstring'''




								_lowerCAmelCase			:			Dict        =					[self.sep_token_id]
								_lowerCAmelCase			:			Optional[Any]        =					[self.cls_token_id]

								if token_ids_a is None:
												return len(cls + token_ids_a + sep					) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]

