




from __future__ import annotations

import unittest

from transformers import CTRLConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
	import tensorflow as tf

	from transformers.models.ctrl.modeling_tf_ctrl import (
	    TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
	    TFCTRLForSequenceClassification,
	    TFCTRLLMHeadModel,
	    TFCTRLModel,
	)
class 				__snake_case     (_a						):



				def __init__(					self					:						Union[str, Any]						,		_UpperCAmelCase					:						str						,		)     ->			Dict:





								'''simple docstring'''




								_lowerCAmelCase			:			Any        =					parent
								_lowerCAmelCase			:			str        =					13
								_lowerCAmelCase			:			List[Any]        =					7
								_lowerCAmelCase			:			Any        =					True
								_lowerCAmelCase			:			List[Any]        =					True
								_lowerCAmelCase			:			Union[str, Any]        =					True
								_lowerCAmelCase			:			Dict        =					True
								_lowerCAmelCase			:			List[Any]        =					True
								_lowerCAmelCase			:			Union[str, Any]        =					99
								_lowerCAmelCase			:			List[str]        =					32
								_lowerCAmelCase			:			List[Any]        =					2
								_lowerCAmelCase			:			Any        =					4
								_lowerCAmelCase			:			Optional[Any]        =					37
								_lowerCAmelCase			:			Any        =					"""gelu"""
								_lowerCAmelCase			:			Optional[int]        =					0.1
								_lowerCAmelCase			:			Tuple        =					0.1
								_lowerCAmelCase			:			Dict        =					512
								_lowerCAmelCase			:			Union[str, Any]        =					16
								_lowerCAmelCase			:			int        =					2
								_lowerCAmelCase			:			Dict        =					0.02
								_lowerCAmelCase			:			Optional[int]        =					3
								_lowerCAmelCase			:			Optional[int]        =					4
								_lowerCAmelCase			:			Tuple        =					None
								_lowerCAmelCase			:			List[str]        =					self.vocab_size - 1



				def 					SCREAMING_SNAKE_CASE				(					self					:						Dict					)     ->			Dict:





								'''simple docstring'''




								_lowerCAmelCase			:			Union[str, Any]        =					ids_tensor([self.batch_size, self.seq_length]						,		self.vocab_size					)

								_lowerCAmelCase			:			str        =					None
								if self.use_input_mask:
												_lowerCAmelCase			:			Optional[Any]        =					random_attention_mask([self.batch_size, self.seq_length]					)

								_lowerCAmelCase			:			str        =					None
								if self.use_token_type_ids:
												_lowerCAmelCase			:			Optional[Any]        =					ids_tensor([self.batch_size, self.seq_length]						,		self.type_vocab_size					)

								_lowerCAmelCase			:			Any        =					None
								if self.use_mc_token_ids:
												_lowerCAmelCase			:			int        =					ids_tensor([self.batch_size, self.num_choices]						,		self.seq_length					)

								_lowerCAmelCase			:			List[str]        =					None
								_lowerCAmelCase			:			Optional[Any]        =					None
								_lowerCAmelCase			:			Any        =					None
								if self.use_labels:
												_lowerCAmelCase			:			List[str]        =					ids_tensor([self.batch_size]						,		self.type_sequence_label_size					)
												_lowerCAmelCase			:			Any        =					ids_tensor([self.batch_size, self.seq_length]						,		self.num_labels					)
												_lowerCAmelCase			:			List[Any]        =					ids_tensor([self.batch_size]						,		self.num_choices					)

								_lowerCAmelCase			:			str        =					CTRLConfig(
								    vocab_size=self.vocab_size						,		n_embd=self.hidden_size						,		n_layer=self.num_hidden_layers						,		n_head=self.num_attention_heads						,		n_positions=self.max_position_embeddings						,		pad_token_id=self.pad_token_id						,		)

								_lowerCAmelCase			:			List[str]        =					ids_tensor([self.num_hidden_layers, self.num_attention_heads]						,		2					)

								return (
								    config,
								    input_ids,
								    input_mask,
								    head_mask,
								    token_type_ids,
								    mc_token_ids,
								    sequence_labels,
								    token_labels,
								    choice_labels,
								)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Tuple						,		_UpperCAmelCase					:						List[Any]						,		_UpperCAmelCase					:						Optional[int]						,		_UpperCAmelCase					:						Tuple						,		_UpperCAmelCase					:						Tuple						,		_UpperCAmelCase					:						int						,		*_UpperCAmelCase					:						Optional[int]					)     ->			Dict:





								'''simple docstring'''




								_lowerCAmelCase			:			Dict        =					TFCTRLModel(config=_UpperCAmelCase					)
								_lowerCAmelCase			:			List[Any]        =					{"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
								_lowerCAmelCase			:			List[str]        =					model(_UpperCAmelCase					)

								_lowerCAmelCase			:			Any        =					[input_ids, None, input_mask]  # None is the input for 'past'
								_lowerCAmelCase			:			int        =					model(_UpperCAmelCase					)

								_lowerCAmelCase			:			List[Any]        =					model(_UpperCAmelCase					)

								self.parent.assertEqual(result.last_hidden_state.shape						,		(self.batch_size, self.seq_length, self.hidden_size)					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Any						,		_UpperCAmelCase					:						int						,		_UpperCAmelCase					:						int						,		_UpperCAmelCase					:						str						,		_UpperCAmelCase					:						Dict						,		_UpperCAmelCase					:						Optional[Any]						,		*_UpperCAmelCase					:						Tuple					)     ->			Optional[Any]:





								'''simple docstring'''




								_lowerCAmelCase			:			Tuple        =					TFCTRLLMHeadModel(config=_UpperCAmelCase					)
								_lowerCAmelCase			:			Dict        =					{"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
								_lowerCAmelCase			:			Dict        =					model(_UpperCAmelCase					)
								self.parent.assertEqual(result.logits.shape						,		(self.batch_size, self.seq_length, self.vocab_size)					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Optional[int]						,		_UpperCAmelCase					:						Optional[int]						,		_UpperCAmelCase					:						int						,		_UpperCAmelCase					:						Optional[Any]						,		_UpperCAmelCase					:						Optional[int]						,		_UpperCAmelCase					:						int						,		*_UpperCAmelCase					:						Optional[int]					)     ->			Union[str, Any]:





								'''simple docstring'''




								_lowerCAmelCase			:			List[Any]        =					self.num_labels
								_lowerCAmelCase			:			Tuple        =					ids_tensor([self.batch_size]						,		self.type_sequence_label_size					)
								_lowerCAmelCase			:			Optional[Any]        =					{
								    """input_ids""": input_ids,
								    """token_type_ids""": token_type_ids,
								    """labels""": sequence_labels,
								}
								_lowerCAmelCase			:			str        =					TFCTRLForSequenceClassification(_UpperCAmelCase					)
								_lowerCAmelCase			:			List[Any]        =					model(_UpperCAmelCase					)
								self.parent.assertEqual(result.logits.shape						,		(self.batch_size, self.num_labels)					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						List[str]					)     ->			Optional[int]:





								'''simple docstring'''




								_lowerCAmelCase			:			List[str]        =					self.prepare_config_and_inputs()

								(
								    (
								    _lowerCAmelCase
								)    ,			(
								    _lowerCAmelCase
								)    ,			(
								    _lowerCAmelCase
								)    ,			(
								    _lowerCAmelCase
								)    ,			(
								    _lowerCAmelCase
								)    ,			(
								    _lowerCAmelCase
								)    ,			(
								    _lowerCAmelCase
								)    ,			(
								    _lowerCAmelCase
								)    ,			(
								    _lowerCAmelCase
								)    ,			
								)			:			str        =					config_and_inputs

								_lowerCAmelCase			:			Union[str, Any]        =					{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
								return config, inputs_dict






@require_tf
class 				__snake_case     (_a  ,		_a  ,		unittest.TestCase						):
				lowerCAmelCase__			=   (TFCTRLModel, TFCTRLLMHeadModel, TFCTRLForSequenceClassification) if is_tf_available() else ()
				lowerCAmelCase__			=   (TFCTRLLMHeadModel,) if is_tf_available() else ()
				lowerCAmelCase__			=   (
				    {
				        "feature-extraction": TFCTRLModel,
				        "text-classification": TFCTRLForSequenceClassification,
				        "text-generation": TFCTRLLMHeadModel,
				        "zero-shot": TFCTRLForSequenceClassification,
				    }
				    if is_tf_available()
				    else {}
				)
				lowerCAmelCase__			=   False
				lowerCAmelCase__			=   False



				def 					SCREAMING_SNAKE_CASE				(					self					:						str						,		_UpperCAmelCase					:						Any						,		_UpperCAmelCase					:						Optional[Any]						,		_UpperCAmelCase					:						List[Any]						,		_UpperCAmelCase					:						str						,		_UpperCAmelCase					:						int					)     ->			Optional[Any]:





								'''simple docstring'''




								if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
												# Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
												# `CTRLConfig` was never used in pipeline tests, either because of a missing checkpoint or because a tiny
												# config could not be created.
												return True

								return False



				def 					SCREAMING_SNAKE_CASE				(					self					:						Union[str, Any]					)     ->			Tuple:





								'''simple docstring'''




								_lowerCAmelCase			:			Optional[Any]        =					TFCTRLModelTester(self					)
								_lowerCAmelCase			:			str        =					ConfigTester(self						,		config_class=_UpperCAmelCase						,		n_embd=37					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						str					)     ->			List[Any]:





								'''simple docstring'''




								self.config_tester.run_common_tests()



				def 					SCREAMING_SNAKE_CASE				(					self					:						List[str]					)     ->			int:





								'''simple docstring'''




								_lowerCAmelCase			:			List[Any]        =					self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_ctrl_model(*_UpperCAmelCase					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Tuple					)     ->			int:





								'''simple docstring'''




								_lowerCAmelCase			:			int        =					self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_ctrl_lm_head(*_UpperCAmelCase					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Any					)     ->			List[str]:





								'''simple docstring'''




								_lowerCAmelCase			:			Union[str, Any]        =					self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_ctrl_for_sequence_classification(*_UpperCAmelCase					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						str					)     ->			Any:





								'''simple docstring'''




								_lowerCAmelCase    ,			_lowerCAmelCase			:			List[str]        =					self.model_tester.prepare_config_and_inputs_for_common()
								_lowerCAmelCase			:			str        =					[TFCTRLLMHeadModel]
								_lowerCAmelCase			:			str        =					[TFCTRLForSequenceClassification]

								for model_class in self.all_model_classes:
												_lowerCAmelCase			:			List[str]        =					model_class(_UpperCAmelCase					)
												model.build()  # may be needed for the get_bias() call below
												assert isinstance(model.get_input_embeddings()						,		tf.keras.layers.Layer					)

												if model_class in list_lm_models:
																_lowerCAmelCase			:			Union[str, Any]        =					model.get_output_embeddings()
																assert isinstance(_UpperCAmelCase						,		tf.keras.layers.Layer					)
																_lowerCAmelCase			:			Tuple        =					model.get_bias()
																assert isinstance(_UpperCAmelCase						,		_UpperCAmelCase					)
																for k, v in name.items():
																				assert isinstance(_UpperCAmelCase						,		tf.Variable					)
												elif model_class in list_other_models_with_output_ebd:
																_lowerCAmelCase			:			Tuple        =					model.get_output_embeddings()
																assert isinstance(_UpperCAmelCase						,		tf.keras.layers.Layer					)
																_lowerCAmelCase			:			int        =					model.get_bias()
																assert name is None
												else:
																_lowerCAmelCase			:			Union[str, Any]        =					model.get_output_embeddings()
																assert x is None
																_lowerCAmelCase			:			str        =					model.get_bias()
																assert name is None



				@slow
				def 					SCREAMING_SNAKE_CASE				(					self					:						Dict					)     ->			Optional[int]:





								'''simple docstring'''




								for model_name in TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												_lowerCAmelCase			:			List[str]        =					TFCTRLModel.from_pretrained(_UpperCAmelCase					)
												self.assertIsNotNone(_UpperCAmelCase					)






@require_tf
class 				__snake_case     (unittest.TestCase						):



				@slow
				def 					SCREAMING_SNAKE_CASE				(					self					:						Optional[int]					)     ->			Any:





								'''simple docstring'''




								_lowerCAmelCase			:			List[Any]        =					TFCTRLLMHeadModel.from_pretrained("""ctrl"""					)
								_lowerCAmelCase			:			Union[str, Any]        =					tf.convert_to_tensor([[1_1859, 0, 1611, 8]]						,		dtype=tf.intaa					)  # Legal the president is
								_lowerCAmelCase			:			Any        =					[
								    1_1859,
								    0,
								    1611,
								    8,
								    5,
								    150,
								    2_6449,
								    2,
								    19,
								    348,
								    469,
								    3,
								    2595,
								    48,
								    2_0740,
								    24_6533,
								    24_6533,
								    19,
								    30,
								    5,
								]  # Legal the president is a good guy and I don't want to lose my job. \n \n I have a

								_lowerCAmelCase			:			str        =					model.generate(_UpperCAmelCase						,		do_sample=_UpperCAmelCase					)
								self.assertListEqual(output_ids[0].numpy().tolist()						,		_UpperCAmelCase					)







import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


_lowerCamelCase					:     Optional[Any]					  =  logging.get_logger(__name__)

_lowerCamelCase					:     Dict					  =  {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

_lowerCamelCase					:     List[str]					  =  {
    "vocab_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
    },
    "tokenizer_file": {
        "allenai/longformer-base-4096": (
            "https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json"
        ),
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json"
        ),
    },
}

_lowerCamelCase					:     Dict					  =  {
    "allenai/longformer-base-4096": 4_0_9_6,
    "allenai/longformer-large-4096": 4_0_9_6,
    "allenai/longformer-large-4096-finetuned-triviaqa": 4_0_9_6,
    "allenai/longformer-base-4096-extra.pos.embd.only": 4_0_9_6,
    "allenai/longformer-large-4096-extra.pos.embd.only": 4_0_9_6,
}
class 				__snake_case     (_a						):
				lowerCAmelCase__			=   VOCAB_FILES_NAMES
				lowerCAmelCase__			=   PRETRAINED_VOCAB_FILES_MAP
				lowerCAmelCase__			=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCAmelCase__			=   ["input_ids", "attention_mask"]
				lowerCAmelCase__			=   LongformerTokenizer



				def __init__(					self					:						Optional[Any]						,		_UpperCAmelCase					:						Tuple=None						,		_UpperCAmelCase					:						List[str]=None						,		_UpperCAmelCase					:						List[str]=None						,		_UpperCAmelCase					:						Union[str, Any]="replace"						,		_UpperCAmelCase					:						str="<s>"						,		_UpperCAmelCase					:						str="</s>"						,		_UpperCAmelCase					:						int="</s>"						,		_UpperCAmelCase					:						str="<s>"						,		_UpperCAmelCase					:						Optional[Any]="<unk>"						,		_UpperCAmelCase					:						Optional[Any]="<pad>"						,		_UpperCAmelCase					:						Any="<mask>"						,		_UpperCAmelCase					:						Optional[int]=False						,		_UpperCAmelCase					:						List[str]=True						,		**_UpperCAmelCase					:						Optional[int]						,		)     ->			str:





								'''simple docstring'''




								super().__init__(
								    _UpperCAmelCase						,		_UpperCAmelCase						,		tokenizer_file=_UpperCAmelCase						,		errors=_UpperCAmelCase						,		bos_token=_UpperCAmelCase						,		eos_token=_UpperCAmelCase						,		sep_token=_UpperCAmelCase						,		cls_token=_UpperCAmelCase						,		unk_token=_UpperCAmelCase						,		pad_token=_UpperCAmelCase						,		mask_token=_UpperCAmelCase						,		add_prefix_space=_UpperCAmelCase						,		trim_offsets=_UpperCAmelCase						,		**_UpperCAmelCase						,		)

								_lowerCAmelCase			:			Tuple        =					json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()					)
								if pre_tok_state.get("""add_prefix_space"""						,		_UpperCAmelCase					) != add_prefix_space:
												_lowerCAmelCase			:			str        =					getattr(_UpperCAmelCase						,		pre_tok_state.pop("""type"""					)					)
												_lowerCAmelCase			:			Dict        =					add_prefix_space
												_lowerCAmelCase			:			Union[str, Any]        =					pre_tok_class(**_UpperCAmelCase					)

								_lowerCAmelCase			:			List[Any]        =					add_prefix_space

								_lowerCAmelCase			:			List[str]        =					"""post_processor"""
								_lowerCAmelCase			:			str        =					getattr(self.backend_tokenizer						,		_UpperCAmelCase						,		_UpperCAmelCase					)
								if tokenizer_component_instance:
												_lowerCAmelCase			:			Any        =					json.loads(tokenizer_component_instance.__getstate__()					)

												# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
												if "sep" in state:
																_lowerCAmelCase			:			str        =					tuple(state["""sep"""]					)
												if "cls" in state:
																_lowerCAmelCase			:			int        =					tuple(state["""cls"""]					)

												_lowerCAmelCase			:			Any        =					False

												if state.get("""add_prefix_space"""						,		_UpperCAmelCase					) != add_prefix_space:
																_lowerCAmelCase			:			int        =					add_prefix_space
																_lowerCAmelCase			:			List[str]        =					True

												if state.get("""trim_offsets"""						,		_UpperCAmelCase					) != trim_offsets:
																_lowerCAmelCase			:			List[str]        =					trim_offsets
																_lowerCAmelCase			:			List[str]        =					True

												if changes_to_apply:
																_lowerCAmelCase			:			List[str]        =					getattr(_UpperCAmelCase						,		state.pop("""type"""					)					)
																_lowerCAmelCase			:			Any        =					component_class(**_UpperCAmelCase					)
																setattr(self.backend_tokenizer						,		_UpperCAmelCase						,		_UpperCAmelCase					)



				@property
				def 					SCREAMING_SNAKE_CASE				(					self					:						List[Any]					)     ->			str:





								'''simple docstring'''




								if self._mask_token is None:
												if self.verbose:
																logger.error("""Using mask_token, but it is not set yet."""					)
												return None
								return str(self._mask_token					)



				@mask_token.setter
				def 					SCREAMING_SNAKE_CASE				(					self					:						Any						,		_UpperCAmelCase					:						Any					)     ->			Tuple:





								'''simple docstring'''




								_lowerCAmelCase			:			Tuple        =					AddedToken(_UpperCAmelCase						,		lstrip=_UpperCAmelCase						,		rstrip=_UpperCAmelCase					) if isinstance(_UpperCAmelCase						,		_UpperCAmelCase					) else value
								_lowerCAmelCase			:			List[Any]        =					value



				def 					SCREAMING_SNAKE_CASE				(					self					:						int						,		*_UpperCAmelCase					:						str						,		**_UpperCAmelCase					:						Union[str, Any]					)     ->			BatchEncoding:





								'''simple docstring'''




								_lowerCAmelCase			:			Dict        =					kwargs.get("""is_split_into_words"""						,		_UpperCAmelCase					)
								assert self.add_prefix_space or not is_split_into_words, (
								    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
								    "to use it with pretokenized inputs."
								)

								return super()._batch_encode_plus(*_UpperCAmelCase						,		**_UpperCAmelCase					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						str						,		*_UpperCAmelCase					:						Any						,		**_UpperCAmelCase					:						Optional[Any]					)     ->			BatchEncoding:





								'''simple docstring'''




								_lowerCAmelCase			:			Any        =					kwargs.get("""is_split_into_words"""						,		_UpperCAmelCase					)

								assert self.add_prefix_space or not is_split_into_words, (
								    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
								    "to use it with pretokenized inputs."
								)

								return super()._encode_plus(*_UpperCAmelCase						,		**_UpperCAmelCase					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Dict						,		_UpperCAmelCase					:						str						,		_UpperCAmelCase					:						Optional[str] = None					)     ->			Tuple[str]:





								'''simple docstring'''




								_lowerCAmelCase			:			int        =					self._tokenizer.model.save(_UpperCAmelCase						,		name=_UpperCAmelCase					)
								return tuple(_UpperCAmelCase					)



				def 					SCREAMING_SNAKE_CASE				(					self					:						Any						,		_UpperCAmelCase					:						Any						,		_UpperCAmelCase					:						str=None					)     ->			List[Any]:





								'''simple docstring'''




								_lowerCAmelCase			:			Any        =					[self.bos_token_id] + token_ids_a + [self.eos_token_id]
								if token_ids_a is None:
												return output

								return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]



				def 					SCREAMING_SNAKE_CASE				(					self					:						List[Any]						,		_UpperCAmelCase					:						List[int]						,		_UpperCAmelCase					:						Optional[List[int]] = None					)     ->			List[int]:





								'''simple docstring'''




								_lowerCAmelCase			:			Dict        =					[self.sep_token_id]
								_lowerCAmelCase			:			Optional[Any]        =					[self.cls_token_id]

								if token_ids_a is None:
												return len(cls + token_ids_a + sep					) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]

