




import unittest

from transformers import PegasusConfig, PegasusTokenizer, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor


if is_flax_available():
			import os

			# The slow tests are often failing with OOM error on GPU
			# This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
			# but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
			UpperCAmelCase__				     = "platform"
			import jax
			import jax.numpy as jnp
			import numpy as np

			from transformers import FlaxPegasusForConditionalGeneration, FlaxPegasusModel







@require_flax
class a :
						"""simple docstring"""

						UpperCamelCase_	:       Dict       	=		PegasusConfig
						UpperCamelCase_	:       Union[str, Any]       	=		{}
						UpperCamelCase_	:       int       	=		'gelu'






						def __init__( self :     List[str]				,							lowerCamelCase__ :     Any				,							lowerCamelCase__ :     int=13				,							lowerCamelCase__ :     Tuple=7				,							lowerCamelCase__ :     Union[str, Any]=True				,							lowerCamelCase__ :     int=False				,							lowerCamelCase__ :     Dict=99				,							lowerCamelCase__ :     List[str]=32				,							lowerCamelCase__ :     List[str]=5				,							lowerCamelCase__ :     int=4				,							lowerCamelCase__ :     Union[str, Any]=37				,							lowerCamelCase__ :     Dict=0.1				,							lowerCamelCase__ :     Any=0.1				,							lowerCamelCase__ :     Optional[Any]=20				,							lowerCamelCase__ :     Union[str, Any]=2				,							lowerCamelCase__ :     str=1				,							lowerCamelCase__ :     Optional[int]=0				,							)    ->						List[str]:
													"""simple docstring"""


													__lowercase      =    parent
													__lowercase      =    batch_size
													__lowercase      =    seq_length
													__lowercase      =    is_training
													__lowercase      =    use_labels
													__lowercase      =    vocab_size
													__lowercase      =    hidden_size
													__lowercase      =    num_hidden_layers
													__lowercase      =    num_attention_heads
													__lowercase      =    intermediate_size

													__lowercase      =    hidden_dropout_prob
													__lowercase      =    attention_probs_dropout_prob
													__lowercase      =    max_position_embeddings
													__lowercase      =    eos_token_id
													__lowercase      =    pad_token_id
													__lowercase      =    bos_token_id






						def     UpperCAmelCase_      ( self :     List[str]				)    ->						Tuple:
													"""simple docstring"""


													__lowercase      =    ids_tensor([self.batch_size, self.seq_length - 1]				,							self.vocab_size				).clip(3				,							self.vocab_size				)
													__lowercase      =    np.expand_dims(np.array([self.eos_token_id] * self.batch_size				)				,							1				)
													__lowercase      =    np.concatenate([input_ids, eos_tensor]				,							axis=1				)

													__lowercase      =    ids_tensor([self.batch_size, self.seq_length]				,							self.vocab_size				)

													__lowercase      =    self.config_cls(
													    vocab_size=self.vocab_size				,							d_model=self.hidden_size				,							encoder_layers=self.num_hidden_layers				,							decoder_layers=self.num_hidden_layers				,							encoder_attention_heads=self.num_attention_heads				,							decoder_attention_heads=self.num_attention_heads				,							encoder_ffn_dim=self.intermediate_size				,							decoder_ffn_dim=self.intermediate_size				,							dropout=self.hidden_dropout_prob				,							attention_dropout=self.attention_probs_dropout_prob				,							max_position_embeddings=self.max_position_embeddings				,							eos_token_ids=[2]				,							bos_token_id=self.bos_token_id				,							pad_token_id=self.pad_token_id				,							decoder_start_token_id=self.pad_token_id				,							**self.config_updates				,							)
													__lowercase      =    prepare_pegasus_inputs_dict(lowerCamelCase__				,							lowerCamelCase__				,							lowerCamelCase__				)
													return config, inputs_dict






						def     UpperCAmelCase_      ( self :     Tuple				,							lowerCamelCase__ :     Tuple				,							lowerCamelCase__ :     Union[str, Any]				,							lowerCamelCase__ :     Union[str, Any]				)    ->						Dict:
													"""simple docstring"""


													__lowercase      =    20
													__lowercase      =    model_class_name(lowerCamelCase__				)

													__lowercase      =    model.encode(inputs_dict['''input_ids''']				)

													__lowercase     ,     __lowercase      =    (
													    inputs_dict['''decoder_input_ids'''],
													    inputs_dict['''decoder_attention_mask'''],
													)

													__lowercase      =    model.init_cache(decoder_input_ids.shape[0]				,							lowerCamelCase__				,							lowerCamelCase__				)
													__lowercase      =    jnp.ones((decoder_input_ids.shape[0], max_decoder_length)				,							dtype='''i4'''				)

													__lowercase      =    jnp.broadcast_to(
													    jnp.arange(decoder_input_ids.shape[-1] - 1				)[None, :]				,							(decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1)				,							)
													__lowercase      =    model.decode(
													    decoder_input_ids[:, :-1]				,							lowerCamelCase__				,							decoder_attention_mask=lowerCamelCase__				,							past_key_values=lowerCamelCase__				,							decoder_position_ids=lowerCamelCase__				,							)

													__lowercase      =    jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]]				,							dtype='''i4'''				)
													__lowercase      =    model.decode(
													    decoder_input_ids[:, -1:]				,							lowerCamelCase__				,							decoder_attention_mask=lowerCamelCase__				,							past_key_values=outputs_cache.past_key_values				,							decoder_position_ids=lowerCamelCase__				,							)

													__lowercase      =    model.decode(lowerCamelCase__				,							lowerCamelCase__				)

													__lowercase      =    np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])				)				)
													self.parent.assertTrue(diff < 1e-3				,							msg=f'Max diff is {diff}'				)






						def     UpperCAmelCase_      ( self :     Optional[Any]				,							lowerCamelCase__ :     Any				,							lowerCamelCase__ :     Dict				,							lowerCamelCase__ :     Union[str, Any]				)    ->						str:
													"""simple docstring"""


													__lowercase      =    20
													__lowercase      =    model_class_name(lowerCamelCase__				)

													__lowercase      =    model.encode(inputs_dict['''input_ids''']				)

													__lowercase     ,     __lowercase      =    (
													    inputs_dict['''decoder_input_ids'''],
													    inputs_dict['''decoder_attention_mask'''],
													)

													__lowercase      =    jnp.concatenate(
													    [
													        decoder_attention_mask,
													        jnp.zeros((decoder_attention_mask.shape[0], max_decoder_length - decoder_attention_mask.shape[1])				),
													    ]				,							axis=-1				,							)

													__lowercase      =    model.init_cache(decoder_input_ids.shape[0]				,							lowerCamelCase__				,							lowerCamelCase__				)
													__lowercase      =    jnp.broadcast_to(
													    jnp.arange(decoder_input_ids.shape[-1] - 1				)[None, :]				,							(decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1)				,							)

													__lowercase      =    model.decode(
													    decoder_input_ids[:, :-1]				,							lowerCamelCase__				,							decoder_attention_mask=lowerCamelCase__				,							past_key_values=lowerCamelCase__				,							decoder_position_ids=lowerCamelCase__				,							)
													__lowercase      =    jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]]				,							dtype='''i4'''				)
													__lowercase      =    model.decode(
													    decoder_input_ids[:, -1:]				,							lowerCamelCase__				,							past_key_values=outputs_cache.past_key_values				,							decoder_attention_mask=lowerCamelCase__				,							decoder_position_ids=lowerCamelCase__				,							)

													__lowercase      =    model.decode(lowerCamelCase__				,							lowerCamelCase__				,							decoder_attention_mask=lowerCamelCase__				)

													__lowercase      =    np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])				)				)
													self.parent.assertTrue(diff < 1e-3				,							msg=f'Max diff is {diff}'				)







def 		_A(       UpperCamelCase__							:			str				,       UpperCamelCase__							:			Dict				,       UpperCamelCase__							:			Optional[int]				,       UpperCamelCase__							:			int=None				,       UpperCamelCase__							:			int=None				,       )    -> Dict:
							'''simple docstring'''


							if attention_mask is None:
														__lowercase      =    np.not_equal(UpperCamelCase__				,       config.pad_token_id				).astype(np.inta				)
							if decoder_attention_mask is None:
														__lowercase      =    np.concatenate(
														    [
														        np.ones(decoder_input_ids[:, :1].shape				,       dtype=np.inta				),
														        np.not_equal(decoder_input_ids[:, 1:]				,       config.pad_token_id				).astype(np.inta				),
														    ]				,       axis=-1				,       )
							return {
							    "input_ids": input_ids,
							    "decoder_input_ids": decoder_input_ids,
							    "attention_mask": attention_mask,
							    "decoder_attention_mask": decoder_attention_mask,
							}







@require_flax
class a (     __SCREAMING_SNAKE_CASE       ,						unittest.TestCase  ):
						"""simple docstring"""

						UpperCamelCase_	:       Union[str, Any]       	=		(
						    (
						        FlaxPegasusForConditionalGeneration,
						        FlaxPegasusModel,
						    )
						    if is_flax_available()
						    else ()
						)
						UpperCamelCase_	:       str       	=		(FlaxPegasusForConditionalGeneration,) if is_flax_available() else ()
						UpperCamelCase_	:       Dict       	=		True
						UpperCamelCase_	:       Dict       	=		False
						UpperCamelCase_	:       Optional[int]       	=		False
						UpperCamelCase_	:       str       	=		False






						def     UpperCAmelCase_      ( self :     Tuple				)    ->						Optional[int]:
													"""simple docstring"""


													__lowercase      =    FlaxPegasusModelTester(self				)
													__lowercase      =    ConfigTester(self				,							config_class=lowerCamelCase__				)






						def     UpperCAmelCase_      ( self :     List[Any]				)    ->						Dict:
													"""simple docstring"""


													self.config_tester.run_common_tests()






						def     UpperCAmelCase_      ( self :     Dict				)    ->						str:
													"""simple docstring"""


													__lowercase     ,     __lowercase      =    self.model_tester.prepare_config_and_inputs_for_common()
													for model_class in self.all_model_classes:
																				self.model_tester.check_use_cache_forward(lowerCamelCase__				,							lowerCamelCase__				,							lowerCamelCase__				)






						def     UpperCAmelCase_      ( self :     List[Any]				)    ->						Union[str, Any]:
													"""simple docstring"""


													__lowercase     ,     __lowercase      =    self.model_tester.prepare_config_and_inputs_for_common()
													for model_class in self.all_model_classes:
																				self.model_tester.check_use_cache_forward_with_attn_mask(lowerCamelCase__				,							lowerCamelCase__				,							lowerCamelCase__				)






						def     UpperCAmelCase_      ( self :     List[str]				)    ->						Tuple:
													"""simple docstring"""


													__lowercase     ,     __lowercase      =    self.model_tester.prepare_config_and_inputs_for_common()

													for model_class in self.all_model_classes:
																				with self.subTest(model_class.__name__				):
																											__lowercase      =    self._prepare_for_class(lowerCamelCase__				,							lowerCamelCase__				)
																											__lowercase      =    model_class(lowerCamelCase__				)

																											@jax.jit
																											def encode_jitted(lowerCamelCase__ :     Any				,							lowerCamelCase__ :     str=None				,							**lowerCamelCase__ :     int				):
																																		return model.encode(input_ids=lowerCamelCase__				,							attention_mask=lowerCamelCase__				)

																											with self.subTest('''JIT Enabled'''				):
																																		__lowercase      =    encode_jitted(**lowerCamelCase__				).to_tuple()

																											with self.subTest('''JIT Disabled'''				):
																																		with jax.disable_jit():
																																									__lowercase      =    encode_jitted(**lowerCamelCase__				).to_tuple()

																											self.assertEqual(len(lowerCamelCase__				)				,							len(lowerCamelCase__				)				)
																											for jitted_output, output in zip(lowerCamelCase__				,							lowerCamelCase__				):
																																		self.assertEqual(jitted_output.shape				,							output.shape				)






						def     UpperCAmelCase_      ( self :     Union[str, Any]				)    ->						Optional[int]:
													"""simple docstring"""


													__lowercase     ,     __lowercase      =    self.model_tester.prepare_config_and_inputs_for_common()

													for model_class in self.all_model_classes:
																				with self.subTest(model_class.__name__				):
																											__lowercase      =    model_class(lowerCamelCase__				)
																											__lowercase      =    model.encode(inputs_dict['''input_ids''']				,							inputs_dict['''attention_mask''']				)

																											__lowercase      =    {
																											    '''decoder_input_ids''': inputs_dict['''decoder_input_ids'''],
																											    '''decoder_attention_mask''': inputs_dict['''decoder_attention_mask'''],
																											    '''encoder_outputs''': encoder_outputs,
																											}

																											@jax.jit
																											def decode_jitted(lowerCamelCase__ :     Dict				,							lowerCamelCase__ :     Union[str, Any]				,							lowerCamelCase__ :     int				):
																																		return model.decode(
																																		    decoder_input_ids=lowerCamelCase__				,							decoder_attention_mask=lowerCamelCase__				,							encoder_outputs=lowerCamelCase__				,							)

																											with self.subTest('''JIT Enabled'''				):
																																		__lowercase      =    decode_jitted(**lowerCamelCase__				).to_tuple()

																											with self.subTest('''JIT Disabled'''				):
																																		with jax.disable_jit():
																																									__lowercase      =    decode_jitted(**lowerCamelCase__				).to_tuple()

																											self.assertEqual(len(lowerCamelCase__				)				,							len(lowerCamelCase__				)				)
																											for jitted_output, output in zip(lowerCamelCase__				,							lowerCamelCase__				):
																																		self.assertEqual(jitted_output.shape				,							output.shape				)






						@slow
						def     UpperCAmelCase_      ( self :     Dict				)    ->						str:
													"""simple docstring"""


													for model_class_name in self.all_model_classes:
																				__lowercase      =    model_class_name.from_pretrained('''google/pegasus-large'''				,							from_pt=lowerCamelCase__				)
																				__lowercase      =    np.ones((1, 1)				)
																				__lowercase      =    model(lowerCamelCase__				)
																				self.assertIsNotNone(lowerCamelCase__				)






						@slow
						def     UpperCAmelCase_      ( self :     int				)    ->						int:
													"""simple docstring"""


													__lowercase      =    FlaxPegasusForConditionalGeneration.from_pretrained('''google/pegasus-xsum'''				)
													__lowercase      =    PegasusTokenizer.from_pretrained('''google/pegasus-xsum'''				)

													__lowercase      =    [
													    ''' PG&E stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow.''',
													    ''' The London trio are up for best UK act and best album, as well as getting two nominations in the best song category."We got told like this morning \'Oh I think you\'re nominated\'", said Dappy."And I was like \'Oh yeah, which one?\' And now we\'ve got nominated for four awards. I mean, wow!"Bandmate Fazer added: "We thought it\'s best of us to come down and mingle with everyone and say hello to the cameras. And now we find we\'ve got four nominations."The band have two shots at the best song prize, getting the nod for their Tynchy Stryder collaboration Number One, and single Strong Again.Their album Uncle B will also go up against records by the likes of Beyonce and Kanye West.N-Dubz picked up the best newcomer Mobo in 2007, but female member Tulisa said they wouldn\'t be too disappointed if they didn\'t win this time around."At the end of the day we\'re grateful to be where we are in our careers."If it don\'t happen then it don\'t happen - live to fight another day and keep on making albums and hits for the fans."Dappy also revealed they could be performing live several times on the night.The group will be doing Number One and also a possible rendition of the War Child single, I Got Soul.The charity song is a  re-working of The Killers\' All These Things That I\'ve Done and is set to feature artists like Chipmunk, Ironik and Pixie Lott.This year\'s Mobos will be held outside of London for the first time, in Glasgow on 30 September.N-Dubz said they were looking forward to performing for their Scottish fans and boasted about their recent shows north of the border."We just done Edinburgh the other day," said Dappy."We smashed up an N-Dubz show over there. We done Aberdeen about three or four months ago - we smashed up that show over there! Everywhere we go we smash it up!" ''',
													]

													__lowercase      =    [
													    '''California\'s largest electricity provider has turned off power to hundreds of thousands of customers.''',
													    '''Pop group N-Dubz have revealed they were surprised to get four nominations for this year\'s Mobo Awards.''',
													]

													__lowercase      =    tokenizer(lowerCamelCase__				,							return_tensors='''np'''				,							truncation=lowerCamelCase__				,							max_length=512				,							padding=lowerCamelCase__				)
													__lowercase      =    model.generate(**lowerCamelCase__				,							num_beams=2				).sequences
													__lowercase      =    tokenizer.batch_decode(lowerCamelCase__				,							skip_special_tokens=lowerCamelCase__				)
													assert tgt_text == decoded












import os
import re
import warnings
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer


if TYPE_CHECKING:
			from ...tokenization_utils_base import TextInput
from ...utils import logging


UpperCAmelCase__				     = logging.get_logger(__name__)

UpperCAmelCase__				     = {"vocab_file": "spiece.model"}

UpperCAmelCase__				     = {
    "vocab_file": {
        "t5-small": "https://huggingface.co/t5-small/resolve/main/spiece.model",
        "t5-base": "https://huggingface.co/t5-base/resolve/main/spiece.model",
        "t5-large": "https://huggingface.co/t5-large/resolve/main/spiece.model",
        "t5-3b": "https://huggingface.co/t5-3b/resolve/main/spiece.model",
        "t5-11b": "https://huggingface.co/t5-11b/resolve/main/spiece.model",
    }
}


# TODO(PVP) - this should be removed in Transformers v5
UpperCAmelCase__				     = {
    "t5-small": 512,
    "t5-base": 512,
    "t5-large": 512,
    "t5-3b": 512,
    "t5-11b": 512,
}

UpperCAmelCase__				     = "▁"







class a (     __SCREAMING_SNAKE_CASE  ):
						"""simple docstring"""

						UpperCamelCase_	:       List[str]       	=		VOCAB_FILES_NAMES
						UpperCamelCase_	:       Any       	=		PRETRAINED_VOCAB_FILES_MAP
						UpperCamelCase_	:       int       	=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCamelCase_	:       Optional[int]       	=		['input_ids', 'attention_mask']






						def __init__( self :     Optional[Any]				,							lowerCamelCase__ :     int				,							lowerCamelCase__ :     Optional[int]="</s>"				,							lowerCamelCase__ :     Optional[int]="<unk>"				,							lowerCamelCase__ :     Dict="<pad>"				,							lowerCamelCase__ :     Optional[int]=100				,							lowerCamelCase__ :     int=None				,							lowerCamelCase__ :     Optional[Dict[str, Any]] = None				,							lowerCamelCase__ :     int=True				,							**lowerCamelCase__ :     Optional[int]				,							)    ->						None:
													"""simple docstring"""


													if extra_ids > 0 and additional_special_tokens is None:
																				__lowercase      =    [f'<extra_id_{i}>' for i in range(lowerCamelCase__				)]
													elif extra_ids > 0 and additional_special_tokens is not None:
																				# Check that we have the right number of extra_id special tokens
																				__lowercase      =    len(set(filter(lambda lowerCamelCase__				: bool('''extra_id''' in str(lowerCamelCase__				)				)				,							lowerCamelCase__				)				)				)
																				if extra_tokens != extra_ids:
																											raise ValueError(
																											    f'Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'
																											    ''' provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids'''
																											    ''' tokens'''				)
													if legacy:
																				logger.warning_once(
																				    f'You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to'
																				    ''' read the related pull request available at https://github.com/huggingface/transformers/pull/24565'''				)

													__lowercase      =    legacy
													__lowercase      =    {} if sp_model_kwargs is None else sp_model_kwargs

													super().__init__(
													    eos_token=lowerCamelCase__				,							unk_token=lowerCamelCase__				,							pad_token=lowerCamelCase__				,							extra_ids=lowerCamelCase__				,							additional_special_tokens=lowerCamelCase__				,							sp_model_kwargs=self.sp_model_kwargs				,							legacy=lowerCamelCase__				,							**lowerCamelCase__				,							)

													__lowercase      =    vocab_file
													__lowercase      =    extra_ids

													__lowercase      =    spm.SentencePieceProcessor(**self.sp_model_kwargs				)
													self.sp_model.Load(lowerCamelCase__				)






						@staticmethod
						def     UpperCAmelCase_      ( lowerCamelCase__ :     int				,							lowerCamelCase__ :     Any				,							lowerCamelCase__ :     List[str]				)    ->						Optional[int]:
													"""simple docstring"""


													if pretrained_model_name_or_path in TaTokenizer.max_model_input_sizes:
																				__lowercase      =    TaTokenizer.max_model_input_sizes[pretrained_model_name_or_path]
																				if init_max_model_length is not None and init_max_model_length != max_model_length:
																											return init_max_model_length
																				elif init_max_model_length is None:
																											warnings.warn(
																											    '''This tokenizer was incorrectly instantiated with a model max length of'''
																											    f' {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this'
																											    ''' behavior is kept to avoid breaking backwards compatibility when padding/encoding with'''
																											    ''' `truncation is True`.\n- Be aware that you SHOULD NOT rely on'''
																											    f' {pretrained_model_name_or_path} automatically truncating your input to'
																											    f' {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences'
																											    f' longer than {deprecated_max_model_length} you can either instantiate this tokenizer with'
																											    ''' `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please'''
																											    ''' instantiate this tokenizer with `model_max_length` set to your preferred value.'''				,							lowerCamelCase__				,							)

													return max_model_length






						@property
						def     UpperCAmelCase_      ( self :     Union[str, Any]				)    ->						int:
													"""simple docstring"""


													return self.sp_model.get_piece_size() + self._extra_ids






						def     UpperCAmelCase_      ( self :     int				)    ->						Dict:
													"""simple docstring"""


													__lowercase      =    {self.convert_ids_to_tokens(lowerCamelCase__				): i for i in range(self.vocab_size				)}
													vocab.update(self.added_tokens_encoder				)
													return vocab






						def     UpperCAmelCase_      ( self :     int				,							lowerCamelCase__ :     List[int]				,							lowerCamelCase__ :     Optional[List[int]] = None				,							lowerCamelCase__ :     bool = False				)    ->						List[int]:
													"""simple docstring"""


													if already_has_special_tokens:
																				return super().get_special_tokens_mask(
																				    token_ids_a=lowerCamelCase__				,							token_ids_a=lowerCamelCase__				,							already_has_special_tokens=lowerCamelCase__				)

													# normal case: some special tokens
													if token_ids_a is None:
																				return ([0] * len(lowerCamelCase__				)) + [1]
													return ([0] * len(lowerCamelCase__				)) + [1] + ([0] * len(lowerCamelCase__				)) + [1]






						def     UpperCAmelCase_      ( self :     Any				)    ->						int:
													"""simple docstring"""


													return list(
													    set(filter(lambda lowerCamelCase__				: bool(re.search(R'''<extra_id_\d+>'''				,							lowerCamelCase__				)				) is not None				,							self.additional_special_tokens				)				)				)






						def     UpperCAmelCase_      ( self :     Dict				)    ->						Optional[int]:
													"""simple docstring"""


													return [self._convert_token_to_id(lowerCamelCase__				) for token in self.get_sentinel_tokens()]






						def     UpperCAmelCase_      ( self :     int				,							lowerCamelCase__ :     List[int]				)    ->						List[int]:
													"""simple docstring"""


													if len(lowerCamelCase__				) > 0 and token_ids[-1] == self.eos_token_id:
																				warnings.warn(
																				    f'This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated'
																				    ''' eos tokens being added.'''				)
																				return token_ids
													else:
																				return token_ids + [self.eos_token_id]






						def     UpperCAmelCase_      ( self :     Tuple				,							lowerCamelCase__ :     List[int]				,							lowerCamelCase__ :     Optional[List[int]] = None				)    ->						List[int]:
													"""simple docstring"""


													__lowercase      =    [self.eos_token_id]

													if token_ids_a is None:
																				return len(token_ids_a + eos				) * [0]
													return len(token_ids_a + eos + token_ids_a + eos				) * [0]






						def     UpperCAmelCase_      ( self :     Tuple				,							lowerCamelCase__ :     List[int]				,							lowerCamelCase__ :     Optional[List[int]] = None				)    ->						List[int]:
													"""simple docstring"""


													__lowercase      =    self._add_eos_if_not_present(lowerCamelCase__				)
													if token_ids_a is None:
																				return token_ids_a
													else:
																				__lowercase      =    self._add_eos_if_not_present(lowerCamelCase__				)
																				return token_ids_a + token_ids_a






						def __getstate__( self :     List[str]				)    ->						Optional[Any]:
													"""simple docstring"""


													__lowercase      =    self.__dict__.copy()
													__lowercase      =    None
													return state






						def __setstate__( self :     Union[str, Any]				,							lowerCamelCase__ :     Optional[Any]				)    ->						int:
													"""simple docstring"""


													__lowercase      =    d

													# for backward compatibility
													if not hasattr(self				,							'''sp_model_kwargs'''				):
																				__lowercase      =    {}

													__lowercase      =    spm.SentencePieceProcessor(**self.sp_model_kwargs				)
													self.sp_model.Load(self.vocab_file				)






						def     UpperCAmelCase_      ( self :     Union[str, Any]				,							lowerCamelCase__ :     "TextInput"				,							**lowerCamelCase__ :     Optional[Any]				)    ->						List[str]:
													"""simple docstring"""


													if not self.legacy:
																				__lowercase      =    SPIECE_UNDERLINE + text.replace(lowerCamelCase__				,							''' '''				)
													return super().tokenize(lowerCamelCase__				,							**lowerCamelCase__				)






						def     UpperCAmelCase_      ( self :     Dict				,							lowerCamelCase__ :     Tuple				,							**lowerCamelCase__ :     Tuple				)    ->						Dict:
													"""simple docstring"""


													if not self.legacy:
																				__lowercase      =    text.startswith(lowerCamelCase__				)
																				if is_first:
																											__lowercase      =    text[1:]

													__lowercase      =    self.sp_model.encode(lowerCamelCase__				,							out_type=lowerCamelCase__				)

													if not self.legacy and not is_first and not text.startswith(''' '''				) and tokens[0].startswith(lowerCamelCase__				):
																				__lowercase      =    ([tokens[0][1:]] if len(tokens[0]				) > 1 else []) + tokens[1:]
													return tokens






						def     UpperCAmelCase_      ( self :     Optional[Any]				,							lowerCamelCase__ :     Tuple				)    ->						str:
													"""simple docstring"""


													if token.startswith('''<extra_id_'''				):
																				__lowercase      =    re.match(R'''<extra_id_(\d+)>'''				,							lowerCamelCase__				)
																				__lowercase      =    int(match.group(1				)				)
																				return self.vocab_size - num - 1
													return self.sp_model.piece_to_id(lowerCamelCase__				)






						def     UpperCAmelCase_      ( self :     Dict				,							lowerCamelCase__ :     Optional[int]				)    ->						Any:
													"""simple docstring"""


													if index < self.sp_model.get_piece_size():
																				__lowercase      =    self.sp_model.IdToPiece(lowerCamelCase__				)
													else:
																				__lowercase      =    f'<extra_id_{self.vocab_size - 1 - index}>'
													return token






						def     UpperCAmelCase_      ( self :     Tuple				,							lowerCamelCase__ :     List[Any]				)    ->						Tuple:
													"""simple docstring"""


													__lowercase      =    []
													__lowercase      =    ''''''
													__lowercase      =    False
													for token in tokens:
																				# make sure that special tokens are not decoded using sentencepiece model
																				if token in self.all_special_tokens:
																											if not prev_is_special:
																																		out_string += " "
																											out_string += self.sp_model.decode(lowerCamelCase__				) + token
																											__lowercase      =    True
																											__lowercase      =    []
																				else:
																											current_sub_tokens.append(lowerCamelCase__				)
																											__lowercase      =    False
													out_string += self.sp_model.decode(lowerCamelCase__				)
													return out_string.strip()






						def     UpperCAmelCase_      ( self :     Tuple				,							lowerCamelCase__ :     str				,							lowerCamelCase__ :     Optional[str] = None				)    ->						Tuple[str]:
													"""simple docstring"""


													if not os.path.isdir(lowerCamelCase__				):
																				logger.error(f'Vocabulary path ({save_directory}) should be a directory'				)
																				return
													__lowercase      =    os.path.join(
													    lowerCamelCase__				,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

													if os.path.abspath(self.vocab_file				) != os.path.abspath(lowerCamelCase__				) and os.path.isfile(self.vocab_file				):
																				copyfile(self.vocab_file				,							lowerCamelCase__				)
													elif not os.path.isfile(self.vocab_file				):
																				with open(lowerCamelCase__				,							'''wb'''				) as fi:
																											__lowercase      =    self.sp_model.serialized_model_proto()
																											fi.write(lowerCamelCase__				)

													return (out_vocab_file,)






