





import inspect
import logging
import os
import random
import shutil
import tempfile
import unittest

import pytest
import torch
from torch import nn
from torch.utils.data import DataLoader, TensorDataset

from accelerate import Accelerator
from accelerate.test_utils import execute_subprocess_async, require_cuda
from accelerate.utils import ProjectConfiguration, set_seed


UpperCamelCase__											= logging.getLogger(__name__)





def     _UpperCamelCase     (a__   :Union[str, Any]=2   ,			a__   :Tuple=3   ,			a__   :Dict=16   ,			a__   :Dict = 10   ,			a__   :str = 2       ):






							"""simple docstring"""





							def get_dataset(a__   :str       ):
														UpperCamelCase__				     =			torch.randn(batch_size * n_batches   ,			1       )
														return TensorDataset(_snake_case   ,			a * x + b + 0.1 * torch.randn(batch_size * n_batches   ,			1       )       )

							UpperCamelCase__				     =			get_dataset(_snake_case       )
							UpperCamelCase__				     =			get_dataset(_snake_case       )
							UpperCamelCase__				     =			DataLoader(_snake_case   ,			shuffle=_snake_case   ,			batch_size=_snake_case   ,			num_workers=4       )
							UpperCamelCase__				     =			DataLoader(_snake_case   ,			shuffle=_snake_case   ,			batch_size=_snake_case   ,			num_workers=4       )
							return (train_dataloader, valid_dataloader)





def     _UpperCamelCase     (a__   :Dict   ,			a__   :Union[str, Any]   ,			a__   :Tuple   ,			a__   :Tuple   ,			a__   :Optional[int]   ,			a__   :Tuple=None       ):






							"""simple docstring"""





							UpperCamelCase__				     =			[]
							for epoch in range(_snake_case       ):
														# Train quickly
														model.train()
														for batch in dataloader:
																					UpperCamelCase__     ,     UpperCamelCase__				     =			batch
																					UpperCamelCase__				     =			model(_snake_case       )
																					UpperCamelCase__				     =			torch.nn.functional.mse_loss(_snake_case   ,			_snake_case       )
																					accelerator.backward(_snake_case       )
																					optimizer.step()
																					optimizer.zero_grad()
														rands.append(random.random()       )  # Introduce some randomness
														if scheduler is not None:
																					scheduler.step()
							return rands

class        __SCREAMING_SNAKE_CASE					(     nn.Module       ):






							def __init__(						self  ):
														super().__init__()
														UpperCamelCase__				     =			nn.Parameter(torch.randn(1  )  )
														UpperCamelCase__				     =			nn.Parameter(torch.randn(1  )  )







							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														return x * self.a + self.b






class        __SCREAMING_SNAKE_CASE					(     unittest.TestCase       ):






							def 				_lowerCamelCase		(						self  ):
														with tempfile.TemporaryDirectory() as tmpdir:
																					set_seed(42  )
																					UpperCamelCase__				     =			DummyModel()
																					UpperCamelCase__				     =			torch.optim.Adam(params=model.parameters()  , lr=1E-3  )
																					UpperCamelCase__     ,     UpperCamelCase__				     =			dummy_dataloaders()
																					UpperCamelCase__				     =			ProjectConfiguration(total_limit=1  , project_dir=__lowerCAmelCase  , automatic_checkpoint_naming=__lowerCAmelCase  )
																					# Train baseline
																					UpperCamelCase__				     =			Accelerator(project_config=__lowerCAmelCase  )
																					UpperCamelCase__     ,     UpperCamelCase__     ,     UpperCamelCase__     ,     UpperCamelCase__				     =			accelerator.prepare(
																					    __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  )
																					# Save initial
																					accelerator.save_state()

																					# Save second state
																					accelerator.save_state()
																					self.assertEqual(len(os.listdir(accelerator.project_dir  )  )  , 1  )






							def 				_lowerCamelCase		(						self  ):
														with tempfile.TemporaryDirectory() as tmpdir:
																					set_seed(42  )
																					UpperCamelCase__				     =			DummyModel()
																					UpperCamelCase__				     =			torch.optim.Adam(params=model.parameters()  , lr=1E-3  )
																					UpperCamelCase__     ,     UpperCamelCase__				     =			dummy_dataloaders()
																					# Train baseline
																					UpperCamelCase__				     =			Accelerator()
																					UpperCamelCase__     ,     UpperCamelCase__     ,     UpperCamelCase__     ,     UpperCamelCase__				     =			accelerator.prepare(
																					    __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  )
																					# Save initial
																					UpperCamelCase__				     =			os.path.join(__lowerCAmelCase  , """initial"""  )
																					accelerator.save_state(__lowerCAmelCase  )
																					((UpperCamelCase__)     ,     (UpperCamelCase__))				     =			model.a.item(), model.b.item()
																					UpperCamelCase__				     =			optimizer.state_dict()
																					UpperCamelCase__				     =			train(3  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  )
																					((UpperCamelCase__)     ,     (UpperCamelCase__))				     =			model.a.item(), model.b.item()
																					UpperCamelCase__				     =			optimizer.state_dict()

																					# Train partially
																					set_seed(42  )
																					UpperCamelCase__				     =			DummyModel()
																					UpperCamelCase__				     =			torch.optim.Adam(params=model.parameters()  , lr=1E-3  )
																					UpperCamelCase__     ,     UpperCamelCase__				     =			dummy_dataloaders()
																					UpperCamelCase__				     =			Accelerator()
																					UpperCamelCase__     ,     UpperCamelCase__     ,     UpperCamelCase__     ,     UpperCamelCase__				     =			accelerator.prepare(
																					    __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  )
																					accelerator.load_state(__lowerCAmelCase  )
																					((UpperCamelCase__)     ,     (UpperCamelCase__))				     =			model.a.item(), model.b.item()
																					UpperCamelCase__				     =			optimizer.state_dict()
																					self.assertEqual(__lowerCAmelCase  , __lowerCAmelCase  )
																					self.assertEqual(__lowerCAmelCase  , __lowerCAmelCase  )
																					self.assertEqual(__lowerCAmelCase  , __lowerCAmelCase  )

																					UpperCamelCase__				     =			train(2  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  )
																					# Save everything
																					UpperCamelCase__				     =			os.path.join(__lowerCAmelCase  , """checkpoint"""  )
																					accelerator.save_state(__lowerCAmelCase  )

																					# Load everything back in and make sure all states work
																					accelerator.load_state(__lowerCAmelCase  )
																					test_rands += train(1  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  )
																					((UpperCamelCase__)     ,     (UpperCamelCase__))				     =			model.a.item(), model.b.item()
																					UpperCamelCase__				     =			optimizer.state_dict()
																					self.assertEqual(__lowerCAmelCase  , __lowerCAmelCase  )
																					self.assertEqual(__lowerCAmelCase  , __lowerCAmelCase  )
																					self.assertEqual(__lowerCAmelCase  , __lowerCAmelCase  )
																					self.assertEqual(__lowerCAmelCase  , __lowerCAmelCase  )






							def 				_lowerCamelCase		(						self  ):
														with tempfile.TemporaryDirectory() as tmpdir:
																					set_seed(42  )
																					UpperCamelCase__				     =			DummyModel()
																					UpperCamelCase__				     =			torch.optim.Adam(params=model.parameters()  , lr=1E-3  )
																					UpperCamelCase__     ,     UpperCamelCase__				     =			dummy_dataloaders()
																					UpperCamelCase__				     =			ProjectConfiguration(automatic_checkpoint_naming=__lowerCAmelCase  )

																					# Train baseline
																					UpperCamelCase__				     =			Accelerator(project_dir=__lowerCAmelCase  , project_config=__lowerCAmelCase  )
																					UpperCamelCase__     ,     UpperCamelCase__     ,     UpperCamelCase__     ,     UpperCamelCase__				     =			accelerator.prepare(
																					    __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  )
																					# Save initial
																					accelerator.save_state()
																					((UpperCamelCase__)     ,     (UpperCamelCase__))				     =			model.a.item(), model.b.item()
																					UpperCamelCase__				     =			optimizer.state_dict()
																					UpperCamelCase__				     =			train(3  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  )
																					((UpperCamelCase__)     ,     (UpperCamelCase__))				     =			model.a.item(), model.b.item()
																					UpperCamelCase__				     =			optimizer.state_dict()

																					# Train partially
																					set_seed(42  )
																					UpperCamelCase__				     =			DummyModel()
																					UpperCamelCase__				     =			torch.optim.Adam(params=model.parameters()  , lr=1E-3  )
																					UpperCamelCase__     ,     UpperCamelCase__				     =			dummy_dataloaders()
																					UpperCamelCase__				     =			ProjectConfiguration(iteration=1  , automatic_checkpoint_naming=__lowerCAmelCase  )
																					UpperCamelCase__				     =			Accelerator(project_dir=__lowerCAmelCase  , project_config=__lowerCAmelCase  )
																					UpperCamelCase__     ,     UpperCamelCase__     ,     UpperCamelCase__     ,     UpperCamelCase__				     =			accelerator.prepare(
																					    __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  )
																					accelerator.load_state(os.path.join(__lowerCAmelCase  , """checkpoints"""  , """checkpoint_0"""  )  )
																					((UpperCamelCase__)     ,     (UpperCamelCase__))				     =			model.a.item(), model.b.item()
																					UpperCamelCase__				     =			optimizer.state_dict()
																					self.assertEqual(__lowerCAmelCase  , __lowerCAmelCase  )
																					self.assertEqual(__lowerCAmelCase  , __lowerCAmelCase  )
																					self.assertEqual(__lowerCAmelCase  , __lowerCAmelCase  )

																					UpperCamelCase__				     =			train(2  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  )
																					# Save everything
																					accelerator.save_state()

																					# Load everything back in and make sure all states work
																					accelerator.load_state(os.path.join(__lowerCAmelCase  , """checkpoints"""  , """checkpoint_1"""  )  )
																					test_rands += train(1  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  )
																					((UpperCamelCase__)     ,     (UpperCamelCase__))				     =			model.a.item(), model.b.item()
																					UpperCamelCase__				     =			optimizer.state_dict()
																					self.assertEqual(__lowerCAmelCase  , __lowerCAmelCase  )
																					self.assertEqual(__lowerCAmelCase  , __lowerCAmelCase  )
																					self.assertEqual(__lowerCAmelCase  , __lowerCAmelCase  )
																					self.assertEqual(__lowerCAmelCase  , __lowerCAmelCase  )






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			torch.tensor([1, 2, 3]  )
														UpperCamelCase__				     =			torch.tensor([2, 3, 4]  )
														UpperCamelCase__				     =			DummyModel()
														UpperCamelCase__				     =			torch.optim.Adam(net.parameters()  )
														UpperCamelCase__				     =			Accelerator()
														with self.assertRaises(__lowerCAmelCase  ) as ve:
																					accelerator.register_for_checkpointing(__lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  )
														UpperCamelCase__				     =			str(ve.exception  )
														self.assertTrue("""Item at index 0""" in message  )
														self.assertTrue("""Item at index 1""" in message  )
														self.assertFalse("""Item at index 2""" in message  )
														self.assertFalse("""Item at index 3""" in message  )






							def 				_lowerCamelCase		(						self  ):
														with tempfile.TemporaryDirectory() as tmpdir:
																					set_seed(42  )
																					UpperCamelCase__				     =			DummyModel()
																					UpperCamelCase__				     =			torch.optim.Adam(params=model.parameters()  , lr=1E-3  )
																					UpperCamelCase__				     =			torch.optim.lr_scheduler.StepLR(__lowerCAmelCase  , step_size=1  , gamma=0.99  )
																					UpperCamelCase__     ,     UpperCamelCase__				     =			dummy_dataloaders()
																					UpperCamelCase__				     =			ProjectConfiguration(automatic_checkpoint_naming=__lowerCAmelCase  )
																					# Train baseline
																					UpperCamelCase__				     =			Accelerator(project_dir=__lowerCAmelCase  , project_config=__lowerCAmelCase  )
																					UpperCamelCase__     ,     UpperCamelCase__     ,     UpperCamelCase__     ,     UpperCamelCase__     ,     UpperCamelCase__				     =			accelerator.prepare(
																					    __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  )
																					# Save initial
																					accelerator.save_state()
																					UpperCamelCase__				     =			scheduler.state_dict()
																					train(3  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  )
																					self.assertNotEqual(__lowerCAmelCase  , scheduler.state_dict()  )

																					# Load everything back in and make sure all states work
																					accelerator.load_state(os.path.join(__lowerCAmelCase  , """checkpoints"""  , """checkpoint_0"""  )  )
																					self.assertEqual(__lowerCAmelCase  , scheduler.state_dict()  )






							def 				_lowerCamelCase		(						self  ):
														with tempfile.TemporaryDirectory() as tmpdir:
																					set_seed(42  )
																					UpperCamelCase__				     =			DummyModel()
																					UpperCamelCase__				     =			ProjectConfiguration(automatic_checkpoint_naming=__lowerCAmelCase  , total_limit=2  )
																					# Train baseline
																					UpperCamelCase__				     =			Accelerator(project_dir=__lowerCAmelCase  , project_config=__lowerCAmelCase  )
																					UpperCamelCase__				     =			accelerator.prepare(__lowerCAmelCase  )
																					# Save 3 states:
																					for _ in range(11  ):
																												accelerator.save_state()
																					self.assertTrue(not os.path.exists(os.path.join(__lowerCAmelCase  , """checkpoints"""  , """checkpoint_0"""  )  )  )
																					self.assertTrue(os.path.exists(os.path.join(__lowerCAmelCase  , """checkpoints"""  , """checkpoint_9"""  )  )  )
																					self.assertTrue(os.path.exists(os.path.join(__lowerCAmelCase  , """checkpoints"""  , """checkpoint_10"""  )  )  )







							@require_cuda
							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			["""torchrun""", f"""--nproc_per_node={torch.cuda.device_count()}""", inspect.getfile(self.__class__  )]
														execute_subprocess_async(__lowerCAmelCase  , env=os.environ.copy()  )


if __name__ == "__main__":
					UpperCamelCase__											= "/tmp/accelerate/state_checkpointing"
					UpperCamelCase__											= DummyModel()
					UpperCamelCase__											= torch.optim.Adam(params=model.parameters(), lr=1e-3)
					UpperCamelCase__											= torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.99)
					UpperCamelCase__					, UpperCamelCase__											= dummy_dataloaders()
					UpperCamelCase__											= ProjectConfiguration(automatic_checkpoint_naming=True)
					# Train baseline
					UpperCamelCase__											= Accelerator(project_dir=savedir, project_config=project_config, mixed_precision="no")
					if accelerator.process_index == 0:
										if os.path.exists(savedir):
															shutil.rmtree(savedir)
										os.makedirs(savedir)
					UpperCamelCase__					, UpperCamelCase__					, UpperCamelCase__					, UpperCamelCase__					, UpperCamelCase__											= accelerator.prepare(
					    model, optimizer, train_dataloader, valid_dataloader, scheduler
					)
					UpperCamelCase__					, UpperCamelCase__											= accelerator.prepare(model, optimizer)
					train(3, model, train_dataloader, optimizer, accelerator, scheduler)
					# Check that the intial optimizer is loaded on the GPU
					for group in optimizer.param_groups:
										UpperCamelCase__											= group["params"][0].device
										break
					assert param_device.type == accelerator.device.type
					UpperCamelCase__											= model.cpu()
					accelerator.wait_for_everyone()
					accelerator.save_state()
					accelerator.wait_for_everyone()

					# Check CPU state
					accelerator.load_state(os.path.join(savedir, "checkpoints", "checkpoint_0"), map_location="cpu")
					for group in optimizer.param_groups:
										UpperCamelCase__											= group["params"][0].device
										break
					assert (
					    param_device.type == torch.device("cpu").type
					), f"Loaded optimizer states did not match, expected to be loaded on the CPU but got {param_device}"

					# Check device state
					model.to(accelerator.device)
					accelerator.load_state(os.path.join(savedir, "checkpoints", "checkpoint_0"), map_location="on_device")
					for group in optimizer.param_groups:
										UpperCamelCase__											= group["params"][0].device
										break
					assert (
					    param_device.type == accelerator.device.type
					), f"Loaded optimizer states did not match, expected to be loaded on {accelerator.device} but got {param_device}"

					# Check error
					with pytest.raises(TypeError, match="Unsupported optimizer map location passed"):
										accelerator.load_state(os.path.join(savedir, "checkpoints", "checkpoint_0"), map_location="invalid")
					accelerator.wait_for_everyone()
					if accelerator.process_index == 0:
										shutil.rmtree(savedir)
					accelerator.wait_for_everyone()











import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__											= "▁"

UpperCamelCase__											= {"vocab_file": "spiece.model"}

UpperCamelCase__											= {
    "vocab_file": {"google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"}
}

UpperCamelCase__											= {
    "google/pegasus-xsum": 512,
}


UpperCamelCase__											= logging.get_logger(__name__)

class        __SCREAMING_SNAKE_CASE					(     _a       ):
							snake_case     :      Union[str, Any]						     =       VOCAB_FILES_NAMES

							snake_case     :      str						     =       VOCAB_FILES_NAMES
							snake_case     :      Any						     =       PRETRAINED_VOCAB_FILES_MAP
							snake_case     :      int						     =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							snake_case     :      Optional[int]						     =       ["""input_ids""", """attention_mask"""]






							def __init__(						self  , __lowerCAmelCase  , __lowerCAmelCase="<pad>"  , __lowerCAmelCase="</s>"  , __lowerCAmelCase="<unk>"  , __lowerCAmelCase="<mask_2>"  , __lowerCAmelCase="<mask_1>"  , __lowerCAmelCase=None  , __lowerCAmelCase=103  , __lowerCAmelCase = None  , **__lowerCAmelCase  , ):
														UpperCamelCase__				     =			offset
														if additional_special_tokens is not None:
																					if not isinstance(__lowerCAmelCase  , __lowerCAmelCase  ):
																												raise TypeError(
																												    f"""additional_special_tokens should be of type {type(__lowerCAmelCase  )}, but is"""
																												    f""" {type(__lowerCAmelCase  )}"""  )

																					UpperCamelCase__				     =			(
																					    ([mask_token_sent] + additional_special_tokens)
																					    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
																					    else additional_special_tokens
																					)
																					# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
																					additional_special_tokens_extended += [
																					    f"""<unk_{i}>""" for i in range(len(__lowerCAmelCase  )  , self.offset - 1  )
																					]

																					if len(set(__lowerCAmelCase  )  ) != len(__lowerCAmelCase  ):
																												raise ValueError(
																												    """Please make sure that the provided additional_special_tokens do not contain an incorrectly"""
																												    f""" shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}."""  )
																					UpperCamelCase__				     =			additional_special_tokens_extended
														else:
																					UpperCamelCase__				     =			[mask_token_sent] if mask_token_sent is not None else []
																					additional_special_tokens += [f"""<unk_{i}>""" for i in range(2  , self.offset  )]

														UpperCamelCase__				     =			{} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    eos_token=__lowerCAmelCase  , unk_token=__lowerCAmelCase  , mask_token=__lowerCAmelCase  , pad_token=__lowerCAmelCase  , mask_token_sent=__lowerCAmelCase  , offset=__lowerCAmelCase  , additional_special_tokens=__lowerCAmelCase  , sp_model_kwargs=self.sp_model_kwargs  , **__lowerCAmelCase  , )
														UpperCamelCase__				     =			mask_token_sent
														UpperCamelCase__				     =			vocab_file
														UpperCamelCase__				     =			spm.SentencePieceProcessor(**self.sp_model_kwargs  )
														self.sp_model.Load(__lowerCAmelCase  )

														# add special tokens to encoder dict
														UpperCamelCase__				     =			{
														    0: self.pad_token,
														    1: self.eos_token,
														}

														if self.mask_token_sent is not None:
																					self.encoder.update(
																					    {
																					        2: self.mask_token_sent,
																					        3: self.mask_token,
																					    }  )

														if self.offset > 0:
																					# entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
																					# mask_token_sent is already added to list -> so start at 1
																					self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1  , self.offset - 1  )}  )

														UpperCamelCase__				     =			{v: k for k, v in self.encoder.items()}






							@property
							def 				_lowerCamelCase		(						self  ):
														return len(self.sp_model  ) + self.offset






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			{self.convert_ids_to_tokens(__lowerCAmelCase  ): i for i in range(self.vocab_size  )}
														vocab.update(self.added_tokens_encoder  )
														return vocab






							def __getstate__(						self  ):
														UpperCamelCase__				     =			self.__dict__.copy()
														UpperCamelCase__				     =			None
														return state






							def __setstate__(						self  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			d

														# for backward compatibility
														if not hasattr(self  , """sp_model_kwargs"""  ):
																					UpperCamelCase__				     =			{}

														UpperCamelCase__				     =			spm.SentencePieceProcessor(**self.sp_model_kwargs  )
														self.sp_model.Load(self.vocab_file  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														return self.sp_model.encode(__lowerCAmelCase  , out_type=__lowerCAmelCase  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														if token in self.decoder:
																					return self.decoder[token]
														elif token in self.added_tokens_decoder:
																					return self.added_tokens_decoder[token]
														UpperCamelCase__				     =			self.sp_model.piece_to_id(__lowerCAmelCase  )
														return sp_id + self.offset






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														if index in self.encoder:
																					return self.encoder[index]
														elif index in self.added_tokens_encoder:
																					return self.added_tokens_encoder[index]
														else:
																					UpperCamelCase__				     =			self.sp_model.IdToPiece(index - self.offset  )
														return token






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			[]
														UpperCamelCase__				     =			""""""
														for token in tokens:
																					# make sure that special tokens are not decoded using sentencepiece model
																					if token in self.all_special_tokens:
																												out_string += self.sp_model.decode(__lowerCAmelCase  ) + token
																												UpperCamelCase__				     =			[]
																					else:
																												current_sub_tokens.append(__lowerCAmelCase  )
														out_string += self.sp_model.decode(__lowerCAmelCase  )
														return out_string.strip()






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase=False  ):
														return 1






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			set(self.all_special_ids  )  # call it once instead of inside list comp
														all_special_ids.remove(self.unk_token_id  )  # <unk> is only sometimes special

														return [1 if x in all_special_ids else 0 for x in seq]






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase = None  , __lowerCAmelCase = False  ):
														if already_has_special_tokens:
																					return self._special_token_mask(__lowerCAmelCase  )
														elif token_ids_a is None:
																					return self._special_token_mask(__lowerCAmelCase  ) + [1]
														else:
																					return self._special_token_mask(token_ids_a + token_ids_a  ) + [1]






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase=None  ):
														if token_ids_a is None:
																					return token_ids_a + [self.eos_token_id]
														# We don't expect to process pairs, but leave the pair logic for API consistency
														return token_ids_a + token_ids_a + [self.eos_token_id]







							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase = None  ):
														if not os.path.isdir(__lowerCAmelCase  ):
																					logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""  )
																					return
														UpperCamelCase__				     =			os.path.join(
														    __lowerCAmelCase  , (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )

														if os.path.abspath(self.vocab_file  ) != os.path.abspath(__lowerCAmelCase  ) and os.path.isfile(self.vocab_file  ):
																					copyfile(self.vocab_file  , __lowerCAmelCase  )
														elif not os.path.isfile(self.vocab_file  ):
																					with open(__lowerCAmelCase  , """wb"""  ) as fi:
																												UpperCamelCase__				     =			self.sp_model.serialized_model_proto()
																												fi.write(__lowerCAmelCase  )

														return (out_vocab_file,)




