


import os
import re
import warnings
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer


if TYPE_CHECKING:
						from ...tokenization_utils_base import TextInput
from ...utils import logging


__UpperCamelCase    		=       logging.get_logger(__name__)

__UpperCamelCase    		=       {'vocab_file': 'spiece.model'}

__UpperCamelCase    		=       {
    'vocab_file': {
        't5-small': 'https://huggingface.co/t5-small/resolve/main/spiece.model',
        't5-base': 'https://huggingface.co/t5-base/resolve/main/spiece.model',
        't5-large': 'https://huggingface.co/t5-large/resolve/main/spiece.model',
        't5-3b': 'https://huggingface.co/t5-3b/resolve/main/spiece.model',
        't5-11b': 'https://huggingface.co/t5-11b/resolve/main/spiece.model',
    }
}


# TODO(PVP) - this should be removed in Transformers v5
__UpperCamelCase    		=       {
    't5-small': 5_1_2,
    't5-base': 5_1_2,
    't5-large': 5_1_2,
    't5-3b': 5_1_2,
    't5-11b': 5_1_2,
}

__UpperCamelCase    		=       '▁'







class        lowerCamelCase__	(							UpperCAmelCase      ):






			"""simple docstring"""






			_UpperCamelCase		:					str    =		VOCAB_FILES_NAMES
			_UpperCamelCase		:					List[str]    =		PRETRAINED_VOCAB_FILES_MAP
			_UpperCamelCase		:					str    =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCamelCase		:					Dict    =		['input_ids', 'attention_mask']




			def __init__(							self      ,						snake_case      ,						snake_case="</s>"      ,						snake_case="<unk>"      ,						snake_case="<pad>"      ,						snake_case=100      ,						snake_case=None      ,						snake_case = None      ,						snake_case=True      ,						**snake_case      ,						):

						'''simple docstring'''


						if extra_ids > 0 and additional_special_tokens is None:
									UpperCamelCase__					=       [F'''<extra_id_{i}>''' for i in range(snake_case     )]
						elif extra_ids > 0 and additional_special_tokens is not None:
									# Check that we have the right number of extra_id special tokens
									UpperCamelCase__					=       len(set(filter(lambda snake_case     : bool("extra_id" in str(snake_case     )     )      ,						snake_case     )     )     )
									if extra_tokens != extra_ids:
												raise ValueError(
												    F'''Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'''
												    " provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"
												    " tokens"     )
						if legacy:
									logger.warning_once(
									    F'''You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to'''
									    " read the related pull request available at https://github.com/huggingface/transformers/pull/24565"     )

						UpperCamelCase__					=       legacy
						UpperCamelCase__					=       {} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    eos_token=snake_case      ,						unk_token=snake_case      ,						pad_token=snake_case      ,						extra_ids=snake_case      ,						additional_special_tokens=snake_case      ,						sp_model_kwargs=self.sp_model_kwargs      ,						legacy=snake_case      ,						**snake_case      ,						)

						UpperCamelCase__					=       vocab_file
						UpperCamelCase__					=       extra_ids

						UpperCamelCase__					=       spm.SentencePieceProcessor(**self.sp_model_kwargs     )
						self.sp_model.Load(snake_case     )




			@staticmethod
			def 				snake_case__				(							snake_case      ,						snake_case      ,						snake_case     ):

						'''simple docstring'''


						if pretrained_model_name_or_path in TaTokenizer.max_model_input_sizes:
									UpperCamelCase__					=       TaTokenizer.max_model_input_sizes[pretrained_model_name_or_path]
									if init_max_model_length is not None and init_max_model_length != max_model_length:
												return init_max_model_length
									elif init_max_model_length is None:
												warnings.warn(
												    "This tokenizer was incorrectly instantiated with a model max length of"
												    F''' {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this'''
												    " behavior is kept to avoid breaking backwards compatibility when padding/encoding with"
												    " `truncation is True`.\n- Be aware that you SHOULD NOT rely on"
												    F''' {pretrained_model_name_or_path} automatically truncating your input to'''
												    F''' {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences'''
												    F''' longer than {deprecated_max_model_length} you can either instantiate this tokenizer with'''
												    " `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please"
												    " instantiate this tokenizer with `model_max_length` set to your preferred value."      ,						snake_case      ,						)

						return max_model_length




			@property
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						return self.sp_model.get_piece_size() + self._extra_ids




			def 				snake_case__				(							self     ):

						'''simple docstring'''


						UpperCamelCase__					=       {self.convert_ids_to_tokens(snake_case     ): i for i in range(self.vocab_size     )}
						vocab.update(self.added_tokens_encoder     )
						return vocab




			def 				snake_case__				(							self      ,						snake_case      ,						snake_case = None      ,						snake_case = False     ):

						'''simple docstring'''


						if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=snake_case      ,						token_ids_a=snake_case      ,						already_has_special_tokens=snake_case     )

						# normal case: some special tokens
						if token_ids_a is None:
									return ([0] * len(snake_case     )) + [1]
						return ([0] * len(snake_case     )) + [1] + ([0] * len(snake_case     )) + [1]




			def 				snake_case__				(							self     ):

						'''simple docstring'''


						return list(
						    set(filter(lambda snake_case     : bool(re.search(R"<extra_id_\d+>"      ,						snake_case     )     ) is not None      ,						self.additional_special_tokens     )     )     )




			def 				snake_case__				(							self     ):

						'''simple docstring'''


						return [self._convert_token_to_id(snake_case     ) for token in self.get_sentinel_tokens()]




			def 				snake_case__				(							self      ,						snake_case     ):

						'''simple docstring'''


						if len(snake_case     ) > 0 and token_ids[-1] == self.eos_token_id:
									warnings.warn(
									    F'''This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated'''
									    " eos tokens being added."     )
									return token_ids
						else:
									return token_ids + [self.eos_token_id]




			def 				snake_case__				(							self      ,						snake_case      ,						snake_case = None     ):

						'''simple docstring'''


						UpperCamelCase__					=       [self.eos_token_id]

						if token_ids_a is None:
									return len(token_ids_a + eos     ) * [0]
						return len(token_ids_a + eos + token_ids_a + eos     ) * [0]




			def 				snake_case__				(							self      ,						snake_case      ,						snake_case = None     ):

						'''simple docstring'''


						UpperCamelCase__					=       self._add_eos_if_not_present(snake_case     )
						if token_ids_a is None:
									return token_ids_a
						else:
									UpperCamelCase__					=       self._add_eos_if_not_present(snake_case     )
									return token_ids_a + token_ids_a




			def __getstate__(							self     ):

						'''simple docstring'''


						UpperCamelCase__					=       self.__dict__.copy()
						UpperCamelCase__					=       None
						return state




			def __setstate__(							self      ,						snake_case     ):

						'''simple docstring'''


						UpperCamelCase__					=       d

						# for backward compatibility
						if not hasattr(self      ,						"sp_model_kwargs"     ):
									UpperCamelCase__					=       {}

						UpperCamelCase__					=       spm.SentencePieceProcessor(**self.sp_model_kwargs     )
						self.sp_model.Load(self.vocab_file     )




			def 				snake_case__				(							self      ,						snake_case      ,						**snake_case     ):

						'''simple docstring'''


						if not self.legacy:
									UpperCamelCase__					=       SPIECE_UNDERLINE + text.replace(snake_case      ,						" "     )
						return super().tokenize(snake_case      ,						**snake_case     )




			def 				snake_case__				(							self      ,						snake_case      ,						**snake_case     ):

						'''simple docstring'''


						if not self.legacy:
									UpperCamelCase__					=       text.startswith(snake_case     )
									if is_first:
												UpperCamelCase__					=       text[1:]

						UpperCamelCase__					=       self.sp_model.encode(snake_case      ,						out_type=snake_case     )

						if not self.legacy and not is_first and not text.startswith(" "     ) and tokens[0].startswith(snake_case     ):
									UpperCamelCase__					=       ([tokens[0][1:]] if len(tokens[0]     ) > 1 else []) + tokens[1:]
						return tokens




			def 				snake_case__				(							self      ,						snake_case     ):

						'''simple docstring'''


						if token.startswith("<extra_id_"     ):
									UpperCamelCase__					=       re.match(R"<extra_id_(\d+)>"      ,						snake_case     )
									UpperCamelCase__					=       int(match.group(1     )     )
									return self.vocab_size - num - 1
						return self.sp_model.piece_to_id(snake_case     )




			def 				snake_case__				(							self      ,						snake_case     ):

						'''simple docstring'''


						if index < self.sp_model.get_piece_size():
									UpperCamelCase__					=       self.sp_model.IdToPiece(snake_case     )
						else:
									UpperCamelCase__					=       F'''<extra_id_{self.vocab_size - 1 - index}>'''
						return token




			def 				snake_case__				(							self      ,						snake_case     ):

						'''simple docstring'''


						UpperCamelCase__					=       []
						UpperCamelCase__					=       ""
						UpperCamelCase__					=       False
						for token in tokens:
									# make sure that special tokens are not decoded using sentencepiece model
									if token in self.all_special_tokens:
												if not prev_is_special:
															out_string += " "
												out_string += self.sp_model.decode(snake_case     ) + token
												UpperCamelCase__					=       True
												UpperCamelCase__					=       []
									else:
												current_sub_tokens.append(snake_case     )
												UpperCamelCase__					=       False
						out_string += self.sp_model.decode(snake_case     )
						return out_string.strip()







			def 				snake_case__				(							self      ,						snake_case      ,						snake_case = None     ):

						'''simple docstring'''


						if not os.path.isdir(snake_case     ):
									logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''     )
									return
						UpperCamelCase__					=       os.path.join(
						    snake_case      ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

						if os.path.abspath(self.vocab_file     ) != os.path.abspath(snake_case     ) and os.path.isfile(self.vocab_file     ):
									copyfile(self.vocab_file      ,						snake_case     )
						elif not os.path.isfile(self.vocab_file     ):
									with open(snake_case      ,						"wb"     ) as fi:
												UpperCamelCase__					=       self.sp_model.serialized_model_proto()
												fi.write(snake_case     )

						return (out_vocab_file,)






import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


__UpperCamelCase    		=       logging.get_logger(__name__)
def        UpperCamelCase_(				)->		Any:
			# Get the sagemaker specific mp parameters from smp_options variable.
			UpperCamelCase__					=       os.getenv("SM_HP_MP_PARAMETERS"				,     "{}"  )
			try:
						# Parse it and check the field "partitions" is included, it is required for model parallel.
						UpperCamelCase__					=       json.loads(_A  )
						if "partitions" not in smp_options:
									return False
			except json.JSONDecodeError:
						return False

			# Get the sagemaker specific framework parameters from mpi_options variable.
			UpperCamelCase__					=       os.getenv("SM_FRAMEWORK_PARAMS"				,     "{}"  )
			try:
						# Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
						UpperCamelCase__					=       json.loads(_A  )
						if not mpi_options.get("sagemaker_mpi_enabled"				,     _A  ):
									return False
			except json.JSONDecodeError:
						return False
			# Lastly, check if the `smdistributed` module is present.
			return importlib.util.find_spec("smdistributed"  ) is not None


if is_sagemaker_model_parallel_available():
						import smdistributed.modelparallel.torch as smp

						smp.init()







@dataclass
class        lowerCamelCase__	(							UpperCAmelCase      ):






			"""simple docstring"""






			_UpperCamelCase		:					str    =		field(
			    default=''					,	metadata={'help': 'Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer'}					,	)




			def 				snake_case__				(							self     ):

						'''simple docstring'''


						super().__post_init__()
						warnings.warn(
						    "`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use "
						    "`TrainingArguments` instead."      ,						snake_case      ,						)




			@cached_property
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						logger.info("PyTorch: setting up devices"     )
						if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
									logger.warning(
									    "torch.distributed process group is initialized, but local_rank == -1. "
									    "In order to use Torch DDP, launch your script with `python -m torch.distributed.launch"     )
						if self.no_cuda:
									UpperCamelCase__					=       torch.device("cpu"     )
									UpperCamelCase__					=       0
						elif is_sagemaker_model_parallel_available():
									UpperCamelCase__					=       smp.local_rank()
									UpperCamelCase__					=       torch.device("cuda"      ,						snake_case     )
									UpperCamelCase__					=       1
						elif is_sagemaker_dp_enabled():
									import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

									torch.distributed.init_process_group(backend="smddp"      ,						timeout=self.ddp_timeout_delta     )
									UpperCamelCase__					=       int(os.getenv("SMDATAPARALLEL_LOCAL_RANK"     )     )
									UpperCamelCase__					=       torch.device("cuda"      ,						self.local_rank     )
									UpperCamelCase__					=       1
						elif self.local_rank == -1:
									# if n_gpu is > 1 we'll use nn.DataParallel.
									# If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
									# Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
									# trigger an error that a device index is missing. Index 0 takes into account the
									# GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
									# will use the first GPU in that env, i.e. GPU#1
									UpperCamelCase__					=       torch.device("cuda:0" if torch.cuda.is_available() else "cpu"     )
									# Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
									# the default value.
									UpperCamelCase__					=       torch.cuda.device_count()
						else:
									# Here, we'll use torch.distributed.
									# Initializes the distributed backend which will take care of synchronizing nodes/GPUs
									if not torch.distributed.is_initialized():
												torch.distributed.init_process_group(backend="nccl"      ,						timeout=self.ddp_timeout_delta     )
									UpperCamelCase__					=       torch.device("cuda"      ,						self.local_rank     )
									UpperCamelCase__					=       1

						if device.type == "cuda":
									torch.cuda.set_device(snake_case     )

						return device




			@property
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						if is_sagemaker_model_parallel_available():
									return smp.dp_size()

						return super().world_size




			@property
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						return not is_sagemaker_model_parallel_available()







			@property
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						return False


