
'''simple docstring'''







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_pegasus import PegasusTokenizer
else:
							_lowercase			:  str   							=	None


_lowercase			:  Union[str, Any]   							=	logging.get_logger(__name__)


_lowercase			:  Any   							=	"▁"

_lowercase			:  Tuple   							=	{"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

_lowercase			:  str   							=	{
    "vocab_file": {"google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"},
    "tokenizer_file": {
        "google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json"
    },
}

_lowercase			:  Optional[int]   							=	{
    "google/pegasus-xsum": 5_1_2,
}






class 			lowerCAmelCase__       (  lowerCamelCase_					):
					lowerCAmelCase_					       =					VOCAB_FILES_NAMES
					lowerCAmelCase_					       =					PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_					       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_					       =					PegasusTokenizer
					lowerCAmelCase_					       =					['''input_ids''', '''attention_mask''']






					def __init__(       self    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE="<pad>"    ,							__SCREAMING_SNAKE_CASE="</s>"    ,							__SCREAMING_SNAKE_CASE="<unk>"    ,							__SCREAMING_SNAKE_CASE="<mask_2>"    ,							__SCREAMING_SNAKE_CASE="<mask_1>"    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE=1_03    ,							**__SCREAMING_SNAKE_CASE    ,							):


											"""simple docstring"""





											lowercase_     :					Any													=	offset

											if additional_special_tokens is not None:
																	if not isinstance(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):
																							raise TypeError(
																							    F'''additional_special_tokens should be of type {type(__SCREAMING_SNAKE_CASE		)}, but is'''
																							    F''' {type(__SCREAMING_SNAKE_CASE		)}'''		)

																	lowercase_     :					Dict													=	(
																	    ([mask_token_sent] + additional_special_tokens)
																	    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
																	    else additional_special_tokens
																	)
																	# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
																	additional_special_tokens_extended += [
																	    F'''<unk_{i}>''' for i in range(len(__SCREAMING_SNAKE_CASE		)    ,							self.offset - 1		)
																	]

																	if len(set(__SCREAMING_SNAKE_CASE		)		) != len(__SCREAMING_SNAKE_CASE		):
																							raise ValueError(
																							    '''Please make sure that the provided additional_special_tokens do not contain an incorrectly'''
																							    F''' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.'''		)
																	lowercase_     :					List[Any]													=	additional_special_tokens_extended
											else:
																	lowercase_     :					Dict													=	[mask_token_sent] if mask_token_sent is not None else []
																	additional_special_tokens += [F'''<unk_{i}>''' for i in range(2    ,							self.offset		)]

											super().__init__(
											    __SCREAMING_SNAKE_CASE    ,							tokenizer_file=__SCREAMING_SNAKE_CASE    ,							pad_token=__SCREAMING_SNAKE_CASE    ,							eos_token=__SCREAMING_SNAKE_CASE    ,							unk_token=__SCREAMING_SNAKE_CASE    ,							mask_token=__SCREAMING_SNAKE_CASE    ,							mask_token_sent=__SCREAMING_SNAKE_CASE    ,							offset=__SCREAMING_SNAKE_CASE    ,							additional_special_tokens=__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE    ,							)
											lowercase_     :					Tuple													=	vocab_file
											lowercase_     :					Dict													=	False if not self.vocab_file else True






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Optional[Any]													=	set(self.all_special_ids		)  # call it once instead of inside list comp
											all_special_ids.remove(self.unk_token_id		)  # <unk> is only sometimes special

											if all_special_ids != set(range(len(self.additional_special_tokens		) + 3		)		):
																	raise ValueError(
																	    '''There should be 3 special tokens: mask_token, pad_token, and eos_token +'''
																	    F''' {len(self.additional_special_tokens		)} additional_special_tokens, but got {all_special_ids}'''		)

											return [1 if x in all_special_ids else 0 for x in seq]






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None    ,							__SCREAMING_SNAKE_CASE = False		):


											"""simple docstring"""





											if already_has_special_tokens:
																	return self._special_token_mask(__SCREAMING_SNAKE_CASE		)
											elif token_ids_a is None:
																	return self._special_token_mask(__SCREAMING_SNAKE_CASE		) + [1]
											else:
																	return self._special_token_mask(token_ids_a + token_ids_a		) + [1]






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE=None		):


											"""simple docstring"""





											if token_ids_a is None:
																	return token_ids_a + [self.eos_token_id]
											# We don't expect to process pairs, but leave the pair logic for API consistency
											return token_ids_a + token_ids_a + [self.eos_token_id]






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											if not self.can_save_slow_tokenizer:
																	raise ValueError(
																	    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
																	    '''tokenizer.'''		)

											if not os.path.isdir(__SCREAMING_SNAKE_CASE		):
																	logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''		)
																	return
											lowercase_     :					Tuple													=	os.path.join(
											    __SCREAMING_SNAKE_CASE    ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

											if os.path.abspath(self.vocab_file		) != os.path.abspath(__SCREAMING_SNAKE_CASE		):
																	copyfile(self.vocab_file    ,							__SCREAMING_SNAKE_CASE		)

											return (out_vocab_file,)





'''simple docstring'''







import logging
import os

from .state import PartialState






class 			lowerCAmelCase__       (  logging.LoggerAdapter					):






					@staticmethod
					def      _snake_case   (       __SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Optional[Any]													=	PartialState()
											return not main_process_only or (main_process_only and state.is_main_process)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							*__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if PartialState._shared_state == {}:
																	raise RuntimeError(
																	    '''You must initialize the accelerate state by calling either `PartialState()` or `Accelerator()` before using the logging utility.'''		)
											lowercase_     :					Tuple													=	kwargs.pop('''main_process_only'''    ,							__SCREAMING_SNAKE_CASE		)
											lowercase_     :					Optional[int]													=	kwargs.pop('''in_order'''    ,							__SCREAMING_SNAKE_CASE		)

											if self.isEnabledFor(__SCREAMING_SNAKE_CASE		):
																	if self._should_log(__SCREAMING_SNAKE_CASE		):
																							lowercase_						,			lowercase_     :					Optional[Any]													=	self.process(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		)
																							self.logger.log(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							*__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		)

																	elif in_order:
																							lowercase_     :					Optional[Any]													=	PartialState()
																							for i in range(state.num_processes		):
																													if i == state.process_index:
																																			lowercase_						,			lowercase_     :					Optional[int]													=	self.process(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		)
																																			self.logger.log(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							*__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		)
																													state.wait_for_everyone()







def  snake_case_		(    __SCREAMING_SNAKE_CASE :			str				,				__SCREAMING_SNAKE_CASE :			str = None							):



						"""simple docstring"""





						if log_level is None:
												lowercase_     :					Any													=	os.environ.get('''ACCELERATE_LOG_LEVEL'''				,				__SCREAMING_SNAKE_CASE							)
						lowercase_     :					List[str]													=	logging.getLogger(__SCREAMING_SNAKE_CASE							)
						if log_level is not None:
												logger.setLevel(log_level.upper()							)
												logger.root.setLevel(log_level.upper()							)
						return MultiProcessAdapter(__SCREAMING_SNAKE_CASE				,				{}							)



