






'''simple docstring'''







import os
import string
import sys


__lowerCAmelCase        =							1 << 8

__lowerCAmelCase        =							{
    'tab': ord("\t"),
    'newline': ord("\r"),
    'esc': 27,
    'up': 65 + ARROW_KEY_FLAG,
    'down': 66 + ARROW_KEY_FLAG,
    'right': 67 + ARROW_KEY_FLAG,
    'left': 68 + ARROW_KEY_FLAG,
    'mod_int': 91,
    'undefined': sys.maxsize,
    'interrupt': 3,
    'insert': 50,
    'delete': 51,
    'pg_up': 53,
    'pg_down': 54,
}

__lowerCAmelCase        =							KEYMAP['up']
__lowerCAmelCase        =							KEYMAP['left']

if sys.platform == "win32":
  __lowerCAmelCase        =							[]
  __lowerCAmelCase        =							{
      B'\xe0H': KEYMAP['up'] - ARROW_KEY_FLAG,
      B'\x00H': KEYMAP['up'] - ARROW_KEY_FLAG,
      B'\xe0P': KEYMAP['down'] - ARROW_KEY_FLAG,
      B'\x00P': KEYMAP['down'] - ARROW_KEY_FLAG,
      B'\xe0M': KEYMAP['right'] - ARROW_KEY_FLAG,
      B'\x00M': KEYMAP['right'] - ARROW_KEY_FLAG,
      B'\xe0K': KEYMAP['left'] - ARROW_KEY_FLAG,
      B'\x00K': KEYMAP['left'] - ARROW_KEY_FLAG,
  }

for i in range(10):
  __lowerCAmelCase        =							ord(str(i))


def       __UpperCamelCase					(      ):
       """simple docstring"""




       if os.name == "nt":
              import msvcrt

              a_						     =			'mbcs'
              # Flush the keyboard buffer
              while msvcrt.kbhit():
                     msvcrt.getch()
              if len(lowercase_   ) == 0:
                     # Read the keystroke
                     a_						     =			msvcrt.getch()

                     # If it is a prefix char, get second part
                     if ch in (b"\x00", b"\xe0"):
                            a_						     =			ch + msvcrt.getch()
                            # Translate actual Win chars to bullet char types
                            try:
                                   a_						     =			chr(WIN_KEYMAP[cha]   )
                                   WIN_CH_BUFFER.append(chr(KEYMAP['mod_int']   )   )
                                   WIN_CH_BUFFER.append(lowercase_   )
                                   if ord(lowercase_   ) in (
                                       KEYMAP["insert"] - 1 << 9,
                                       KEYMAP["delete"] - 1 << 9,
                                       KEYMAP["pg_up"] - 1 << 9,
                                       KEYMAP["pg_down"] - 1 << 9,
                                   ):
                                          WIN_CH_BUFFER.append(chr(126   )   )
                                   a_						     =			chr(KEYMAP['esc']   )
                            except KeyError:
                                   a_						     =			cha[1]
                     else:
                            a_						     =			ch.decode(lowercase_   )
              else:
                     a_						     =			WIN_CH_BUFFER.pop(0   )
       elif os.name == "posix":
              import termios
              import tty

              a_						     =			sys.stdin.fileno()
              a_						     =			termios.tcgetattr(lowercase_   )
              try:
                     tty.setraw(lowercase_   )
                     a_						     =			sys.stdin.read(1   )
              finally:
                     termios.tcsetattr(lowercase_   ,       termios.TCSADRAIN   ,       lowercase_   )
       return ch


def       __UpperCamelCase					(      ):
       """simple docstring"""




       a_						     =			get_raw_chars()
       if ord(lowercase_   ) in [KEYMAP["interrupt"], KEYMAP["newline"]]:
              return char

       elif ord(lowercase_   ) == KEYMAP["esc"]:
              a_						     =			get_raw_chars()
              if ord(lowercase_   ) == KEYMAP["mod_int"]:
                     a_						     =			get_raw_chars()
                     if ord(lowercase_   ) >= KEYMAP["arrow_begin"] - ARROW_KEY_FLAG and ord(lowercase_   ) <= KEYMAP["arrow_end"] - ARROW_KEY_FLAG:
                            return chr(ord(lowercase_   ) + ARROW_KEY_FLAG   )
                     else:
                            return KEYMAP["undefined"]
              else:
                     return get_raw_chars()

       else:
              if char in string.printable:
                     return char
              else:
                     return KEYMAP["undefined"]








import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
	from .tokenization_pegasus import PegasusTokenizer
else:
	A			:						List[Any]        =  None


A			:						Optional[Any]        =  logging.get_logger(__name__)


A			:						str        =  '▁'

A			:						Optional[Any]        =  {'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

A			:						Any        =  {
    'vocab_file': {'google/pegasus-xsum': 'https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'},
    'tokenizer_file': {
        'google/pegasus-xsum': 'https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json'
    },
}

A			:						Optional[int]        =  {
    'google/pegasus-xsum': 5_1_2,
}



class 				A       (			UpperCAmelCase__				):
					'''simple docstring'''






					A__     =					VOCAB_FILES_NAMES
					A__     =					PRETRAINED_VOCAB_FILES_MAP
					A__     =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					A__     =					PegasusTokenizer
					A__     =					['''input_ids''', '''attention_mask''']

					def __init__(self       :		Tuple ,   _UpperCAmelCase       :		Union[str, Any]=None ,   _UpperCAmelCase       :		List[Any]=None ,   _UpperCAmelCase       :		Union[str, Any]="<pad>" ,   _UpperCAmelCase       :		int="</s>" ,   _UpperCAmelCase       :		int="<unk>" ,   _UpperCAmelCase       :		Optional[Any]="<mask_2>" ,   _UpperCAmelCase       :		Optional[Any]="<mask_1>" ,   _UpperCAmelCase       :		Optional[Any]=None ,   _UpperCAmelCase       :		Dict=103 ,   **_UpperCAmelCase       :		Dict ,   )		->  Dict:




							"""simple docstring"""


							lowercase__	     =						offset

							if additional_special_tokens is not None:
									if not isinstance(_UpperCAmelCase ,   _UpperCAmelCase						):
											raise TypeError(
											    f'''additional_special_tokens should be of type {type(_UpperCAmelCase						)}, but is'''
											    f''' {type(_UpperCAmelCase						)}'''						)

									lowercase__	     =						(
									    ([mask_token_sent] + additional_special_tokens)
									    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
									    else additional_special_tokens
									)
									# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
									additional_special_tokens_extended += [
									    f'''<unk_{i}>''' for i in range(len(_UpperCAmelCase						) ,   self.offset - 1						)
									]

									if len(set(_UpperCAmelCase						)						) != len(_UpperCAmelCase						):
											raise ValueError(
											    """Please make sure that the provided additional_special_tokens do not contain an incorrectly"""
											    f''' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.'''						)
									lowercase__	     =						additional_special_tokens_extended
							else:
									lowercase__	     =						[mask_token_sent] if mask_token_sent is not None else []
									additional_special_tokens += [f'''<unk_{i}>''' for i in range(2 ,   self.offset						)]

							super().__init__(
							    _UpperCAmelCase ,   tokenizer_file=_UpperCAmelCase ,   pad_token=_UpperCAmelCase ,   eos_token=_UpperCAmelCase ,   unk_token=_UpperCAmelCase ,   mask_token=_UpperCAmelCase ,   mask_token_sent=_UpperCAmelCase ,   offset=_UpperCAmelCase ,   additional_special_tokens=_UpperCAmelCase ,   **_UpperCAmelCase ,   )
							lowercase__	     =						vocab_file
							lowercase__	     =						False if not self.vocab_file else True

					def     lowerCamelCase__	(self       :		Optional[Any] ,   _UpperCAmelCase       :		Tuple						)		->  Union[str, Any]:




							"""simple docstring"""


							lowercase__	     =						set(self.all_special_ids						)  # call it once instead of inside list comp
							all_special_ids.remove(self.unk_token_id						)  # <unk> is only sometimes special

							if all_special_ids != set(range(len(self.additional_special_tokens						) + 3						)						):
									raise ValueError(
									    """There should be 3 special tokens: mask_token, pad_token, and eos_token +"""
									    f''' {len(self.additional_special_tokens						)} additional_special_tokens, but got {all_special_ids}'''						)

							return [1 if x in all_special_ids else 0 for x in seq]

					def     lowerCamelCase__	(self       :		Optional[Any] ,   _UpperCAmelCase       :		List ,   _UpperCAmelCase       :		Optional[List] = None ,   _UpperCAmelCase       :		bool = False						)		->  List[int]:




							"""simple docstring"""


							if already_has_special_tokens:
									return self._special_token_mask(_UpperCAmelCase						)
							elif token_ids_a is None:
									return self._special_token_mask(_UpperCAmelCase						) + [1]
							else:
									return self._special_token_mask(token_ids_a + token_ids_a						) + [1]

					def     lowerCamelCase__	(self       :		int ,   _UpperCAmelCase       :		Optional[int] ,   _UpperCAmelCase       :		Union[str, Any]=None						)		->  List[int]:




							"""simple docstring"""


							if token_ids_a is None:
									return token_ids_a + [self.eos_token_id]
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return token_ids_a + token_ids_a + [self.eos_token_id]


					def     lowerCamelCase__	(self       :		str ,   _UpperCAmelCase       :		str ,   _UpperCAmelCase       :		Optional[str] = None						)		->  Tuple[str]:




							"""simple docstring"""


							if not self.can_save_slow_tokenizer:
									raise ValueError(
									    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
									    """tokenizer."""						)

							if not os.path.isdir(_UpperCAmelCase						):
									logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
									return
							lowercase__	     =						os.path.join(
							    _UpperCAmelCase ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

							if os.path.abspath(self.vocab_file						) != os.path.abspath(_UpperCAmelCase						):
									copyfile(self.vocab_file ,   _UpperCAmelCase						)

							return (out_vocab_file,)

