
"""simple docstring"""



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__lowercase						:     Dict							      =		"""▁"""

__lowercase						:     Optional[Any]							      =		{"""vocab_file""": """spiece.model"""}

__lowercase						:     List[str]							      =		{
    """vocab_file""": {"""google/pegasus-xsum""": """https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"""}
}

__lowercase						:     Optional[Any]							      =		{
    """google/pegasus-xsum""": 5_1_2,
}


__lowercase						:     int							      =		logging.get_logger(__name__)





class 					lowerCAmelCase			(				snake_case_					):
     """simple docstring"""







     __lowercase						:Optional[int]          =    VOCAB_FILES_NAMES

     __lowercase						:Optional[Any]          =    VOCAB_FILES_NAMES
     __lowercase						:str          =    PRETRAINED_VOCAB_FILES_MAP
     __lowercase						:Optional[int]          =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __lowercase						:int          =    ["input_ids", "attention_mask"]




     def __init__(      self ,		UpperCamelCase__ ,		UpperCamelCase__="<pad>" ,		UpperCamelCase__="</s>" ,		UpperCamelCase__="<unk>" ,		UpperCamelCase__="<mask_2>" ,		UpperCamelCase__="<mask_1>" ,		UpperCamelCase__=None ,		UpperCamelCase__=103 ,		UpperCamelCase__ = None ,		**UpperCamelCase__ ,		)					->  None:







            '''simple docstring'''





            lowerCamelCase_       							=		offset
            if additional_special_tokens is not None:
                   if not isinstance(UpperCamelCase__ ,		UpperCamelCase__							):
                          raise TypeError(
                              F"""additional_special_tokens should be of type {type(UpperCamelCase__							)}, but is"""
                              F""" {type(UpperCamelCase__							)}"""							)

                   lowerCamelCase_       							=		(
                       ([mask_token_sent] + additional_special_tokens)
                       if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
                       else additional_special_tokens
                   )
                   # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
                   additional_special_tokens_extended += [
                       F"""<unk_{i}>""" for i in range(len(UpperCamelCase__							) ,		self.offset - 1							)
                   ]

                   if len(set(UpperCamelCase__							)							) != len(UpperCamelCase__							):
                          raise ValueError(
                              '''Please make sure that the provided additional_special_tokens do not contain an incorrectly'''
                              F""" shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}."""							)
                   lowerCamelCase_       							=		additional_special_tokens_extended
            else:
                   lowerCamelCase_       							=		[mask_token_sent] if mask_token_sent is not None else []
                   additional_special_tokens += [F"""<unk_{i}>""" for i in range(2 ,		self.offset							)]

            lowerCamelCase_       							=		{} if sp_model_kwargs is None else sp_model_kwargs

            super().__init__(
                eos_token=UpperCamelCase__ ,		unk_token=UpperCamelCase__ ,		mask_token=UpperCamelCase__ ,		pad_token=UpperCamelCase__ ,		mask_token_sent=UpperCamelCase__ ,		offset=UpperCamelCase__ ,		additional_special_tokens=UpperCamelCase__ ,		sp_model_kwargs=self.sp_model_kwargs ,		**UpperCamelCase__ ,		)
            lowerCamelCase_       							=		mask_token_sent
            lowerCamelCase_       							=		vocab_file
            lowerCamelCase_       							=		spm.SentencePieceProcessor(**self.sp_model_kwargs							)
            self.sp_model.Load(UpperCamelCase__							)

            # add special tokens to encoder dict
            lowerCamelCase_       							=		{
                0: self.pad_token,
                1: self.eos_token,
            }

            if self.mask_token_sent is not None:
                   self.encoder.update(
                       {
                           2: self.mask_token_sent,
                           3: self.mask_token,
                       }							)

            if self.offset > 0:
                   # entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
                   # mask_token_sent is already added to list -> so start at 1
                   self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1 ,		self.offset - 1							)}							)

            lowerCamelCase_       							=		{v: k for k, v in self.encoder.items()}




     @property
     def    _lowerCAmelCase		(      self							)					->  int:







            '''simple docstring'''





            return len(self.sp_model							) + self.offset




     def    _lowerCAmelCase		(      self							)					->  Dict[str, int]:







            '''simple docstring'''





            lowerCamelCase_       							=		{self.convert_ids_to_tokens(UpperCamelCase__							): i for i in range(self.vocab_size							)}
            vocab.update(self.added_tokens_encoder							)
            return vocab




     def __getstate__(      self							)					->  Optional[int]:







            '''simple docstring'''





            lowerCamelCase_       							=		self.__dict__.copy()
            lowerCamelCase_       							=		None
            return state




     def __setstate__(      self ,		UpperCamelCase__							)					->  str:







            '''simple docstring'''





            lowerCamelCase_       							=		d

            # for backward compatibility
            if not hasattr(self ,		'''sp_model_kwargs'''							):
                   lowerCamelCase_       							=		{}

            lowerCamelCase_       							=		spm.SentencePieceProcessor(**self.sp_model_kwargs							)
            self.sp_model.Load(self.vocab_file							)




     def    _lowerCAmelCase		(      self ,		UpperCamelCase__							)					->  List[str]:







            '''simple docstring'''





            return self.sp_model.encode(UpperCamelCase__ ,		out_type=UpperCamelCase__							)




     def    _lowerCAmelCase		(      self ,		UpperCamelCase__							)					->  int:







            '''simple docstring'''





            if token in self.decoder:
                   return self.decoder[token]
            elif token in self.added_tokens_decoder:
                   return self.added_tokens_decoder[token]
            lowerCamelCase_       							=		self.sp_model.piece_to_id(UpperCamelCase__							)
            return sp_id + self.offset




     def    _lowerCAmelCase		(      self ,		UpperCamelCase__							)					->  str:







            '''simple docstring'''





            if index in self.encoder:
                   return self.encoder[index]
            elif index in self.added_tokens_encoder:
                   return self.added_tokens_encoder[index]
            else:
                   lowerCamelCase_       							=		self.sp_model.IdToPiece(index - self.offset							)
            return token




     def    _lowerCAmelCase		(      self ,		UpperCamelCase__							)					->  Union[str, Any]:







            '''simple docstring'''





            lowerCamelCase_       							=		[]
            lowerCamelCase_       							=		''''''
            for token in tokens:
                   # make sure that special tokens are not decoded using sentencepiece model
                   if token in self.all_special_tokens:
                          out_string += self.sp_model.decode(UpperCamelCase__							) + token
                          lowerCamelCase_       							=		[]
                   else:
                          current_sub_tokens.append(UpperCamelCase__							)
            out_string += self.sp_model.decode(UpperCamelCase__							)
            return out_string.strip()




     def    _lowerCAmelCase		(      self ,		UpperCamelCase__=False							)					->  str:







            '''simple docstring'''





            return 1




     def    _lowerCAmelCase		(      self ,		UpperCamelCase__							)					->  Union[str, Any]:







            '''simple docstring'''





            lowerCamelCase_       							=		set(self.all_special_ids							)  # call it once instead of inside list comp
            all_special_ids.remove(self.unk_token_id							)  # <unk> is only sometimes special

            return [1 if x in all_special_ids else 0 for x in seq]




     def    _lowerCAmelCase		(      self ,		UpperCamelCase__ ,		UpperCamelCase__ = None ,		UpperCamelCase__ = False							)					->  List[int]:







            '''simple docstring'''





            if already_has_special_tokens:
                   return self._special_token_mask(UpperCamelCase__							)
            elif token_ids_a is None:
                   return self._special_token_mask(UpperCamelCase__							) + [1]
            else:
                   return self._special_token_mask(token_ids_a + token_ids_a							) + [1]




     def    _lowerCAmelCase		(      self ,		UpperCamelCase__ ,		UpperCamelCase__=None							)					->  List[int]:







            '''simple docstring'''





            if token_ids_a is None:
                   return token_ids_a + [self.eos_token_id]
            # We don't expect to process pairs, but leave the pair logic for API consistency
            return token_ids_a + token_ids_a + [self.eos_token_id]




     def    _lowerCAmelCase		(      self ,		UpperCamelCase__ ,		UpperCamelCase__ = None							)					->  Tuple[str]:







            '''simple docstring'''





            if not os.path.isdir(UpperCamelCase__							):
                   logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""							)
                   return
            lowerCamelCase_       							=		os.path.join(
                UpperCamelCase__ ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

            if os.path.abspath(self.vocab_file							) != os.path.abspath(UpperCamelCase__							) and os.path.isfile(self.vocab_file							):
                   copyfile(self.vocab_file ,		UpperCamelCase__							)
            elif not os.path.isfile(self.vocab_file							):
                   with open(UpperCamelCase__ ,		'''wb'''							) as fi:
                          lowerCamelCase_       							=		self.sp_model.serialized_model_proto()
                          fi.write(UpperCamelCase__							)

            return (out_vocab_file,)

import enum
import shutil
import sys


__A,		__A							=shutil.get_terminal_size()

__A							={'''UP''': '''A''', '''DOWN''': '''B''', '''RIGHT''': '''C''', '''LEFT''': '''D'''}




class        _SCREAMING_SNAKE_CASE	( enum.Enum		):
	lowerCAmelCase__   =					0
	lowerCAmelCase__   =					1





def      lowerCamelCase_					(							lowerCamelCase__      ,					lowerCamelCase__=""       ):
				sys.stdout.write(str(lowerCamelCase__       ) + end       )
				sys.stdout.flush()





def      lowerCamelCase_					(							lowerCamelCase__      ,					lowerCamelCase__      ,					lowerCamelCase__=""       ):
				forceWrite(F'\u001b[{color}m{content}\u001b[0m'      ,					lowerCamelCase__       )





def      lowerCamelCase_					(							):
				forceWrite("\r"       )





def      lowerCamelCase_					(							lowerCamelCase__      ,					lowerCamelCase__       ):
				forceWrite(F'\033[{num_lines}{CURSOR_TO_CHAR[direction.upper()]}'       )





def      lowerCamelCase_					(							):
				forceWrite(" " * TERMINAL_WIDTH       )
				reset_cursor()





def      lowerCamelCase_					(							):
				reset_cursor()
				forceWrite("-" * TERMINAL_WIDTH       )


