



'''simple docstring'''






import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowerCAmelCase:   str					=		logging.get_logger(__name__)

lowerCAmelCase:   Dict					=		{'vocab_file': 'spiece.model'}

lowerCAmelCase:   Tuple					=		{
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

lowerCAmelCase:   Union[str, Any]					=		{
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
lowerCAmelCase:   Union[str, Any]					=		0
lowerCAmelCase:   List[str]					=		1
lowerCAmelCase:   Union[str, Any]					=		2
lowerCAmelCase:   List[str]					=		3
lowerCAmelCase:   List[str]					=		4

class 				a__(   lowerCamelCase__      ):
    lowercase__					=       VOCAB_FILES_NAMES
    lowercase__					=       PRETRAINED_VOCAB_FILES_MAP
    lowercase__					=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase__					=       """left"""







    def __init__( self					:    Optional[int]		,     __snake_case					:    Optional[Any]		,     __snake_case					:    List[str]=False		,     __snake_case					:    Union[str, Any]=True		,     __snake_case					:    Any=False		,     __snake_case					:    List[str]="<s>"		,     __snake_case					:    Union[str, Any]="</s>"		,     __snake_case					:    Optional[int]="<unk>"		,     __snake_case					:    Tuple="<sep>"		,     __snake_case					:    Dict="<pad>"		,     __snake_case					:    Any="<cls>"		,     __snake_case					:    Union[str, Any]="<mask>"		,     __snake_case					:    Tuple=["<eop>", "<eod>"]		,     __snake_case					:    Optional[Dict[str, Any]] = None		,     **__snake_case					:    List[str]		,     ):
      # Mask token behave like a normal word, i.e. include the space before it
      a		:   str	  =	AddedToken(__snake_case		,     lstrip=__snake_case		,     rstrip=__snake_case      ) if isinstance(__snake_case		,     __snake_case      ) else mask_token

      a		:   Tuple	  =	{} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          do_lower_case=__snake_case		,     remove_space=__snake_case		,     keep_accents=__snake_case		,     bos_token=__snake_case		,     eos_token=__snake_case		,     unk_token=__snake_case		,     sep_token=__snake_case		,     pad_token=__snake_case		,     cls_token=__snake_case		,     mask_token=__snake_case		,     additional_special_tokens=__snake_case		,     sp_model_kwargs=self.sp_model_kwargs		,     **__snake_case		,     )

      a		:   List[str]	  =	3

      a		:   List[Any]	  =	do_lower_case
      a		:   Tuple	  =	remove_space
      a		:   Union[str, Any]	  =	keep_accents
      a		:   List[str]	  =	vocab_file

      a		:   Dict	  =	spm.SentencePieceProcessor(**self.sp_model_kwargs      )
      self.sp_model.Load(__snake_case      )







    @property
    def      lowercase_			( self					:    Dict      ):
      return len(self.sp_model      )







    def      lowercase_			( self					:    Optional[Any]      ):
      a		:   Tuple	  =	{self.convert_ids_to_tokens(__snake_case      ): i for i in range(self.vocab_size      )}
      vocab.update(self.added_tokens_encoder      )
      return vocab







    def __getstate__( self					:    Any      ):
      a		:   str	  =	self.__dict__.copy()
      a		:   Union[str, Any]	  =	None
      return state







    def __setstate__( self					:    List[str]		,     __snake_case					:    str      ):
      a		:   Optional[int]	  =	d

      # for backward compatibility
      if not hasattr(self		,     'sp_model_kwargs'      ):
        a		:   Tuple	  =	{}

      a		:   str	  =	spm.SentencePieceProcessor(**self.sp_model_kwargs      )
      self.sp_model.Load(self.vocab_file      )







    def      lowercase_			( self					:    Union[str, Any]		,     __snake_case					:    str      ):
      if self.remove_space:
        a		:   Optional[int]	  =	' '.join(inputs.strip().split()      )
      else:
        a		:   Dict	  =	inputs
      a		:   Dict	  =	outputs.replace('``'		,     '"'      ).replace('\'\''		,     '"'      )

      if not self.keep_accents:
        a		:   str	  =	unicodedata.normalize('NFKD'		,     __snake_case      )
        a		:   List[str]	  =	''.join([c for c in outputs if not unicodedata.combining(__snake_case      )]      )
      if self.do_lower_case:
        a		:   Any	  =	outputs.lower()

      return outputs







    def      lowercase_			( self					:    int		,     __snake_case					:    str      ):
      a		:   Any	  =	self.preprocess_text(__snake_case      )
      a		:   Optional[Any]	  =	self.sp_model.encode(__snake_case		,     out_type=__snake_case      )
      a		:   List[Any]	  =	[]
      for piece in pieces:
        if len(__snake_case      ) > 1 and piece[-1] == str(','      ) and piece[-2].isdigit():
          a		:   Union[str, Any]	  =	self.sp_model.EncodeAsPieces(piece[:-1].replace(__snake_case		,     ''      )      )
          if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
            if len(cur_pieces[0]      ) == 1:
              a		:   List[str]	  =	cur_pieces[1:]
            else:
              a		:   Any	  =	cur_pieces[0][1:]
          cur_pieces.append(piece[-1]      )
          new_pieces.extend(__snake_case      )
        else:
          new_pieces.append(__snake_case      )

      return new_pieces







    def      lowercase_			( self					:    Tuple		,     __snake_case					:    List[str]      ):
      return self.sp_model.PieceToId(__snake_case      )







    def      lowercase_			( self					:    str		,     __snake_case					:    Union[str, Any]      ):
      return self.sp_model.IdToPiece(__snake_case      )







    def      lowercase_			( self					:    Any		,     __snake_case					:    str      ):
      a		:   List[str]	  =	''.join(__snake_case      ).replace(__snake_case		,     ' '      ).strip()
      return out_string







    def      lowercase_			( self					:    List[Any]		,     __snake_case					:    List[int]		,     __snake_case					:    bool = False		,     __snake_case					:    bool = None		,     __snake_case					:    bool = True		,     **__snake_case					:    Any		,     ):
      a		:   List[Any]	  =	kwargs.pop('use_source_tokenizer'		,     __snake_case      )

      a		:   Tuple	  =	self.convert_ids_to_tokens(__snake_case		,     skip_special_tokens=__snake_case      )

      # To avoid mixing byte-level and unicode for byte-level BPT
      # we need to build string separately for added tokens and byte-level tokens
      # cf. https://github.com/huggingface/transformers/issues/1133
      a		:   int	  =	[]
      a		:   Dict	  =	[]
      for token in filtered_tokens:
        if skip_special_tokens and token in self.all_special_ids:
          continue
        if token in self.added_tokens_encoder:
          if current_sub_text:
            sub_texts.append(self.convert_tokens_to_string(__snake_case      )      )
            a		:   Dict	  =	[]
          sub_texts.append(__snake_case      )
        else:
          current_sub_text.append(__snake_case      )
      if current_sub_text:
        sub_texts.append(self.convert_tokens_to_string(__snake_case      )      )

      # Mimic the behavior of the Rust tokenizer:
      # By default, there are no spaces between special tokens
      a		:   Optional[Any]	  =	''.join(__snake_case      )

      a		:   Optional[int]	  =	(
          clean_up_tokenization_spaces
          if clean_up_tokenization_spaces is not None
          else self.clean_up_tokenization_spaces
      )
      if clean_up_tokenization_spaces:
        a		:   List[str]	  =	self.clean_up_tokenization(__snake_case      )
        return clean_text
      else:
        return text







    def      lowercase_			( self					:    str		,     __snake_case					:    List[int]		,     __snake_case					:    Optional[List[int]] = None      ):
      a		:   Optional[int]	  =	[self.sep_token_id]
      a		:   List[str]	  =	[self.cls_token_id]
      if token_ids_a is None:
        return token_ids_a + sep + cls
      return token_ids_a + sep + token_ids_a + sep + cls







    def      lowercase_			( self					:    Union[str, Any]		,     __snake_case					:    List[int]		,     __snake_case					:    Optional[List[int]] = None		,     __snake_case					:    bool = False      ):

      if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=__snake_case		,     token_ids_a=__snake_case		,     already_has_special_tokens=__snake_case      )

      if token_ids_a is not None:
        return ([0] * len(__snake_case      )) + [1] + ([0] * len(__snake_case      )) + [1, 1]
      return ([0] * len(__snake_case      )) + [1, 1]







    def      lowercase_			( self					:    Optional[int]		,     __snake_case					:    List[int]		,     __snake_case					:    Optional[List[int]] = None      ):
      a		:   List[Any]	  =	[self.sep_token_id]
      a		:   Optional[int]	  =	[2]

      if token_ids_a is None:
        return len(token_ids_a + sep      ) * [0] + cls_segment_id
      return len(token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1] + cls_segment_id







    def      lowercase_			( self					:    int		,     __snake_case					:    str		,     __snake_case					:    Optional[str] = None      ):
      if not os.path.isdir(__snake_case      ):
        logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""      )
        return
      a		:   str	  =	os.path.join(
          __snake_case		,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )

      if os.path.abspath(self.vocab_file      ) != os.path.abspath(__snake_case      ) and os.path.isfile(self.vocab_file      ):
        copyfile(self.vocab_file		,     __snake_case      )
      elif not os.path.isfile(self.vocab_file      ):
        with open(__snake_case		,     'wb'      ) as fi:
          a		:   Optional[int]	  =	self.sp_model.serialized_model_proto()
          fi.write(__snake_case      )

      return (out_vocab_file,)




'''simple docstring'''






from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase:   Tuple					=		logging.get_logger(__name__)

lowerCAmelCase:   Any					=		{
    'google/pegasus-large': 'https://huggingface.co/google/pegasus-large/resolve/main/config.json',
    # See all PEGASUS models at https://huggingface.co/models?filter=pegasus
}

class 				a__(   lowerCamelCase__      ):
    lowercase__					=       """pegasus"""
    lowercase__					=       ["""past_key_values"""]
    lowercase__					=       {"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}







    def __init__( self					:    Union[str, Any]		,     __snake_case					:    Tuple=5_02_65		,     __snake_case					:    List[Any]=10_24		,     __snake_case					:    Optional[Any]=12		,     __snake_case					:    Tuple=40_96		,     __snake_case					:    str=16		,     __snake_case					:    int=12		,     __snake_case					:    Tuple=40_96		,     __snake_case					:    int=16		,     __snake_case					:    Tuple=0.0		,     __snake_case					:    Any=0.0		,     __snake_case					:    Any=True		,     __snake_case					:    Optional[int]=True		,     __snake_case					:    Any="gelu"		,     __snake_case					:    str=10_24		,     __snake_case					:    Dict=0.1		,     __snake_case					:    List[Any]=0.0		,     __snake_case					:    str=0.0		,     __snake_case					:    str=0.02		,     __snake_case					:    Union[str, Any]=0		,     __snake_case					:    List[Any]=False		,     __snake_case					:    Optional[Any]=0		,     __snake_case					:    str=1		,     __snake_case					:    Any=1		,     **__snake_case					:    int		,     ):
      a		:   Optional[int]	  =	vocab_size
      a		:   List[str]	  =	max_position_embeddings
      a		:   List[Any]	  =	d_model
      a		:   Tuple	  =	encoder_ffn_dim
      a		:   List[str]	  =	encoder_layers
      a		:   str	  =	encoder_attention_heads
      a		:   str	  =	decoder_ffn_dim
      a		:   Optional[int]	  =	decoder_layers
      a		:   Optional[int]	  =	decoder_attention_heads
      a		:   Optional[int]	  =	dropout
      a		:   Any	  =	attention_dropout
      a		:   str	  =	activation_dropout
      a		:   List[Any]	  =	activation_function
      a		:   Union[str, Any]	  =	init_std
      a		:   List[Any]	  =	encoder_layerdrop
      a		:   Union[str, Any]	  =	decoder_layerdrop
      a		:   List[Any]	  =	use_cache
      a		:   Any	  =	encoder_layers
      a		:   Optional[int]	  =	scale_embedding  # scale factor will be sqrt(d_model) if True
      super().__init__(
          pad_token_id=__snake_case		,     eos_token_id=__snake_case		,     is_encoder_decoder=__snake_case		,     decoder_start_token_id=__snake_case		,     forced_eos_token_id=__snake_case		,     **__snake_case		,     )







    @property
    def      lowercase_			( self					:    int      ):
      return self.encoder_attention_heads







    @property
    def      lowercase_			( self					:    int      ):
      return self.d_model