






import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation


UpperCAmelCase__							  =						logging.get_logger(__name__)

UpperCAmelCase__							  =						{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase__							  =						{
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/tokenizer.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/tokenizer.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/tokenizer.json",
    },
}

UpperCAmelCase__							  =						{
    "gpt2": 1024,
    "gpt2-medium": 1024,
    "gpt2-large": 1024,
    "gpt2-xl": 1024,
    "distilgpt2": 1024,
}






class 						lowercase_   (      lowercase  ):






      '''simple docstring'''




      __snake_case	    =    VOCAB_FILES_NAMES
      __snake_case	    =    PRETRAINED_VOCAB_FILES_MAP
      __snake_case	    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __snake_case	    =    ['''input_ids''', '''attention_mask''']
      __snake_case	    =    GPTaTokenizer







      def __init__( self :		Optional[int]					,   __UpperCAmelCase :		Dict=None					,   __UpperCAmelCase :		List[Any]=None					,   __UpperCAmelCase :		Any=None					,   __UpperCAmelCase :		List[str]="<|endoftext|>"					,   __UpperCAmelCase :		Union[str, Any]="<|endoftext|>"					,   __UpperCAmelCase :		Optional[int]="<|endoftext|>"					,   __UpperCAmelCase :		str=False					,   **__UpperCAmelCase :		Dict					,   )       ->str:




       """simple docstring"""


       super().__init__(
           __UpperCAmelCase					,   __UpperCAmelCase					,   tokenizer_file=__UpperCAmelCase					,   unk_token=__UpperCAmelCase					,   bos_token=__UpperCAmelCase					,   eos_token=__UpperCAmelCase					,   add_prefix_space=__UpperCAmelCase					,   **__UpperCAmelCase					,   )

       a						    = kwargs.pop('''add_bos_token'''					,   __UpperCAmelCase    )

       a						    = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()    )
       if pre_tok_state.get('''add_prefix_space'''					,   __UpperCAmelCase    ) != add_prefix_space:
        a						    = getattr(__UpperCAmelCase					,   pre_tok_state.pop('''type'''    )    )
        a						    = add_prefix_space
        a						    = pre_tok_class(**__UpperCAmelCase    )

       a						    = add_prefix_space







      def  __lowerCAmelCase   ( self :		List[str]					,   *__UpperCAmelCase :		Dict					,   **__UpperCAmelCase :		int    )       ->BatchEncoding:




       """simple docstring"""


       a						    = kwargs.get('''is_split_into_words'''					,   __UpperCAmelCase    )
       assert self.add_prefix_space or not is_split_into_words, (
           F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
           "to use it with pretokenized inputs."
       )

       return super()._batch_encode_plus(*__UpperCAmelCase					,   **__UpperCAmelCase    )







      def  __lowerCAmelCase   ( self :		List[Any]					,   *__UpperCAmelCase :		Optional[Any]					,   **__UpperCAmelCase :		Tuple    )       ->BatchEncoding:




       """simple docstring"""


       a						    = kwargs.get('''is_split_into_words'''					,   __UpperCAmelCase    )

       assert self.add_prefix_space or not is_split_into_words, (
           F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
           "to use it with pretokenized inputs."
       )

       return super()._encode_plus(*__UpperCAmelCase					,   **__UpperCAmelCase    )







      def  __lowerCAmelCase   ( self :		Any					,   __UpperCAmelCase :		str					,   __UpperCAmelCase :		Optional[str] = None    )       ->Tuple[str]:




       """simple docstring"""


       a						    = self._tokenizer.model.save(__UpperCAmelCase					,   name=__UpperCAmelCase    )
       return tuple(__UpperCAmelCase    )







      def  __lowerCAmelCase   ( self :		str					,   __UpperCAmelCase :		"Conversation"    )       ->List[int]:




       """simple docstring"""


       a						    = []
       for is_user, text in conversation.iter_texts():
        input_ids.extend(self.encode(__UpperCAmelCase					,   add_special_tokens=__UpperCAmelCase    ) + [self.eos_token_id]    )

       if len(__UpperCAmelCase    ) > self.model_max_length:
        a						    = input_ids[-self.model_max_length :]
       return input_ids










import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
     from .tokenization_pegasus import PegasusTokenizer
else:
     UpperCAmelCase__							  =						None


UpperCAmelCase__							  =						logging.get_logger(__name__)


UpperCAmelCase__							  =						"▁"

UpperCAmelCase__							  =						{"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase__							  =						{
    "vocab_file": {"google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"},
    "tokenizer_file": {
        "google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json"
    },
}

UpperCAmelCase__							  =						{
    "google/pegasus-xsum": 512,
}






class 						lowercase_   (      lowercase  ):






      '''simple docstring'''




      __snake_case	    =    VOCAB_FILES_NAMES
      __snake_case	    =    PRETRAINED_VOCAB_FILES_MAP
      __snake_case	    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __snake_case	    =    PegasusTokenizer
      __snake_case	    =    ['''input_ids''', '''attention_mask''']







      def __init__( self :		List[Any]					,   __UpperCAmelCase :		str=None					,   __UpperCAmelCase :		List[Any]=None					,   __UpperCAmelCase :		Any="<pad>"					,   __UpperCAmelCase :		str="</s>"					,   __UpperCAmelCase :		List[str]="<unk>"					,   __UpperCAmelCase :		List[str]="<mask_2>"					,   __UpperCAmelCase :		Tuple="<mask_1>"					,   __UpperCAmelCase :		int=None					,   __UpperCAmelCase :		str=103					,   **__UpperCAmelCase :		Optional[int]					,   )       ->Dict:




       """simple docstring"""


       a						    = offset

       if additional_special_tokens is not None:
        if not isinstance(__UpperCAmelCase					,   __UpperCAmelCase    ):
         raise TypeError(
             F"""additional_special_tokens should be of type {type(__UpperCAmelCase    )}, but is"""
             F""" {type(__UpperCAmelCase    )}"""    )

        a						    = (
            ([mask_token_sent] + additional_special_tokens)
            if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
            else additional_special_tokens
        )
        # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
        additional_special_tokens_extended += [
            F"""<unk_{i}>""" for i in range(len(__UpperCAmelCase    )					,   self.offset - 1    )
        ]

        if len(set(__UpperCAmelCase    )    ) != len(__UpperCAmelCase    ):
         raise ValueError(
             '''Please make sure that the provided additional_special_tokens do not contain an incorrectly'''
             F""" shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}."""    )
        a						    = additional_special_tokens_extended
       else:
        a						    = [mask_token_sent] if mask_token_sent is not None else []
        additional_special_tokens += [F"""<unk_{i}>""" for i in range(2					,   self.offset    )]

       super().__init__(
           __UpperCAmelCase					,   tokenizer_file=__UpperCAmelCase					,   pad_token=__UpperCAmelCase					,   eos_token=__UpperCAmelCase					,   unk_token=__UpperCAmelCase					,   mask_token=__UpperCAmelCase					,   mask_token_sent=__UpperCAmelCase					,   offset=__UpperCAmelCase					,   additional_special_tokens=__UpperCAmelCase					,   **__UpperCAmelCase					,   )
       a						    = vocab_file
       a						    = False if not self.vocab_file else True







      def  __lowerCAmelCase   ( self :		Optional[Any]					,   __UpperCAmelCase :		int    )       ->int:




       """simple docstring"""


       a						    = set(self.all_special_ids    )  # call it once instead of inside list comp
       all_special_ids.remove(self.unk_token_id    )  # <unk> is only sometimes special

       if all_special_ids != set(range(len(self.additional_special_tokens    ) + 3    )    ):
        raise ValueError(
            '''There should be 3 special tokens: mask_token, pad_token, and eos_token +'''
            F""" {len(self.additional_special_tokens    )} additional_special_tokens, but got {all_special_ids}"""    )

       return [1 if x in all_special_ids else 0 for x in seq]







      def  __lowerCAmelCase   ( self :		Dict					,   __UpperCAmelCase :		List					,   __UpperCAmelCase :		Optional[List] = None					,   __UpperCAmelCase :		bool = False    )       ->List[int]:




       """simple docstring"""


       if already_has_special_tokens:
        return self._special_token_mask(__UpperCAmelCase    )
       elif token_ids_a is None:
        return self._special_token_mask(__UpperCAmelCase    ) + [1]
       else:
        return self._special_token_mask(token_ids_a + token_ids_a    ) + [1]







      def  __lowerCAmelCase   ( self :		Union[str, Any]					,   __UpperCAmelCase :		int					,   __UpperCAmelCase :		Optional[int]=None    )       ->List[int]:




       """simple docstring"""


       if token_ids_a is None:
        return token_ids_a + [self.eos_token_id]
       # We don't expect to process pairs, but leave the pair logic for API consistency
       return token_ids_a + token_ids_a + [self.eos_token_id]







      def  __lowerCAmelCase   ( self :		Dict					,   __UpperCAmelCase :		str					,   __UpperCAmelCase :		Optional[str] = None    )       ->Tuple[str]:




       """simple docstring"""


       if not self.can_save_slow_tokenizer:
        raise ValueError(
            '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
            '''tokenizer.'''    )

       if not os.path.isdir(__UpperCAmelCase    ):
        logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""    )
        return
       a						    = os.path.join(
           __UpperCAmelCase					,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )

       if os.path.abspath(self.vocab_file    ) != os.path.abspath(__UpperCAmelCase    ):
        copyfile(self.vocab_file					,   __UpperCAmelCase    )

       return (out_vocab_file,)


