



'''simple docstring'''





import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation


lowercase_	 =						logging.get_logger(__name__)

lowercase_	 =						{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowercase_	 =						{
    '''vocab_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/vocab.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/vocab.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/vocab.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/vocab.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/merges.txt''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/merges.txt''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/merges.txt''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/merges.txt''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/tokenizer.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/tokenizer.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/tokenizer.json''',
    },
}

lowercase_	 =						{
    '''gpt2''': 1_024,
    '''gpt2-medium''': 1_024,
    '''gpt2-large''': 1_024,
    '''gpt2-xl''': 1_024,
    '''distilgpt2''': 1_024,
}






class __A    (						UpperCamelCase__		):



	'''simple docstring'''







	__lowerCamelCase					: Dict    = VOCAB_FILES_NAMES
	__lowerCamelCase					: Any    = PRETRAINED_VOCAB_FILES_MAP
	__lowerCamelCase					: Tuple    = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	__lowerCamelCase					: Optional[Any]    = ["""input_ids""", """attention_mask"""]
	__lowerCamelCase					: Union[str, Any]    = GPTaTokenizer



	def __init__(self				,     A=None				,     A=None				,     A=None				,     A="<|endoftext|>"				,     A="<|endoftext|>"				,     A="<|endoftext|>"				,     A=False				,     **A				,     )					->  int:


						"""simple docstring"""



						super().__init__(
						    A				,     A				,     tokenizer_file=A				,     unk_token=A				,     bos_token=A				,     eos_token=A				,     add_prefix_space=A				,     **A				,     )

						_a											=						kwargs.pop('''add_bos_token'''				,     A )

						_a											=						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__() )
						if pre_tok_state.get('''add_prefix_space'''				,     A ) != add_prefix_space:
											_a											=						getattr(A				,     pre_tok_state.pop('''type''' ) )
											_a											=						add_prefix_space
											_a											=						pre_tok_class(**A )

						_a											=						add_prefix_space



	def       a__						(self				,     *A				,     **A )					->  BatchEncoding:


						"""simple docstring"""



						_a											=						kwargs.get('''is_split_into_words'''				,     A )
						assert self.add_prefix_space or not is_split_into_words, (
						    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
						    "to use it with pretokenized inputs."
						)

						return super()._batch_encode_plus(*A				,     **A )



	def       a__						(self				,     *A				,     **A )					->  BatchEncoding:


						"""simple docstring"""



						_a											=						kwargs.get('''is_split_into_words'''				,     A )

						assert self.add_prefix_space or not is_split_into_words, (
						    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
						    "to use it with pretokenized inputs."
						)

						return super()._encode_plus(*A				,     **A )



	def       a__						(self				,     A				,     A = None )					->  Tuple[str]:


						"""simple docstring"""



						_a											=						self._tokenizer.model.save(A				,     name=A )
						return tuple(A )



	def       a__						(self				,     A )					->  List[int]:


						"""simple docstring"""



						_a											=						[]
						for is_user, text in conversation.iter_texts():
											input_ids.extend(self.encode(A				,     add_special_tokens=A ) + [self.eos_token_id] )

						if len(A ) > self.model_max_length:
											_a											=						input_ids[-self.model_max_length :]
						return input_ids








'''simple docstring'''





from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase_	 =						logging.get_logger(__name__)

lowercase_	 =						{
    "google/pegasus-large": "https://huggingface.co/google/pegasus-large/resolve/main/config.json",
    # See all PEGASUS models at https://huggingface.co/models?filter=pegasus
}






class __A    (						A		):



 '''simple docstring'''







 __lowerCamelCase					: Union[str, Any]    = 'pegasus'
 __lowerCamelCase					: Tuple    = ['past_key_values']
 __lowerCamelCase					: List[Any]    = {'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}



 def __init__(self				,     A=50_265				,     A=1_024				,     A=12				,     A=4_096				,     A=16				,     A=12				,     A=4_096				,     A=16				,     A=0.0				,     A=0.0				,     A=True				,     A=True				,     A="gelu"				,     A=1_024				,     A=0.1				,     A=0.0				,     A=0.0				,     A=0.02				,     A=0				,     A=False				,     A=0				,     A=1				,     A=1				,     **A				,     )					->  str:


      """simple docstring"""



      _a											=						vocab_size
      _a											=						max_position_embeddings
      _a											=						d_model
      _a											=						encoder_ffn_dim
      _a											=						encoder_layers
      _a											=						encoder_attention_heads
      _a											=						decoder_ffn_dim
      _a											=						decoder_layers
      _a											=						decoder_attention_heads
      _a											=						dropout
      _a											=						attention_dropout
      _a											=						activation_dropout
      _a											=						activation_function
      _a											=						init_std
      _a											=						encoder_layerdrop
      _a											=						decoder_layerdrop
      _a											=						use_cache
      _a											=						encoder_layers
      _a											=						scale_embedding  # scale factor will be sqrt(d_model) if True
      super().__init__(
          pad_token_id=A				,     eos_token_id=A				,     is_encoder_decoder=A				,     decoder_start_token_id=A				,     forced_eos_token_id=A				,     **A				,     )



 @property
 def       a__						(self )					->  int:


      """simple docstring"""



      return self.encoder_attention_heads



 @property
 def       a__						(self )					->  int:


      """simple docstring"""



      return self.d_model



