


import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowerCAmelCase			=							logging.get_logger(__name__)


__lowerCAmelCase			=							{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

# See all BART models at https://huggingface.co/models?filter=bart
__lowerCAmelCase			=							{
    '''vocab_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/vocab.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/vocab.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/merges.txt''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/merges.txt''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt''',
    },
}

__lowerCAmelCase			=							{
    '''facebook/bart-base''': 10_24,
    '''facebook/bart-large''': 10_24,
    '''facebook/bart-large-mnli''': 10_24,
    '''facebook/bart-large-cnn''': 10_24,
    '''facebook/bart-large-xsum''': 10_24,
    '''yjernite/bart_eli5''': 10_24,
}




@lru_cache()
def        snake_case_					(  )  ->    Optional[int]:
       lowercase__:       Any        =       (
           list(range(ord('!'					)      ,							ord('~'					) + 1					)					) + list(range(ord('¡'					)      ,							ord('¬'					) + 1					)					) + list(range(ord('®'					)      ,							ord('ÿ'					) + 1					)					)
       )
       lowercase__:       List[Any]        =       bs[:]
       lowercase__:       Dict        =       0
       for b in range(2**8					):
              if b not in bs:
                     bs.append(snake_case					)
                     cs.append(2**8 + n					)
                     n += 1
       lowercase__:       List[str]        =       [chr(snake_case					) for n in cs]
       return dict(zip(snake_case      ,							snake_case					)					)




def        snake_case_					(  snake_case					)  ->    Dict:
       lowercase__:       int        =       set()
       lowercase__:       int        =       word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)					)
              lowercase__:       Union[str, Any]        =       char
       return pairs





class   __a      (   __UpperCamelCase				):
 __lowercase			:							str       					=			VOCAB_FILES_NAMES
 __lowercase			:							Optional[Any]       					=			PRETRAINED_VOCAB_FILES_MAP
 __lowercase			:							int       					=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowercase			:							Tuple       					=			['input_ids', 'attention_mask']
 def __init__(       self    ,			lowerCAmelCase__    ,			lowerCAmelCase__    ,			lowerCAmelCase__="replace"    ,			lowerCAmelCase__="<s>"    ,			lowerCAmelCase__="</s>"    ,			lowerCAmelCase__="</s>"    ,			lowerCAmelCase__="<s>"    ,			lowerCAmelCase__="<unk>"    ,			lowerCAmelCase__="<pad>"    ,			lowerCAmelCase__="<mask>"    ,			lowerCAmelCase__=False    ,			**lowerCAmelCase__    ,			) ->    Any:




        '''simple docstring'''
        lowercase__:       int        =       AddedToken(lowerCAmelCase__    ,			lstrip=lowerCAmelCase__    ,			rstrip=lowerCAmelCase__			) if isinstance(lowerCAmelCase__    ,			lowerCAmelCase__			) else bos_token
        lowercase__:       Tuple        =       AddedToken(lowerCAmelCase__    ,			lstrip=lowerCAmelCase__    ,			rstrip=lowerCAmelCase__			) if isinstance(lowerCAmelCase__    ,			lowerCAmelCase__			) else eos_token
        lowercase__:       List[Any]        =       AddedToken(lowerCAmelCase__    ,			lstrip=lowerCAmelCase__    ,			rstrip=lowerCAmelCase__			) if isinstance(lowerCAmelCase__    ,			lowerCAmelCase__			) else sep_token
        lowercase__:       List[str]        =       AddedToken(lowerCAmelCase__    ,			lstrip=lowerCAmelCase__    ,			rstrip=lowerCAmelCase__			) if isinstance(lowerCAmelCase__    ,			lowerCAmelCase__			) else cls_token
        lowercase__:       Optional[Any]        =       AddedToken(lowerCAmelCase__    ,			lstrip=lowerCAmelCase__    ,			rstrip=lowerCAmelCase__			) if isinstance(lowerCAmelCase__    ,			lowerCAmelCase__			) else unk_token
        lowercase__:       List[str]        =       AddedToken(lowerCAmelCase__    ,			lstrip=lowerCAmelCase__    ,			rstrip=lowerCAmelCase__			) if isinstance(lowerCAmelCase__    ,			lowerCAmelCase__			) else pad_token

        # Mask token behave like a normal word, i.e. include the space before it
        lowercase__:       List[Any]        =       AddedToken(lowerCAmelCase__    ,			lstrip=lowerCAmelCase__    ,			rstrip=lowerCAmelCase__			) if isinstance(lowerCAmelCase__    ,			lowerCAmelCase__			) else mask_token

        super().__init__(
            errors=lowerCAmelCase__    ,			bos_token=lowerCAmelCase__    ,			eos_token=lowerCAmelCase__    ,			unk_token=lowerCAmelCase__    ,			sep_token=lowerCAmelCase__    ,			cls_token=lowerCAmelCase__    ,			pad_token=lowerCAmelCase__    ,			mask_token=lowerCAmelCase__    ,			add_prefix_space=lowerCAmelCase__    ,			**lowerCAmelCase__    ,			)

        with open(lowerCAmelCase__    ,			encoding='utf-8'			) as vocab_handle:
               lowercase__:       List[str]        =       json.load(lowerCAmelCase__			)
        lowercase__:       int        =       {v: k for k, v in self.encoder.items()}
        lowercase__:       int        =       errors  # how to handle errors in decoding
        lowercase__:       List[Any]        =       bytes_to_unicode()
        lowercase__:       str        =       {v: k for k, v in self.byte_encoder.items()}
        with open(lowerCAmelCase__    ,			encoding='utf-8'			) as merges_handle:
               lowercase__:       List[str]        =       merges_handle.read().split('\n'			)[1:-1]
        lowercase__:       Tuple        =       [tuple(merge.split()			) for merge in bpe_merges]
        lowercase__:       Tuple        =       dict(zip(lowerCAmelCase__    ,			range(len(lowerCAmelCase__			)			)			)			)
        lowercase__:       Dict        =       {}
        lowercase__:       Tuple        =       add_prefix_space

        # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
        lowercase__:       Tuple        =       re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'			)
 @property
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    Optional[Any]:




        '''simple docstring'''
        return len(self.encoder			)
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    Optional[Any]:




        '''simple docstring'''
        return dict(self.encoder    ,			**self.added_tokens_encoder			)
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__			) ->    Optional[int]:




        '''simple docstring'''
        if token in self.cache:
               return self.cache[token]
        lowercase__:       str        =       tuple(lowerCAmelCase__			)
        lowercase__:       List[str]        =       get_pairs(lowerCAmelCase__			)

        if not pairs:
               return token

        while True:
               lowercase__:       Tuple        =       min(lowerCAmelCase__    ,			key=lambda lowerCAmelCase__			: self.bpe_ranks.get(lowerCAmelCase__    ,			float('inf'			)			)			)
               if bigram not in self.bpe_ranks:
                      break
               lowercase__      ,						lowercase__:       Optional[int]        =       bigram
               lowercase__:       Union[str, Any]        =       []
               lowercase__:       Any        =       0
               while i < len(lowerCAmelCase__			):
                      try:
                             lowercase__:       List[str]        =       word.index(lowerCAmelCase__    ,			lowerCAmelCase__			)
                      except ValueError:
                             new_word.extend(word[i:]			)
                             break
                      else:
                             new_word.extend(word[i:j]			)
                             lowercase__:       Union[str, Any]        =       j

                      if word[i] == first and i < len(lowerCAmelCase__			) - 1 and word[i + 1] == second:
                             new_word.append(first + second			)
                             i += 2
                      else:
                             new_word.append(word[i]			)
                             i += 1
               lowercase__:       Optional[Any]        =       tuple(lowerCAmelCase__			)
               lowercase__:       Any        =       new_word
               if len(lowerCAmelCase__			) == 1:
                      break
               else:
                      lowercase__:       List[str]        =       get_pairs(lowerCAmelCase__			)
        lowercase__:       int        =       ' '.join(lowerCAmelCase__			)
        lowercase__:       int        =       word
        return word
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__			) ->    List[str]:




        '''simple docstring'''
        lowercase__:       Dict        =       []
        for token in re.findall(self.pat    ,			lowerCAmelCase__			):
               lowercase__:       Optional[int]        =       ''.join(
                   self.byte_encoder[b] for b in token.encode('utf-8'			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
               bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCAmelCase__			).split(' '			)			)
        return bpe_tokens
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__			) ->    Tuple:




        '''simple docstring'''
        return self.encoder.get(lowerCAmelCase__    ,			self.encoder.get(self.unk_token			)			)
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__			) ->    Tuple:




        '''simple docstring'''
        return self.decoder.get(lowerCAmelCase__			)
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__			) ->    int:




        '''simple docstring'''
        lowercase__:       Tuple        =       ''.join(lowerCAmelCase__			)
        lowercase__:       Optional[int]        =       bytearray([self.byte_decoder[c] for c in text]			).decode('utf-8'    ,			errors=self.errors			)
        return text
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__    ,			lowerCAmelCase__ = None			) ->    Tuple[str]:




        '''simple docstring'''
        if not os.path.isdir(lowerCAmelCase__			):
               logger.error(F'Vocabulary path ({save_directory}) should be a directory'			)
               return
        lowercase__:       Dict        =       os.path.join(
            lowerCAmelCase__    ,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)
        lowercase__:       Optional[int]        =       os.path.join(
            lowerCAmelCase__    ,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']			)

        with open(lowerCAmelCase__    ,			'w'    ,			encoding='utf-8'			) as f:
               f.write(json.dumps(self.encoder    ,			indent=2    ,			sort_keys=lowerCAmelCase__    ,			ensure_ascii=lowerCAmelCase__			) + '\n'			)

        lowercase__:       List[str]        =       0
        with open(lowerCAmelCase__    ,			'w'    ,			encoding='utf-8'			) as writer:
               writer.write('#version: 0.2\n'			)
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,			key=lambda lowerCAmelCase__			: kv[1]			):
                      if index != token_index:
                             logger.warning(
                                 F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                                 ' Please check that the tokenizer is not corrupted!'			)
                             lowercase__:       str        =       token_index
                      writer.write(' '.join(lowerCAmelCase__			) + '\n'			)
                      index += 1

        return vocab_file, merge_file
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__    ,			lowerCAmelCase__ = None			) ->    List[int]:




        '''simple docstring'''
        if token_ids_a is None:
               return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        lowercase__:       Dict        =       [self.cls_token_id]
        lowercase__:       Optional[int]        =       [self.sep_token_id]
        return cls + token_ids_a + sep + sep + token_ids_a + sep
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__    ,			lowerCAmelCase__ = None    ,			lowerCAmelCase__ = False			) ->    List[int]:




        '''simple docstring'''
        if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=lowerCAmelCase__    ,			token_ids_a=lowerCAmelCase__    ,			already_has_special_tokens=lowerCAmelCase__			)

        if token_ids_a is None:
               return [1] + ([0] * len(lowerCAmelCase__			)) + [1]
        return [1] + ([0] * len(lowerCAmelCase__			)) + [1, 1] + ([0] * len(lowerCAmelCase__			)) + [1]
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__    ,			lowerCAmelCase__ = None			) ->    List[int]:




        '''simple docstring'''
        lowercase__:       Any        =       [self.sep_token_id]
        lowercase__:       Optional[Any]        =       [self.cls_token_id]

        if token_ids_a is None:
               return len(cls + token_ids_a + sep			) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__    ,			lowerCAmelCase__=False    ,			**lowerCAmelCase__			) ->    Optional[Any]:




        '''simple docstring'''
        lowercase__:       List[Any]        =       kwargs.pop('add_prefix_space'    ,			self.add_prefix_space			)
        if (is_split_into_words or add_prefix_space) and (len(lowerCAmelCase__			) > 0 and not text[0].isspace()):
               lowercase__:       Dict        =       ' ' + text
        return (text, kwargs)






from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCAmelCase			=							logging.get_logger(__name__)

__lowerCAmelCase			=							{
    '''google/pegasus-large''': '''https://huggingface.co/google/pegasus-large/resolve/main/config.json''',
    # See all PEGASUS models at https://huggingface.co/models?filter=pegasus
}





class   __a      (   __UpperCamelCase				):
 __lowercase			:							Any       					=			'pegasus'
 __lowercase			:							Union[str, Any]       					=			['past_key_values']
 __lowercase			:							Any       					=			{'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}
 def __init__(       self    ,			lowerCAmelCase__=50_265    ,			lowerCAmelCase__=1_024    ,			lowerCAmelCase__=12    ,			lowerCAmelCase__=4_096    ,			lowerCAmelCase__=16    ,			lowerCAmelCase__=12    ,			lowerCAmelCase__=4_096    ,			lowerCAmelCase__=16    ,			lowerCAmelCase__=0.0    ,			lowerCAmelCase__=0.0    ,			lowerCAmelCase__=True    ,			lowerCAmelCase__=True    ,			lowerCAmelCase__="gelu"    ,			lowerCAmelCase__=1_024    ,			lowerCAmelCase__=0.1    ,			lowerCAmelCase__=0.0    ,			lowerCAmelCase__=0.0    ,			lowerCAmelCase__=0.0_2    ,			lowerCAmelCase__=0    ,			lowerCAmelCase__=False    ,			lowerCAmelCase__=0    ,			lowerCAmelCase__=1    ,			lowerCAmelCase__=1    ,			**lowerCAmelCase__    ,			) ->    Union[str, Any]:




        '''simple docstring'''
        lowercase__:       int        =       vocab_size
        lowercase__:       Optional[int]        =       max_position_embeddings
        lowercase__:       List[str]        =       d_model
        lowercase__:       Optional[Any]        =       encoder_ffn_dim
        lowercase__:       Optional[Any]        =       encoder_layers
        lowercase__:       Union[str, Any]        =       encoder_attention_heads
        lowercase__:       Optional[int]        =       decoder_ffn_dim
        lowercase__:       Tuple        =       decoder_layers
        lowercase__:       Union[str, Any]        =       decoder_attention_heads
        lowercase__:       Dict        =       dropout
        lowercase__:       List[str]        =       attention_dropout
        lowercase__:       List[str]        =       activation_dropout
        lowercase__:       Optional[int]        =       activation_function
        lowercase__:       Dict        =       init_std
        lowercase__:       Optional[Any]        =       encoder_layerdrop
        lowercase__:       List[str]        =       decoder_layerdrop
        lowercase__:       Union[str, Any]        =       use_cache
        lowercase__:       Any        =       encoder_layers
        lowercase__:       List[str]        =       scale_embedding  # scale factor will be sqrt(d_model) if True
        super().__init__(
            pad_token_id=lowerCAmelCase__    ,			eos_token_id=lowerCAmelCase__    ,			is_encoder_decoder=lowerCAmelCase__    ,			decoder_start_token_id=lowerCAmelCase__    ,			forced_eos_token_id=lowerCAmelCase__    ,			**lowerCAmelCase__    ,			)
 @property
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    int:




        '''simple docstring'''
        return self.encoder_attention_heads
 @property
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    int:




        '''simple docstring'''
        return self.d_model


