


'''simple docstring'''







import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


lowerCAmelCase     :  Optional[int]											=    logging.get_logger(__name__)


lowerCAmelCase     :  List[Any]											=    {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all LED models at https://huggingface.co/models?filter=LED
lowerCAmelCase     :  List[str]											=    {
    """vocab_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json""",
    },
}

lowerCAmelCase     :  Optional[int]											=    {
    """allenai/led-base-16384""": 1_63_84,
}






@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def       lowercase					():







       """simple docstring"""







       _lowerCAmelCase     :				List[str]       =							(
           list(range(ord('!'       ) ,    ord('~'       ) + 1       )       ) + list(range(ord('¡'       ) ,    ord('¬'       ) + 1       )       ) + list(range(ord('®'       ) ,    ord('ÿ'       ) + 1       )       )
       )
       _lowerCAmelCase     :				str       =							bs[:]
       _lowerCAmelCase     :				List[Any]       =							0
       for b in range(2**8       ):
              if b not in bs:
                     bs.append(_A       )
                     cs.append(2**8 + n       )
                     n += 1
       _lowerCAmelCase     :				Tuple       =							[chr(_A       ) for n in cs]
       return dict(zip(_A ,    _A       )       )






def       lowercase					(_A       ):







       """simple docstring"""







       _lowerCAmelCase     :				str       =							set()
       _lowerCAmelCase     :				Dict       =							word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)       )
              _lowerCAmelCase     :				str       =							char
       return pairs
class 						UpperCamelCase__       (			SCREAMING_SNAKE_CASE_						):



     """simple docstring"""






     __magic_name__           =     VOCAB_FILES_NAMES
     __magic_name__           =     PRETRAINED_VOCAB_FILES_MAP
     __magic_name__           =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __magic_name__           =     ["input_ids", "attention_mask"]







     def __init__(			self   ,						snake_case__   ,						snake_case__   ,						snake_case__="replace"   ,						snake_case__="<s>"   ,						snake_case__="</s>"   ,						snake_case__="</s>"   ,						snake_case__="<s>"   ,						snake_case__="<unk>"   ,						snake_case__="<pad>"   ,						snake_case__="<mask>"   ,						snake_case__=False   ,						**snake_case__   ,						):

            '''simple docstring'''

            _lowerCAmelCase     :				List[Any]       =							AddedToken(snake_case__   ,						lstrip=snake_case__   ,						rstrip=snake_case__ ) if isinstance(snake_case__   ,						snake_case__ ) else bos_token
            _lowerCAmelCase     :				str       =							AddedToken(snake_case__   ,						lstrip=snake_case__   ,						rstrip=snake_case__ ) if isinstance(snake_case__   ,						snake_case__ ) else eos_token
            _lowerCAmelCase     :				Optional[Any]       =							AddedToken(snake_case__   ,						lstrip=snake_case__   ,						rstrip=snake_case__ ) if isinstance(snake_case__   ,						snake_case__ ) else sep_token
            _lowerCAmelCase     :				Union[str, Any]       =							AddedToken(snake_case__   ,						lstrip=snake_case__   ,						rstrip=snake_case__ ) if isinstance(snake_case__   ,						snake_case__ ) else cls_token
            _lowerCAmelCase     :				Optional[Any]       =							AddedToken(snake_case__   ,						lstrip=snake_case__   ,						rstrip=snake_case__ ) if isinstance(snake_case__   ,						snake_case__ ) else unk_token
            _lowerCAmelCase     :				Any       =							AddedToken(snake_case__   ,						lstrip=snake_case__   ,						rstrip=snake_case__ ) if isinstance(snake_case__   ,						snake_case__ ) else pad_token

            # Mask token behave like a normal word, i.e. include the space before it
            _lowerCAmelCase     :				str       =							AddedToken(snake_case__   ,						lstrip=snake_case__   ,						rstrip=snake_case__ ) if isinstance(snake_case__   ,						snake_case__ ) else mask_token

            super().__init__(
                errors=snake_case__   ,						bos_token=snake_case__   ,						eos_token=snake_case__   ,						unk_token=snake_case__   ,						sep_token=snake_case__   ,						cls_token=snake_case__   ,						pad_token=snake_case__   ,						mask_token=snake_case__   ,						add_prefix_space=snake_case__   ,						**snake_case__   ,						)

            with open(snake_case__   ,						encoding='utf-8' ) as vocab_handle:
                   _lowerCAmelCase     :				List[str]       =							json.load(snake_case__ )
            _lowerCAmelCase     :				Union[str, Any]       =							{v: k for k, v in self.encoder.items()}
            _lowerCAmelCase     :				str       =							errors  # how to handle errors in decoding
            _lowerCAmelCase     :				str       =							bytes_to_unicode()
            _lowerCAmelCase     :				Union[str, Any]       =							{v: k for k, v in self.byte_encoder.items()}
            with open(snake_case__   ,						encoding='utf-8' ) as merges_handle:
                   _lowerCAmelCase     :				List[str]       =							merges_handle.read().split('\n' )[1:-1]
            _lowerCAmelCase     :				List[Any]       =							[tuple(merge.split() ) for merge in bpe_merges]
            _lowerCAmelCase     :				Union[str, Any]       =							dict(zip(snake_case__   ,						range(len(snake_case__ ) ) ) )
            _lowerCAmelCase     :				List[str]       =							{}
            _lowerCAmelCase     :				Any       =							add_prefix_space

            # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
            _lowerCAmelCase     :				Dict       =							re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+' )







     @property
     # Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
     def a						(			self ):

            '''simple docstring'''

            return len(self.encoder )







     def a						(			self ):

            '''simple docstring'''

            return dict(self.encoder   ,						**self.added_tokens_encoder )







     def a						(			self   ,						snake_case__ ):

            '''simple docstring'''

            if token in self.cache:
                   return self.cache[token]
            _lowerCAmelCase     :				Optional[int]       =							tuple(snake_case__ )
            _lowerCAmelCase     :				Any       =							get_pairs(snake_case__ )

            if not pairs:
                   return token

            while True:
                   _lowerCAmelCase     :				List[Any]       =							min(snake_case__   ,						key=lambda snake_case__ : self.bpe_ranks.get(snake_case__   ,						float('inf' ) ) )
                   if bigram not in self.bpe_ranks:
                          break
                   _lowerCAmelCase      ,  _lowerCAmelCase     :				Optional[int]       =							bigram
                   _lowerCAmelCase     :				List[str]       =							[]
                   _lowerCAmelCase     :				Union[str, Any]       =							0
                   while i < len(snake_case__ ):
                          try:
                                 _lowerCAmelCase     :				Optional[int]       =							word.index(snake_case__   ,						snake_case__ )
                          except ValueError:
                                 new_word.extend(word[i:] )
                                 break
                          else:
                                 new_word.extend(word[i:j] )
                                 _lowerCAmelCase     :				str       =							j

                          if word[i] == first and i < len(snake_case__ ) - 1 and word[i + 1] == second:
                                 new_word.append(first + second )
                                 i += 2
                          else:
                                 new_word.append(word[i] )
                                 i += 1
                   _lowerCAmelCase     :				Optional[Any]       =							tuple(snake_case__ )
                   _lowerCAmelCase     :				List[str]       =							new_word
                   if len(snake_case__ ) == 1:
                          break
                   else:
                          _lowerCAmelCase     :				Any       =							get_pairs(snake_case__ )
            _lowerCAmelCase     :				Any       =							' '.join(snake_case__ )
            _lowerCAmelCase     :				Optional[int]       =							word
            return word







     def a						(			self   ,						snake_case__ ):

            '''simple docstring'''

            _lowerCAmelCase     :				str       =							[]
            for token in re.findall(self.pat   ,						snake_case__ ):
                   _lowerCAmelCase     :				int       =							''.join(
                       self.byte_encoder[b] for b in token.encode('utf-8' ) )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                   bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case__ ).split(' ' ) )
            return bpe_tokens







     def a						(			self   ,						snake_case__ ):

            '''simple docstring'''

            return self.encoder.get(snake_case__   ,						self.encoder.get(self.unk_token ) )







     def a						(			self   ,						snake_case__ ):

            '''simple docstring'''

            return self.decoder.get(snake_case__ )







     def a						(			self   ,						snake_case__ ):

            '''simple docstring'''

            _lowerCAmelCase     :				Optional[int]       =							''.join(snake_case__ )
            _lowerCAmelCase     :				Union[str, Any]       =							bytearray([self.byte_decoder[c] for c in text] ).decode('utf-8'   ,						errors=self.errors )
            return text







     def a						(			self   ,						snake_case__   ,						snake_case__ = None ):

            '''simple docstring'''

            if not os.path.isdir(snake_case__ ):
                   logger.error(F'Vocabulary path ({save_directory}) should be a directory' )
                   return
            _lowerCAmelCase     :				Dict       =							os.path.join(
                snake_case__   ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] )
            _lowerCAmelCase     :				List[str]       =							os.path.join(
                snake_case__   ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'] )

            with open(snake_case__   ,						'w'   ,						encoding='utf-8' ) as f:
                   f.write(json.dumps(self.encoder   ,						indent=2   ,						sort_keys=snake_case__   ,						ensure_ascii=snake_case__ ) + '\n' )

            _lowerCAmelCase     :				List[Any]       =							0
            with open(snake_case__   ,						'w'   ,						encoding='utf-8' ) as writer:
                   writer.write('#version: 0.2\n' )
                   for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,						key=lambda snake_case__ : kv[1] ):
                          if index != token_index:
                                 logger.warning(
                                     F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                                     ' Please check that the tokenizer is not corrupted!' )
                                 _lowerCAmelCase     :				Union[str, Any]       =							token_index
                          writer.write(' '.join(snake_case__ ) + '\n' )
                          index += 1

            return vocab_file, merge_file







     def a						(			self   ,						snake_case__   ,						snake_case__ = None ):

            '''simple docstring'''

            if token_ids_a is None:
                   return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
            _lowerCAmelCase     :				List[str]       =							[self.cls_token_id]
            _lowerCAmelCase     :				Dict       =							[self.sep_token_id]
            return cls + token_ids_a + sep + sep + token_ids_a + sep







     def a						(			self   ,						snake_case__   ,						snake_case__ = None   ,						snake_case__ = False ):

            '''simple docstring'''

            if already_has_special_tokens:
                   return super().get_special_tokens_mask(
                       token_ids_a=snake_case__   ,						token_ids_a=snake_case__   ,						already_has_special_tokens=snake_case__ )

            if token_ids_a is None:
                   return [1] + ([0] * len(snake_case__ )) + [1]
            return [1] + ([0] * len(snake_case__ )) + [1, 1] + ([0] * len(snake_case__ )) + [1]







     def a						(			self   ,						snake_case__   ,						snake_case__ = None ):

            '''simple docstring'''

            _lowerCAmelCase     :				Tuple       =							[self.sep_token_id]
            _lowerCAmelCase     :				List[str]       =							[self.cls_token_id]

            if token_ids_a is None:
                   return len(cls + token_ids_a + sep ) * [0]
            return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]







     def a						(			self   ,						snake_case__   ,						snake_case__=False   ,						**snake_case__ ):

            '''simple docstring'''

            _lowerCAmelCase     :				str       =							kwargs.pop('add_prefix_space'   ,						self.add_prefix_space )
            if (is_split_into_words or add_prefix_space) and (len(snake_case__ ) > 0 and not text[0].isspace()):
                   _lowerCAmelCase     :				Any       =							' ' + text
            return (text, kwargs)







     def a						(			self   ,						snake_case__   ,						snake_case__ = None   ,						snake_case__ = PaddingStrategy.DO_NOT_PAD   ,						snake_case__ = None   ,						snake_case__ = None   ,						):

            '''simple docstring'''

            _lowerCAmelCase     :				Dict       =							super()._pad(
                encoded_inputs=snake_case__   ,						max_length=snake_case__   ,						padding_strategy=snake_case__   ,						pad_to_multiple_of=snake_case__   ,						return_attention_mask=snake_case__   ,						)

            # Load from model defaults
            if return_attention_mask is None:
                   _lowerCAmelCase     :				List[Any]       =							'attention_mask' in self.model_input_names

            if return_attention_mask and "global_attention_mask" in encoded_inputs:
                   _lowerCAmelCase     :				List[Any]       =							encoded_inputs[self.model_input_names[0]]
                   # `global_attention_mask` need to have the same length as other (sequential) inputs.
                   _lowerCAmelCase     :				Optional[Any]       =							len(encoded_inputs['global_attention_mask'] ) != len(snake_case__ )

                   if needs_to_be_padded:
                          _lowerCAmelCase     :				Tuple       =							len(snake_case__ ) - len(encoded_inputs['global_attention_mask'] )

                          if self.padding_side == "right":
                                 # Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
                                 _lowerCAmelCase     :				Optional[Any]       =							(
                                     encoded_inputs['global_attention_mask'] + [-1] * difference
                                 )
                          elif self.padding_side == "left":
                                 _lowerCAmelCase     :				List[Any]       =							[-1] * difference + encoded_inputs[
                                     'global_attention_mask'
                                 ]
                          else:
                                 raise ValueError('Invalid padding strategy:' + str(self.padding_side ) )

            return encoded_inputs





'''simple docstring'''







import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
       from transformers.pipelines.conversational import Conversation


lowerCAmelCase     :  Tuple											=    logging.get_logger(__name__)

lowerCAmelCase     :  Union[str, Any]											=    {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase     :  str											=    {
    """vocab_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/vocab.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/vocab.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/vocab.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/vocab.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/merges.txt""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/merges.txt""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/merges.txt""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/merges.txt""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/tokenizer.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/tokenizer.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/tokenizer.json""",
    },
}

lowerCAmelCase     :  str											=    {
    """gpt2""": 10_24,
    """gpt2-medium""": 10_24,
    """gpt2-large""": 10_24,
    """gpt2-xl""": 10_24,
    """distilgpt2""": 10_24,
}
class 						UpperCamelCase__       (			SCREAMING_SNAKE_CASE_						):



     """simple docstring"""






     __magic_name__           =     VOCAB_FILES_NAMES
     __magic_name__           =     PRETRAINED_VOCAB_FILES_MAP
     __magic_name__           =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __magic_name__           =     ["input_ids", "attention_mask"]
     __magic_name__           =     GPTaTokenizer







     def __init__(			self   ,						snake_case__=None   ,						snake_case__=None   ,						snake_case__=None   ,						snake_case__="<|endoftext|>"   ,						snake_case__="<|endoftext|>"   ,						snake_case__="<|endoftext|>"   ,						snake_case__=False   ,						**snake_case__   ,						):

            '''simple docstring'''

            super().__init__(
                snake_case__   ,						snake_case__   ,						tokenizer_file=snake_case__   ,						unk_token=snake_case__   ,						bos_token=snake_case__   ,						eos_token=snake_case__   ,						add_prefix_space=snake_case__   ,						**snake_case__   ,						)

            _lowerCAmelCase     :				str       =							kwargs.pop('add_bos_token'   ,						snake_case__ )

            _lowerCAmelCase     :				List[Any]       =							json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__() )
            if pre_tok_state.get('add_prefix_space'   ,						snake_case__ ) != add_prefix_space:
                   _lowerCAmelCase     :				str       =							getattr(snake_case__   ,						pre_tok_state.pop('type' ) )
                   _lowerCAmelCase     :				List[str]       =							add_prefix_space
                   _lowerCAmelCase     :				Dict       =							pre_tok_class(**snake_case__ )

            _lowerCAmelCase     :				Any       =							add_prefix_space







     def a						(			self   ,						*snake_case__   ,						**snake_case__ ):

            '''simple docstring'''

            _lowerCAmelCase     :				int       =							kwargs.get('is_split_into_words'   ,						snake_case__ )
            assert self.add_prefix_space or not is_split_into_words, (
                F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
                "to use it with pretokenized inputs."
            )

            return super()._batch_encode_plus(*snake_case__   ,						**snake_case__ )







     def a						(			self   ,						*snake_case__   ,						**snake_case__ ):

            '''simple docstring'''

            _lowerCAmelCase     :				int       =							kwargs.get('is_split_into_words'   ,						snake_case__ )

            assert self.add_prefix_space or not is_split_into_words, (
                F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
                "to use it with pretokenized inputs."
            )

            return super()._encode_plus(*snake_case__   ,						**snake_case__ )







     def a						(			self   ,						snake_case__   ,						snake_case__ = None ):

            '''simple docstring'''

            _lowerCAmelCase     :				Optional[int]       =							self._tokenizer.model.save(snake_case__   ,						name=snake_case__ )
            return tuple(snake_case__ )







     def a						(			self   ,						snake_case__ ):

            '''simple docstring'''

            _lowerCAmelCase     :				Any       =							[]
            for is_user, text in conversation.iter_texts():
                   input_ids.extend(self.encode(snake_case__   ,						add_special_tokens=snake_case__ ) + [self.eos_token_id] )

            if len(snake_case__ ) > self.model_max_length:
                   _lowerCAmelCase     :				str       =							input_ids[-self.model_max_length :]
            return input_ids

