



'''simple docstring'''





import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


A__    :   List[str]           =					logging.get_logger(__name__)


A__    :   List[Any]           =					{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

# See all LED models at https://huggingface.co/models?filter=LED
A__    :   int           =					{
    '''vocab_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json''',
    },
}

A__    :   Tuple           =					{
    '''allenai/led-base-16384''': 1_6_3_8_4,
}





@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def   a_   (		)    ->  Optional[int]:
      __snake_case :   Optional[Any]													=    (
          list(range(ord('!'				) ,ord('~'				) + 1				)				) + list(range(ord('¡'				) ,ord('¬'				) + 1				)				) + list(range(ord('®'				) ,ord('ÿ'				) + 1				)				)
      )
      __snake_case :   Optional[int]													=    bs[:]
      __snake_case :   List[str]													=    0
      for b in range(2**8				):
            if b not in bs:
                  bs.append(_UpperCAmelCase				)
                  cs.append(2**8 + n				)
                  n += 1
      __snake_case :   List[Any]													=    [chr(_UpperCAmelCase				) for n in cs]
      return dict(zip(_UpperCAmelCase ,_UpperCAmelCase				)				)





def   a_   (		_UpperCAmelCase    :      Union[str, Any]				)    ->  Union[str, Any]:
      __snake_case :   int													=    set()
      __snake_case :   List[Any]													=    word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)				)
            __snake_case :   Optional[Any]													=    char
      return pairs





class  snake_case__						(    SCREAMING_SNAKE_CASE_				):
 A__				= VOCAB_FILES_NAMES
 A__				= PRETRAINED_VOCAB_FILES_MAP
 A__				= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 A__				= ['''input_ids''', '''attention_mask''']
 def __init__(   self						:							List[str]      ,     __a						:							Dict      ,     __a						:							List[str]      ,     __a						:							List[str]="replace"      ,     __a						:							Tuple="<s>"      ,     __a						:							Optional[Any]="</s>"      ,     __a						:							Tuple="</s>"      ,     __a						:							Union[str, Any]="<s>"      ,     __a						:							List[str]="<unk>"      ,     __a						:							Optional[Any]="<pad>"      ,     __a						:							Tuple="<mask>"      ,     __a						:							Optional[int]=False      ,     **__a						:							int      ,     )      ->   Optional[Any]:







       '''simple docstring'''
       __snake_case :   Tuple													=    AddedToken(__a      ,     lstrip=__a      ,     rstrip=__a  ) if isinstance(__a      ,     __a  ) else bos_token
       __snake_case :   Optional[int]													=    AddedToken(__a      ,     lstrip=__a      ,     rstrip=__a  ) if isinstance(__a      ,     __a  ) else eos_token
       __snake_case :   List[Any]													=    AddedToken(__a      ,     lstrip=__a      ,     rstrip=__a  ) if isinstance(__a      ,     __a  ) else sep_token
       __snake_case :   Optional[int]													=    AddedToken(__a      ,     lstrip=__a      ,     rstrip=__a  ) if isinstance(__a      ,     __a  ) else cls_token
       __snake_case :   Optional[int]													=    AddedToken(__a      ,     lstrip=__a      ,     rstrip=__a  ) if isinstance(__a      ,     __a  ) else unk_token
       __snake_case :   str													=    AddedToken(__a      ,     lstrip=__a      ,     rstrip=__a  ) if isinstance(__a      ,     __a  ) else pad_token

       # Mask token behave like a normal word, i.e. include the space before it
       __snake_case :   List[Any]													=    AddedToken(__a      ,     lstrip=__a      ,     rstrip=__a  ) if isinstance(__a      ,     __a  ) else mask_token

       super().__init__(
           errors=__a      ,     bos_token=__a      ,     eos_token=__a      ,     unk_token=__a      ,     sep_token=__a      ,     cls_token=__a      ,     pad_token=__a      ,     mask_token=__a      ,     add_prefix_space=__a      ,     **__a      ,     )

       with open(__a      ,     encoding='utf-8'  ) as vocab_handle:
             __snake_case :   Any													=    json.load(__a  )
       __snake_case :   Optional[Any]													=    {v: k for k, v in self.encoder.items()}
       __snake_case :   Tuple													=    errors  # how to handle errors in decoding
       __snake_case :   Tuple													=    bytes_to_unicode()
       __snake_case :   Tuple													=    {v: k for k, v in self.byte_encoder.items()}
       with open(__a      ,     encoding='utf-8'  ) as merges_handle:
             __snake_case :   Tuple													=    merges_handle.read().split('\n'  )[1:-1]
       __snake_case :   Tuple													=    [tuple(merge.split()  ) for merge in bpe_merges]
       __snake_case :   int													=    dict(zip(__a      ,     range(len(__a  )  )  )  )
       __snake_case :   Optional[Any]													=    {}
       __snake_case :   int													=    add_prefix_space

       # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
       __snake_case :   Union[str, Any]													=    re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'  )
 @property
 # Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
 def A_   (   self						:							Dict  )      ->   Dict:







       '''simple docstring'''
       return len(self.encoder  )
 def A_   (   self						:							List[Any]  )      ->   List[Any]:







       '''simple docstring'''
       return dict(self.encoder      ,     **self.added_tokens_encoder  )
 def A_   (   self						:							Tuple      ,     __a						:							Dict  )      ->   str:







       '''simple docstring'''
       if token in self.cache:
             return self.cache[token]
       __snake_case :   Any													=    tuple(__a  )
       __snake_case :   Union[str, Any]													=    get_pairs(__a  )

       if not pairs:
             return token

       while True:
             __snake_case :   List[Any]													=    min(__a      ,     key=lambda __a  : self.bpe_ranks.get(__a      ,     float('inf'  )  )  )
             if bigram not in self.bpe_ranks:
                   break
             __snake_case							,     __snake_case :   Union[str, Any]													=    bigram
             __snake_case :   int													=    []
             __snake_case :   Optional[Any]													=    0
             while i < len(__a  ):
                   try:
                         __snake_case :   Dict													=    word.index(__a      ,     __a  )
                   except ValueError:
                         new_word.extend(word[i:]  )
                         break
                   else:
                         new_word.extend(word[i:j]  )
                         __snake_case :   Optional[Any]													=    j

                   if word[i] == first and i < len(__a  ) - 1 and word[i + 1] == second:
                         new_word.append(first + second  )
                         i += 2
                   else:
                         new_word.append(word[i]  )
                         i += 1
             __snake_case :   Any													=    tuple(__a  )
             __snake_case :   List[Any]													=    new_word
             if len(__a  ) == 1:
                   break
             else:
                   __snake_case :   List[str]													=    get_pairs(__a  )
       __snake_case :   List[str]													=    ' '.join(__a  )
       __snake_case :   str													=    word
       return word
 def A_   (   self						:							Dict      ,     __a						:							str  )      ->   Dict:







       '''simple docstring'''
       __snake_case :   str													=    []
       for token in re.findall(self.pat      ,     __a  ):
             __snake_case :   Optional[Any]													=    ''.join(
                 self.byte_encoder[b] for b in token.encode('utf-8'  )  )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
             bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__a  ).split(' '  )  )
       return bpe_tokens
 def A_   (   self						:							Tuple      ,     __a						:							Optional[int]  )      ->   Any:







       '''simple docstring'''
       return self.encoder.get(__a      ,     self.encoder.get(self.unk_token  )  )
 def A_   (   self						:							List[Any]      ,     __a						:							List[Any]  )      ->   Optional[int]:







       '''simple docstring'''
       return self.decoder.get(__a  )
 def A_   (   self						:							str      ,     __a						:							int  )      ->   Optional[Any]:







       '''simple docstring'''
       __snake_case :   Optional[Any]													=    ''.join(__a  )
       __snake_case :   int													=    bytearray([self.byte_decoder[c] for c in text]  ).decode('utf-8'      ,     errors=self.errors  )
       return text
 def A_   (   self						:							List[str]      ,     __a						:							str      ,     __a						:							Optional[str] = None  )      ->   Tuple[str]:







       '''simple docstring'''
       if not os.path.isdir(__a  ):
             logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''  )
             return
       __snake_case :   List[str]													=    os.path.join(
           __a      ,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )
       __snake_case :   Optional[Any]													=    os.path.join(
           __a      ,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']  )

       with open(__a      ,     'w'      ,     encoding='utf-8'  ) as f:
             f.write(json.dumps(self.encoder      ,     indent=2      ,     sort_keys=__a      ,     ensure_ascii=__a  ) + '\n'  )

       __snake_case :   Dict													=    0
       with open(__a      ,     'w'      ,     encoding='utf-8'  ) as writer:
             writer.write('#version: 0.2\n'  )
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,     key=lambda __a  : kv[1]  ):
                   if index != token_index:
                         logger.warning(
                             f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                             ' Please check that the tokenizer is not corrupted!'  )
                         __snake_case :   Tuple													=    token_index
                   writer.write(' '.join(__a  ) + '\n'  )
                   index += 1

       return vocab_file, merge_file
 def A_   (   self						:							Any      ,     __a						:							List[int]      ,     __a						:							Optional[List[int]] = None  )      ->   List[int]:







       '''simple docstring'''
       if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       __snake_case :   Union[str, Any]													=    [self.cls_token_id]
       __snake_case :   Union[str, Any]													=    [self.sep_token_id]
       return cls + token_ids_a + sep + sep + token_ids_a + sep
 def A_   (   self						:							Optional[int]      ,     __a						:							List[int]      ,     __a						:							Optional[List[int]] = None      ,     __a						:							bool = False  )      ->   List[int]:







       '''simple docstring'''
       if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=__a      ,     token_ids_a=__a      ,     already_has_special_tokens=__a  )

       if token_ids_a is None:
             return [1] + ([0] * len(__a  )) + [1]
       return [1] + ([0] * len(__a  )) + [1, 1] + ([0] * len(__a  )) + [1]
 def A_   (   self						:							Union[str, Any]      ,     __a						:							List[int]      ,     __a						:							Optional[List[int]] = None  )      ->   List[int]:







       '''simple docstring'''
       __snake_case :   Dict													=    [self.sep_token_id]
       __snake_case :   List[Any]													=    [self.cls_token_id]

       if token_ids_a is None:
             return len(cls + token_ids_a + sep  ) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]
 def A_   (   self						:							Dict      ,     __a						:							List[Any]      ,     __a						:							int=False      ,     **__a						:							Dict  )      ->   List[Any]:







       '''simple docstring'''
       __snake_case :   Tuple													=    kwargs.pop('add_prefix_space'      ,     self.add_prefix_space  )
       if (is_split_into_words or add_prefix_space) and (len(__a  ) > 0 and not text[0].isspace()):
             __snake_case :   List[str]													=    ' ' + text
       return (text, kwargs)





 def A_   (   self						:							Optional[Any]      ,     __a						:							Union[Dict[str, EncodedInput], BatchEncoding]      ,     __a						:							Optional[int] = None      ,     __a						:							PaddingStrategy = PaddingStrategy.DO_NOT_PAD      ,     __a						:							Optional[int] = None      ,     __a						:							Optional[bool] = None      ,     )      ->   dict:







       '''simple docstring'''
       __snake_case :   Any													=    super()._pad(
           encoded_inputs=__a      ,     max_length=__a      ,     padding_strategy=__a      ,     pad_to_multiple_of=__a      ,     return_attention_mask=__a      ,     )

       # Load from model defaults
       if return_attention_mask is None:
             __snake_case :   Union[str, Any]													=    'attention_mask' in self.model_input_names

       if return_attention_mask and "global_attention_mask" in encoded_inputs:
             __snake_case :   int													=    encoded_inputs[self.model_input_names[0]]
             # `global_attention_mask` need to have the same length as other (sequential) inputs.
             __snake_case :   Optional[int]													=    len(encoded_inputs['global_attention_mask']  ) != len(__a  )

             if needs_to_be_padded:
                   __snake_case :   List[str]													=    len(__a  ) - len(encoded_inputs['global_attention_mask']  )

                   if self.padding_side == "right":
                         # Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
                         __snake_case :   Dict													=    (
                             encoded_inputs['global_attention_mask'] + [-1] * difference
                         )
                   elif self.padding_side == "left":
                         __snake_case :   str													=    [-1] * difference + encoded_inputs[
                             'global_attention_mask'
                         ]
                   else:
                         raise ValueError('Invalid padding strategy:' + str(self.padding_side  )  )

       return encoded_inputs








'''simple docstring'''





import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ..auto.configuration_auto import CONFIG_MAPPING


A__    :   Union[str, Any]           =					logging.get_logger(__name__)



class  snake_case__						(    SCREAMING_SNAKE_CASE_				):
 A__				= '''upernet'''
 def __init__(   self						:							Optional[int]      ,     __a						:							Dict=None      ,     __a						:							List[str]=512      ,     __a						:							int=0.0_2      ,     __a						:							Optional[Any]=[1, 2, 3, 6]      ,     __a						:							Union[str, Any]=True      ,     __a						:							Any=0.4      ,     __a						:							int=384      ,     __a						:							int=256      ,     __a						:							str=1      ,     __a						:							List[Any]=False      ,     __a						:							Dict=255      ,     **__a						:							Dict      ,     )      ->   Optional[int]:







       '''simple docstring'''
       super().__init__(**__a  )

       if backbone_config is None:
             logger.info('`backbone_config` is `None`. Initializing the config with the default `ResNet` backbone.'  )
             __snake_case :   Optional[int]													=    CONFIG_MAPPING['resnet'](out_features=['stage1', 'stage2', 'stage3', 'stage4']  )
       elif isinstance(__a      ,     __a  ):
             __snake_case :   List[Any]													=    backbone_config.get('model_type'  )
             __snake_case :   List[str]													=    CONFIG_MAPPING[backbone_model_type]
             __snake_case :   int													=    config_class.from_dict(__a  )

       __snake_case :   Any													=    backbone_config
       __snake_case :   Tuple													=    hidden_size
       __snake_case :   str													=    initializer_range
       __snake_case :   Dict													=    pool_scales
       __snake_case :   Optional[Any]													=    use_auxiliary_head
       __snake_case :   Optional[Any]													=    auxiliary_loss_weight
       __snake_case :   Optional[int]													=    auxiliary_in_channels
       __snake_case :   Dict													=    auxiliary_channels
       __snake_case :   Any													=    auxiliary_num_convs
       __snake_case :   Optional[Any]													=    auxiliary_concat_input
       __snake_case :   List[str]													=    loss_ignore_index





 def A_   (   self						:							List[Any]  )      ->   Union[str, Any]:







       '''simple docstring'''
       __snake_case :   Union[str, Any]													=    copy.deepcopy(self.__dict__  )
       __snake_case :   str													=    self.backbone_config.to_dict()
       __snake_case :   Optional[int]													=    self.__class__.model_type
       return output



