






import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


SCREAMING_SNAKE_CASE__				     =			logging.get_logger(__name__)


SCREAMING_SNAKE_CASE__				     =			{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

# See all LED models at https://huggingface.co/models?filter=LED
SCREAMING_SNAKE_CASE__				     =			{
    '''vocab_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json''',
    },
}

SCREAMING_SNAKE_CASE__				     =			{
    '''allenai/led-base-16384''': 1_6384,
}

@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def        UpperCAmelCase__		(						):
			__a						:  List[str]									=     (
			    list(range(ord('!'				)   ,  ord('~'				) + 1				)				) + list(range(ord('¡'				)   ,  ord('¬'				) + 1				)				) + list(range(ord('®'				)   ,  ord('ÿ'				) + 1				)				)
			)
			__a						:  str									=     bs[:]
			__a						:  List[Any]									=     0
			for b in range(2**8				):
						if b not in bs:
									bs.append(lowerCamelCase_				)
									cs.append(2**8 + n				)
									n += 1
			__a						:  Dict									=     [chr(lowerCamelCase_				) for n in cs]
			return dict(zip(lowerCamelCase_   ,  lowerCamelCase_				)				)

def        UpperCAmelCase__		(						lowerCamelCase_       :       Optional[Any]				):
			__a						:  Optional[Any]									=     set()
			__a						:  Optional[Any]									=     word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)				)
						__a						:  Dict									=     char
			return pairs






class 			_UpperCamelCase(       __lowerCamelCase		):
		__SCREAMING_SNAKE_CASE  :       Tuple							=							VOCAB_FILES_NAMES
		__SCREAMING_SNAKE_CASE  :       Any							=							PRETRAINED_VOCAB_FILES_MAP
		__SCREAMING_SNAKE_CASE  :       List[Any]							=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__SCREAMING_SNAKE_CASE  :       int							=							['''input_ids''', '''attention_mask''']

		def __init__(					self		:    Dict    ,       SCREAMING_SNAKE_CASE__		:    List[Any]    ,       SCREAMING_SNAKE_CASE__		:    List[str]    ,       SCREAMING_SNAKE_CASE__		:    Tuple="replace"    ,       SCREAMING_SNAKE_CASE__		:    int="<s>"    ,       SCREAMING_SNAKE_CASE__		:    List[str]="</s>"    ,       SCREAMING_SNAKE_CASE__		:    int="</s>"    ,       SCREAMING_SNAKE_CASE__		:    List[Any]="<s>"    ,       SCREAMING_SNAKE_CASE__		:    List[Any]="<unk>"    ,       SCREAMING_SNAKE_CASE__		:    Union[str, Any]="<pad>"    ,       SCREAMING_SNAKE_CASE__		:    List[Any]="<mask>"    ,       SCREAMING_SNAKE_CASE__		:    Optional[int]=False    ,       **SCREAMING_SNAKE_CASE__		:    Optional[Any]    ,       ):



					'''simple docstring'''



					__a						:  Union[str, Any]									=     AddedToken(SCREAMING_SNAKE_CASE__    ,       lstrip=SCREAMING_SNAKE_CASE__    ,       rstrip=SCREAMING_SNAKE_CASE__    ) if isinstance(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ) else bos_token
					__a						:  List[Any]									=     AddedToken(SCREAMING_SNAKE_CASE__    ,       lstrip=SCREAMING_SNAKE_CASE__    ,       rstrip=SCREAMING_SNAKE_CASE__    ) if isinstance(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ) else eos_token
					__a						:  int									=     AddedToken(SCREAMING_SNAKE_CASE__    ,       lstrip=SCREAMING_SNAKE_CASE__    ,       rstrip=SCREAMING_SNAKE_CASE__    ) if isinstance(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ) else sep_token
					__a						:  Optional[int]									=     AddedToken(SCREAMING_SNAKE_CASE__    ,       lstrip=SCREAMING_SNAKE_CASE__    ,       rstrip=SCREAMING_SNAKE_CASE__    ) if isinstance(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ) else cls_token
					__a						:  Optional[int]									=     AddedToken(SCREAMING_SNAKE_CASE__    ,       lstrip=SCREAMING_SNAKE_CASE__    ,       rstrip=SCREAMING_SNAKE_CASE__    ) if isinstance(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ) else unk_token
					__a						:  Dict									=     AddedToken(SCREAMING_SNAKE_CASE__    ,       lstrip=SCREAMING_SNAKE_CASE__    ,       rstrip=SCREAMING_SNAKE_CASE__    ) if isinstance(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ) else pad_token

					# Mask token behave like a normal word, i.e. include the space before it
					__a						:  Tuple									=     AddedToken(SCREAMING_SNAKE_CASE__    ,       lstrip=SCREAMING_SNAKE_CASE__    ,       rstrip=SCREAMING_SNAKE_CASE__    ) if isinstance(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ) else mask_token

					super().__init__(
					    errors=SCREAMING_SNAKE_CASE__    ,       bos_token=SCREAMING_SNAKE_CASE__    ,       eos_token=SCREAMING_SNAKE_CASE__    ,       unk_token=SCREAMING_SNAKE_CASE__    ,       sep_token=SCREAMING_SNAKE_CASE__    ,       cls_token=SCREAMING_SNAKE_CASE__    ,       pad_token=SCREAMING_SNAKE_CASE__    ,       mask_token=SCREAMING_SNAKE_CASE__    ,       add_prefix_space=SCREAMING_SNAKE_CASE__    ,       **SCREAMING_SNAKE_CASE__    ,       )

					with open(SCREAMING_SNAKE_CASE__    ,       encoding='utf-8'    ) as vocab_handle:
								__a						:  List[Any]									=     json.load(SCREAMING_SNAKE_CASE__    )
					__a						:  List[str]									=     {v: k for k, v in self.encoder.items()}
					__a						:  Dict									=     errors  # how to handle errors in decoding
					__a						:  List[Any]									=     bytes_to_unicode()
					__a						:  List[Any]									=     {v: k for k, v in self.byte_encoder.items()}
					with open(SCREAMING_SNAKE_CASE__    ,       encoding='utf-8'    ) as merges_handle:
								__a						:  str									=     merges_handle.read().split('\n'    )[1:-1]
					__a						:  List[str]									=     [tuple(merge.split()    ) for merge in bpe_merges]
					__a						:  str									=     dict(zip(SCREAMING_SNAKE_CASE__    ,       range(len(SCREAMING_SNAKE_CASE__    )    )    )    )
					__a						:  Tuple									=     {}
					__a						:  Union[str, Any]									=     add_prefix_space

					# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
					__a						:  Optional[Any]									=     re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'    )

		@property
		# Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
		def    __lowerCAmelCase  (					self		:    List[str]    ):



					'''simple docstring'''



					return len(self.encoder    )

		def    __lowerCAmelCase  (					self		:    Dict    ):



					'''simple docstring'''



					return dict(self.encoder    ,       **self.added_tokens_encoder    )

		def    __lowerCAmelCase  (					self		:    int    ,       SCREAMING_SNAKE_CASE__		:    Dict    ):



					'''simple docstring'''



					if token in self.cache:
								return self.cache[token]
					__a						:  List[Any]									=     tuple(SCREAMING_SNAKE_CASE__    )
					__a						:  Tuple									=     get_pairs(SCREAMING_SNAKE_CASE__    )

					if not pairs:
								return token

					while True:
								__a						:  Tuple									=     min(SCREAMING_SNAKE_CASE__    ,       key=lambda SCREAMING_SNAKE_CASE__    : self.bpe_ranks.get(SCREAMING_SNAKE_CASE__    ,       float('inf'    )    )    )
								if bigram not in self.bpe_ranks:
											break
								__a						,					__a						:  str									=     bigram
								__a						:  Optional[Any]									=     []
								__a						:  List[str]									=     0
								while i < len(SCREAMING_SNAKE_CASE__    ):
											try:
														__a						:  str									=     word.index(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    )
											except ValueError:
														new_word.extend(word[i:]    )
														break
											else:
														new_word.extend(word[i:j]    )
														__a						:  List[Any]									=     j

											if word[i] == first and i < len(SCREAMING_SNAKE_CASE__    ) - 1 and word[i + 1] == second:
														new_word.append(first + second    )
														i += 2
											else:
														new_word.append(word[i]    )
														i += 1
								__a						:  Optional[int]									=     tuple(SCREAMING_SNAKE_CASE__    )
								__a						:  Union[str, Any]									=     new_word
								if len(SCREAMING_SNAKE_CASE__    ) == 1:
											break
								else:
											__a						:  Dict									=     get_pairs(SCREAMING_SNAKE_CASE__    )
					__a						:  Tuple									=     ' '.join(SCREAMING_SNAKE_CASE__    )
					__a						:  List[Any]									=     word
					return word

		def    __lowerCAmelCase  (					self		:    str    ,       SCREAMING_SNAKE_CASE__		:    Any    ):



					'''simple docstring'''



					__a						:  int									=     []
					for token in re.findall(self.pat    ,       SCREAMING_SNAKE_CASE__    ):
								__a						:  Optional[Any]									=     ''.join(
								    self.byte_encoder[b] for b in token.encode('utf-8'    )    )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
								bpe_tokens.extend(bpe_token for bpe_token in self.bpe(SCREAMING_SNAKE_CASE__    ).split(' '    )    )
					return bpe_tokens

		def    __lowerCAmelCase  (					self		:    Union[str, Any]    ,       SCREAMING_SNAKE_CASE__		:    str    ):



					'''simple docstring'''



					return self.encoder.get(SCREAMING_SNAKE_CASE__    ,       self.encoder.get(self.unk_token    )    )

		def    __lowerCAmelCase  (					self		:    List[Any]    ,       SCREAMING_SNAKE_CASE__		:    Optional[Any]    ):



					'''simple docstring'''



					return self.decoder.get(SCREAMING_SNAKE_CASE__    )

		def    __lowerCAmelCase  (					self		:    Optional[Any]    ,       SCREAMING_SNAKE_CASE__		:    Union[str, Any]    ):



					'''simple docstring'''



					__a						:  Optional[Any]									=     ''.join(SCREAMING_SNAKE_CASE__    )
					__a						:  Optional[Any]									=     bytearray([self.byte_decoder[c] for c in text]    ).decode('utf-8'    ,       errors=self.errors    )
					return text

		def    __lowerCAmelCase  (					self		:    Optional[Any]    ,       SCREAMING_SNAKE_CASE__		:    str    ,       SCREAMING_SNAKE_CASE__		:    Optional[str] = None    ):



					'''simple docstring'''



					if not os.path.isdir(SCREAMING_SNAKE_CASE__    ):
								logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''    )
								return
					__a						:  int									=     os.path.join(
					    SCREAMING_SNAKE_CASE__    ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )
					__a						:  Optional[int]									=     os.path.join(
					    SCREAMING_SNAKE_CASE__    ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']    )

					with open(SCREAMING_SNAKE_CASE__    ,       'w'    ,       encoding='utf-8'    ) as f:
								f.write(json.dumps(self.encoder    ,       indent=2    ,       sort_keys=SCREAMING_SNAKE_CASE__    ,       ensure_ascii=SCREAMING_SNAKE_CASE__    ) + '\n'    )

					__a						:  List[Any]									=     0
					with open(SCREAMING_SNAKE_CASE__    ,       'w'    ,       encoding='utf-8'    ) as writer:
								writer.write('#version: 0.2\n'    )
								for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,       key=lambda SCREAMING_SNAKE_CASE__    : kv[1]    ):
											if index != token_index:
														logger.warning(
														    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
														    ' Please check that the tokenizer is not corrupted!'    )
														__a						:  Optional[int]									=     token_index
											writer.write(' '.join(SCREAMING_SNAKE_CASE__    ) + '\n'    )
											index += 1

					return vocab_file, merge_file

		def    __lowerCAmelCase  (					self		:    Dict    ,       SCREAMING_SNAKE_CASE__		:    List[int]    ,       SCREAMING_SNAKE_CASE__		:    Optional[List[int]] = None    ):



					'''simple docstring'''



					if token_ids_a is None:
								return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
					__a						:  Union[str, Any]									=     [self.cls_token_id]
					__a						:  str									=     [self.sep_token_id]
					return cls + token_ids_a + sep + sep + token_ids_a + sep

		def    __lowerCAmelCase  (					self		:    Union[str, Any]    ,       SCREAMING_SNAKE_CASE__		:    List[int]    ,       SCREAMING_SNAKE_CASE__		:    Optional[List[int]] = None    ,       SCREAMING_SNAKE_CASE__		:    bool = False    ):



					'''simple docstring'''



					if already_has_special_tokens:
								return super().get_special_tokens_mask(
								    token_ids_a=SCREAMING_SNAKE_CASE__    ,       token_ids_a=SCREAMING_SNAKE_CASE__    ,       already_has_special_tokens=SCREAMING_SNAKE_CASE__    )

					if token_ids_a is None:
								return [1] + ([0] * len(SCREAMING_SNAKE_CASE__    )) + [1]
					return [1] + ([0] * len(SCREAMING_SNAKE_CASE__    )) + [1, 1] + ([0] * len(SCREAMING_SNAKE_CASE__    )) + [1]

		def    __lowerCAmelCase  (					self		:    int    ,       SCREAMING_SNAKE_CASE__		:    List[int]    ,       SCREAMING_SNAKE_CASE__		:    Optional[List[int]] = None    ):



					'''simple docstring'''



					__a						:  Any									=     [self.sep_token_id]
					__a						:  Union[str, Any]									=     [self.cls_token_id]

					if token_ids_a is None:
								return len(cls + token_ids_a + sep    ) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]

		def    __lowerCAmelCase  (					self		:    Any    ,       SCREAMING_SNAKE_CASE__		:    int    ,       SCREAMING_SNAKE_CASE__		:    List[str]=False    ,       **SCREAMING_SNAKE_CASE__		:    Optional[Any]    ):



					'''simple docstring'''



					__a						:  Dict									=     kwargs.pop('add_prefix_space'    ,       self.add_prefix_space    )
					if (is_split_into_words or add_prefix_space) and (len(SCREAMING_SNAKE_CASE__    ) > 0 and not text[0].isspace()):
								__a						:  Tuple									=     ' ' + text
					return (text, kwargs)



		def    __lowerCAmelCase  (					self		:    Union[str, Any]    ,       SCREAMING_SNAKE_CASE__		:    Union[Dict[str, EncodedInput], BatchEncoding]    ,       SCREAMING_SNAKE_CASE__		:    Optional[int] = None    ,       SCREAMING_SNAKE_CASE__		:    PaddingStrategy = PaddingStrategy.DO_NOT_PAD    ,       SCREAMING_SNAKE_CASE__		:    Optional[int] = None    ,       SCREAMING_SNAKE_CASE__		:    Optional[bool] = None    ,       ):



					'''simple docstring'''



					__a						:  Union[str, Any]									=     super()._pad(
					    encoded_inputs=SCREAMING_SNAKE_CASE__    ,       max_length=SCREAMING_SNAKE_CASE__    ,       padding_strategy=SCREAMING_SNAKE_CASE__    ,       pad_to_multiple_of=SCREAMING_SNAKE_CASE__    ,       return_attention_mask=SCREAMING_SNAKE_CASE__    ,       )

					# Load from model defaults
					if return_attention_mask is None:
								__a						:  Tuple									=     'attention_mask' in self.model_input_names

					if return_attention_mask and "global_attention_mask" in encoded_inputs:
								__a						:  Optional[Any]									=     encoded_inputs[self.model_input_names[0]]
								# `global_attention_mask` need to have the same length as other (sequential) inputs.
								__a						:  int									=     len(encoded_inputs['global_attention_mask']    ) != len(SCREAMING_SNAKE_CASE__    )

								if needs_to_be_padded:
											__a						:  Any									=     len(SCREAMING_SNAKE_CASE__    ) - len(encoded_inputs['global_attention_mask']    )

											if self.padding_side == "right":
														# Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
														__a						:  Optional[Any]									=     (
														    encoded_inputs['global_attention_mask'] + [-1] * difference
														)
											elif self.padding_side == "left":
														__a						:  List[str]									=     [-1] * difference + encoded_inputs[
														    'global_attention_mask'
														]
											else:
														raise ValueError('Invalid padding strategy:' + str(self.padding_side    )    )

					return encoded_inputs










def 	SCREAMING_SNAKE_CASE_							(	_snake_case	:bytes				)  ->     str:
   return "".join([hex(_snake_case				)[2:].zfill(2				).upper() for byte in list(_snake_case				)]				)

def 	SCREAMING_SNAKE_CASE_							(	_snake_case	:str				)  ->     bytes:
   # Check data validity, following RFC3548
   # https://www.ietf.org/rfc/rfc3548.txt
   if (len(_snake_case				) % 2) != 0:
      raise ValueError(
          '''Base16 encoded data is invalid:
Data does not have an even number of hex digits.'''				)
   # Check the character set - the standard base16 alphabet
   # is uppercase according to RFC3548 section 6
   if not set(_snake_case				) <= set('''0123456789ABCDEF'''				):
      raise ValueError(
          '''Base16 encoded data is invalid:
Data is not uppercase hex or it contains invalid characters.'''				)
   # For every two hexadecimal digits (= a byte), turn it into an integer.
   # Then, string the result together into bytes, and return it.
   return bytes(int(data[i] + data[i + 1]	,   16				) for i in range(0	,   len(_snake_case				)	,   2				)				)


if __name__ == "__main__":
   import doctest

   doctest.testmod()



