


from binascii import hexlify
from hashlib import shaaaa
from os import urandom

# RFC 3526 - More Modular Exponential (MODP) Diffie-Hellman groups for
# Internet Key Exchange (IKE) https://tools.ietf.org/html/rfc3526

a : Dict		=				{
    # 1536-bit
    5: {
        '''prime''': int(
            '''FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1'''
            + '''29024E088A67CC74020BBEA63B139B22514A08798E3404DD'''
            + '''EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245'''
            + '''E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED'''
            + '''EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D'''
            + '''C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F'''
            + '''83655D23DCA3AD961C62F356208552BB9ED529077096966D'''
            + '''670C354E4ABC9804F1746C08CA237327FFFFFFFFFFFFFFFF''',
            base=16,
        ),
        '''generator''': 2,
    },
    # 2048-bit
    14: {
        '''prime''': int(
            '''FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1'''
            + '''29024E088A67CC74020BBEA63B139B22514A08798E3404DD'''
            + '''EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245'''
            + '''E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED'''
            + '''EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D'''
            + '''C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F'''
            + '''83655D23DCA3AD961C62F356208552BB9ED529077096966D'''
            + '''670C354E4ABC9804F1746C08CA18217C32905E462E36CE3B'''
            + '''E39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9'''
            + '''DE2BCBF6955817183995497CEA956AE515D2261898FA0510'''
            + '''15728E5A8AACAA68FFFFFFFFFFFFFFFF''',
            base=16,
        ),
        '''generator''': 2,
    },
    # 3072-bit
    15: {
        '''prime''': int(
            '''FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1'''
            + '''29024E088A67CC74020BBEA63B139B22514A08798E3404DD'''
            + '''EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245'''
            + '''E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED'''
            + '''EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D'''
            + '''C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F'''
            + '''83655D23DCA3AD961C62F356208552BB9ED529077096966D'''
            + '''670C354E4ABC9804F1746C08CA18217C32905E462E36CE3B'''
            + '''E39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9'''
            + '''DE2BCBF6955817183995497CEA956AE515D2261898FA0510'''
            + '''15728E5A8AAAC42DAD33170D04507A33A85521ABDF1CBA64'''
            + '''ECFB850458DBEF0A8AEA71575D060C7DB3970F85A6E1E4C7'''
            + '''ABF5AE8CDB0933D71E8C94E04A25619DCEE3D2261AD2EE6B'''
            + '''F12FFA06D98A0864D87602733EC86A64521F2B18177B200C'''
            + '''BBE117577A615D6C770988C0BAD946E208E24FA074E5AB31'''
            + '''43DB5BFCE0FD108E4B82D120A93AD2CAFFFFFFFFFFFFFFFF''',
            base=16,
        ),
        '''generator''': 2,
    },
    # 4096-bit
    16: {
        '''prime''': int(
            '''FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1'''
            + '''29024E088A67CC74020BBEA63B139B22514A08798E3404DD'''
            + '''EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245'''
            + '''E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED'''
            + '''EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D'''
            + '''C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F'''
            + '''83655D23DCA3AD961C62F356208552BB9ED529077096966D'''
            + '''670C354E4ABC9804F1746C08CA18217C32905E462E36CE3B'''
            + '''E39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9'''
            + '''DE2BCBF6955817183995497CEA956AE515D2261898FA0510'''
            + '''15728E5A8AAAC42DAD33170D04507A33A85521ABDF1CBA64'''
            + '''ECFB850458DBEF0A8AEA71575D060C7DB3970F85A6E1E4C7'''
            + '''ABF5AE8CDB0933D71E8C94E04A25619DCEE3D2261AD2EE6B'''
            + '''F12FFA06D98A0864D87602733EC86A64521F2B18177B200C'''
            + '''BBE117577A615D6C770988C0BAD946E208E24FA074E5AB31'''
            + '''43DB5BFCE0FD108E4B82D120A92108011A723C12A787E6D7'''
            + '''88719A10BDBA5B2699C327186AF4E23C1A946834B6150BDA'''
            + '''2583E9CA2AD44CE8DBBBC2DB04DE8EF92E8EFC141FBECAA6'''
            + '''287C59474E6BC05D99B2964FA090C3A2233BA186515BE7ED'''
            + '''1F612970CEE2D7AFB81BDD762170481CD0069127D5B05AA9'''
            + '''93B4EA988D8FDDC186FFB7DC90A6C08F4DF435C934063199'''
            + '''FFFFFFFFFFFFFFFF''',
            base=16,
        ),
        '''generator''': 2,
    },
    # 6144-bit
    17: {
        '''prime''': int(
            '''FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E08'''
            + '''8A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B'''
            + '''302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9'''
            + '''A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE6'''
            + '''49286651ECE45B3DC2007CB8A163BF0598DA48361C55D39A69163FA8'''
            + '''FD24CF5F83655D23DCA3AD961C62F356208552BB9ED529077096966D'''
            + '''670C354E4ABC9804F1746C08CA18217C32905E462E36CE3BE39E772C'''
            + '''180E86039B2783A2EC07A28FB5C55DF06F4C52C9DE2BCBF695581718'''
            + '''3995497CEA956AE515D2261898FA051015728E5A8AAAC42DAD33170D'''
            + '''04507A33A85521ABDF1CBA64ECFB850458DBEF0A8AEA71575D060C7D'''
            + '''B3970F85A6E1E4C7ABF5AE8CDB0933D71E8C94E04A25619DCEE3D226'''
            + '''1AD2EE6BF12FFA06D98A0864D87602733EC86A64521F2B18177B200C'''
            + '''BBE117577A615D6C770988C0BAD946E208E24FA074E5AB3143DB5BFC'''
            + '''E0FD108E4B82D120A92108011A723C12A787E6D788719A10BDBA5B26'''
            + '''99C327186AF4E23C1A946834B6150BDA2583E9CA2AD44CE8DBBBC2DB'''
            + '''04DE8EF92E8EFC141FBECAA6287C59474E6BC05D99B2964FA090C3A2'''
            + '''233BA186515BE7ED1F612970CEE2D7AFB81BDD762170481CD0069127'''
            + '''D5B05AA993B4EA988D8FDDC186FFB7DC90A6C08F4DF435C934028492'''
            + '''36C3FAB4D27C7026C1D4DCB2602646DEC9751E763DBA37BDF8FF9406'''
            + '''AD9E530EE5DB382F413001AEB06A53ED9027D831179727B0865A8918'''
            + '''DA3EDBEBCF9B14ED44CE6CBACED4BB1BDB7F1447E6CC254B33205151'''
            + '''2BD7AF426FB8F401378CD2BF5983CA01C64B92ECF032EA15D1721D03'''
            + '''F482D7CE6E74FEF6D55E702F46980C82B5A84031900B1C9E59E7C97F'''
            + '''BEC7E8F323A97A7E36CC88BE0F1D45B7FF585AC54BD407B22B4154AA'''
            + '''CC8F6D7EBF48E1D814CC5ED20F8037E0A79715EEF29BE32806A1D58B'''
            + '''B7C5DA76F550AA3D8A1FBFF0EB19CCB1A313D55CDA56C9EC2EF29632'''
            + '''387FE8D76E3C0468043E8F663F4860EE12BF2D5B0B7474D6E694F91E'''
            + '''6DCC4024FFFFFFFFFFFFFFFF''',
            base=16,
        ),
        '''generator''': 2,
    },
    # 8192-bit
    18: {
        '''prime''': int(
            '''FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1'''
            + '''29024E088A67CC74020BBEA63B139B22514A08798E3404DD'''
            + '''EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245'''
            + '''E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED'''
            + '''EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3D'''
            + '''C2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F'''
            + '''83655D23DCA3AD961C62F356208552BB9ED529077096966D'''
            + '''670C354E4ABC9804F1746C08CA18217C32905E462E36CE3B'''
            + '''E39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9'''
            + '''DE2BCBF6955817183995497CEA956AE515D2261898FA0510'''
            + '''15728E5A8AAAC42DAD33170D04507A33A85521ABDF1CBA64'''
            + '''ECFB850458DBEF0A8AEA71575D060C7DB3970F85A6E1E4C7'''
            + '''ABF5AE8CDB0933D71E8C94E04A25619DCEE3D2261AD2EE6B'''
            + '''F12FFA06D98A0864D87602733EC86A64521F2B18177B200C'''
            + '''BBE117577A615D6C770988C0BAD946E208E24FA074E5AB31'''
            + '''43DB5BFCE0FD108E4B82D120A92108011A723C12A787E6D7'''
            + '''88719A10BDBA5B2699C327186AF4E23C1A946834B6150BDA'''
            + '''2583E9CA2AD44CE8DBBBC2DB04DE8EF92E8EFC141FBECAA6'''
            + '''287C59474E6BC05D99B2964FA090C3A2233BA186515BE7ED'''
            + '''1F612970CEE2D7AFB81BDD762170481CD0069127D5B05AA9'''
            + '''93B4EA988D8FDDC186FFB7DC90A6C08F4DF435C934028492'''
            + '''36C3FAB4D27C7026C1D4DCB2602646DEC9751E763DBA37BD'''
            + '''F8FF9406AD9E530EE5DB382F413001AEB06A53ED9027D831'''
            + '''179727B0865A8918DA3EDBEBCF9B14ED44CE6CBACED4BB1B'''
            + '''DB7F1447E6CC254B332051512BD7AF426FB8F401378CD2BF'''
            + '''5983CA01C64B92ECF032EA15D1721D03F482D7CE6E74FEF6'''
            + '''D55E702F46980C82B5A84031900B1C9E59E7C97FBEC7E8F3'''
            + '''23A97A7E36CC88BE0F1D45B7FF585AC54BD407B22B4154AA'''
            + '''CC8F6D7EBF48E1D814CC5ED20F8037E0A79715EEF29BE328'''
            + '''06A1D58BB7C5DA76F550AA3D8A1FBFF0EB19CCB1A313D55C'''
            + '''DA56C9EC2EF29632387FE8D76E3C0468043E8F663F4860EE'''
            + '''12BF2D5B0B7474D6E694F91E6DBE115974A3926F12FEE5E4'''
            + '''38777CB6A932DF8CD8BEC4D073B931BA3BC832B68D9DD300'''
            + '''741FA7BF8AFC47ED2576F6936BA424663AAB639C5AE4F568'''
            + '''3423B4742BF1C978238F16CBE39D652DE3FDB8BEFC848AD9'''
            + '''22222E04A4037C0713EB57A81A23F0C73473FC646CEA306B'''
            + '''4BCBC8862F8385DDFA9D4B7FA2C087E879683303ED5BDD3A'''
            + '''062B3CF5B3A278A66D2A13F83F44F82DDF310EE074AB6A36'''
            + '''4597E899A0255DC164F31CC50846851DF9AB48195DED7EA1'''
            + '''B1D510BD7EE74D73FAF36BC31ECFA268359046F4EB879F92'''
            + '''4009438B481C6CD7889A002ED5EE382BC9190DA6FC026E47'''
            + '''9558E4475677E9AA9E3050E2765694DFC81F56E880B96E71'''
            + '''60C980DD98EDD3DFFFFFFFFFFFFFFFFF''',
            base=16,
        ),
        '''generator''': 2,
    },
}
class 	lowerCamelCase_       :



     '''simple docstring'''







     def __init__(				self ,       snake_case_ = 1_4						)   ->			Tuple:







        '''simple docstring'''

        if group not in primes:
           raise ValueError('''Unsupported Group'''						)
        __lowercase 					=   primes[group]["""prime"""]
        __lowercase 					=   primes[group]["""generator"""]

        __lowercase 					=   int(hexlify(urandom(3_2						)						) ,       base=1_6						)






     def A      (				self						)   ->			Optional[Any]:







        '''simple docstring'''

        return hex(self.__private_key						)[2:]






     def A      (				self						)   ->			Dict:







        '''simple docstring'''

        __lowercase 					=   pow(self.generator ,       self.__private_key ,       self.prime						)
        return hex(lowerCamelCase_						)[2:]






     def A      (				self ,       snake_case_						)   ->			int:







        '''simple docstring'''

        return (
            2 <= key <= self.prime - 2
            and pow(lowerCamelCase_ ,       (self.prime - 1) // 2 ,       self.prime						) == 1
        )






     def A      (				self ,       snake_case_						)   ->			int:







        '''simple docstring'''

        __lowercase 					=   int(lowerCamelCase_ ,       base=1_6						)
        if not self.is_valid_public_key(lowerCamelCase_						):
           raise ValueError('''Invalid public key'''						)
        __lowercase 					=   pow(lowerCamelCase_ ,       self.__private_key ,       self.prime						)
        return shaaaa(str(lowerCamelCase_						).encode()						).hexdigest()






     @staticmethod
     def A      (				snake_case_ ,       snake_case_						)   ->			Tuple:







        '''simple docstring'''

        return (
            2 <= remote_public_key_str <= prime - 2
            and pow(lowerCamelCase_ ,       (prime - 1) // 2 ,       lowerCamelCase_						) == 1
        )






     @staticmethod
     def A      (				snake_case_ ,       snake_case_ ,       snake_case_ = 1_4						)   ->			Dict:







        '''simple docstring'''

        __lowercase 					=   int(lowerCamelCase_ ,       base=1_6						)
        __lowercase 					=   int(lowerCamelCase_ ,       base=1_6						)
        __lowercase 					=   primes[group]["""prime"""]
        if not DiffieHellman.is_valid_public_key_static(lowerCamelCase_ ,       lowerCamelCase_						):
           raise ValueError('''Invalid public key'''						)
        __lowercase 					=   pow(lowerCamelCase_ ,       lowerCamelCase_ ,       lowerCamelCase_						)
        return shaaaa(str(lowerCamelCase_						).encode()						).hexdigest()


if __name__ == "__main__":
 import doctest

 doctest.testmod()






import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


a : int		=				logging.get_logger(__name__)


a : Any		=				{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

# See all LED models at https://huggingface.co/models?filter=LED
a : Dict		=				{
    '''vocab_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json''',
    },
}

a : Optional[int]		=				{
    '''allenai/led-base-16384''': 16384,
}


@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def 		lowercase_  (     ):


			'''simple docstring'''







			__lowercase 					=   (
			    list(range(ord('''!'''  )       ,			ord('''~'''  ) + 1  )  ) + list(range(ord('''¡'''  )       ,			ord('''¬'''  ) + 1  )  ) + list(range(ord('''®'''  )       ,			ord('''ÿ'''  ) + 1  )  )
			)
			__lowercase 					=   bs[:]
			__lowercase 					=   0
			for b in range(2**8  ):
						if b not in bs:
									bs.append(_UpperCamelCase  )
									cs.append(2**8 + n  )
									n += 1
			__lowercase 					=   [chr(_UpperCamelCase  ) for n in cs]
			return dict(zip(_UpperCamelCase       ,			_UpperCamelCase  )  )


def 		lowercase_  (     _UpperCamelCase  ):


			'''simple docstring'''







			__lowercase 					=   set()
			__lowercase 					=   word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)  )
						__lowercase 					=   char
			return pairs





class 	lowerCamelCase_       (				lowerCAmelCase__				):



					'''simple docstring'''







					__UpperCAmelCase            =				VOCAB_FILES_NAMES
					__UpperCAmelCase            =				PRETRAINED_VOCAB_FILES_MAP
					__UpperCAmelCase            =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__UpperCAmelCase            =				["input_ids", "attention_mask"]






					def __init__(				self ,       snake_case_ ,       snake_case_ ,       snake_case_="replace" ,       snake_case_="<s>" ,       snake_case_="</s>" ,       snake_case_="</s>" ,       snake_case_="<s>" ,       snake_case_="<unk>" ,       snake_case_="<pad>" ,       snake_case_="<mask>" ,       snake_case_=False ,       **snake_case_ ,       )   ->			List[Any]:







								'''simple docstring'''

								__lowercase 					=   AddedToken(snake_case_ ,       lstrip=snake_case_ ,       rstrip=snake_case_						) if isinstance(snake_case_ ,       snake_case_						) else bos_token
								__lowercase 					=   AddedToken(snake_case_ ,       lstrip=snake_case_ ,       rstrip=snake_case_						) if isinstance(snake_case_ ,       snake_case_						) else eos_token
								__lowercase 					=   AddedToken(snake_case_ ,       lstrip=snake_case_ ,       rstrip=snake_case_						) if isinstance(snake_case_ ,       snake_case_						) else sep_token
								__lowercase 					=   AddedToken(snake_case_ ,       lstrip=snake_case_ ,       rstrip=snake_case_						) if isinstance(snake_case_ ,       snake_case_						) else cls_token
								__lowercase 					=   AddedToken(snake_case_ ,       lstrip=snake_case_ ,       rstrip=snake_case_						) if isinstance(snake_case_ ,       snake_case_						) else unk_token
								__lowercase 					=   AddedToken(snake_case_ ,       lstrip=snake_case_ ,       rstrip=snake_case_						) if isinstance(snake_case_ ,       snake_case_						) else pad_token

								# Mask token behave like a normal word, i.e. include the space before it
								__lowercase 					=   AddedToken(snake_case_ ,       lstrip=snake_case_ ,       rstrip=snake_case_						) if isinstance(snake_case_ ,       snake_case_						) else mask_token

								super().__init__(
								    errors=snake_case_ ,       bos_token=snake_case_ ,       eos_token=snake_case_ ,       unk_token=snake_case_ ,       sep_token=snake_case_ ,       cls_token=snake_case_ ,       pad_token=snake_case_ ,       mask_token=snake_case_ ,       add_prefix_space=snake_case_ ,       **snake_case_ ,       )

								with open(snake_case_ ,       encoding='''utf-8'''						) as vocab_handle:
											__lowercase 					=   json.load(snake_case_						)
								__lowercase 					=   {v: k for k, v in self.encoder.items()}
								__lowercase 					=   errors  # how to handle errors in decoding
								__lowercase 					=   bytes_to_unicode()
								__lowercase 					=   {v: k for k, v in self.byte_encoder.items()}
								with open(snake_case_ ,       encoding='''utf-8'''						) as merges_handle:
											__lowercase 					=   merges_handle.read().split('''\n'''						)[1:-1]
								__lowercase 					=   [tuple(merge.split()						) for merge in bpe_merges]
								__lowercase 					=   dict(zip(snake_case_ ,       range(len(snake_case_						)						)						)						)
								__lowercase 					=   {}
								__lowercase 					=   add_prefix_space

								# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
								__lowercase 					=   re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''						)






					@property
					# Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
					def A      (				self						)   ->			Union[str, Any]:







								'''simple docstring'''

								return len(self.encoder						)






					def A      (				self						)   ->			str:







								'''simple docstring'''

								return dict(self.encoder ,       **self.added_tokens_encoder						)






					def A      (				self ,       snake_case_						)   ->			Optional[int]:







								'''simple docstring'''

								if token in self.cache:
											return self.cache[token]
								__lowercase 					=   tuple(snake_case_						)
								__lowercase 					=   get_pairs(snake_case_						)

								if not pairs:
											return token

								while True:
											__lowercase 					=   min(snake_case_ ,       key=lambda snake_case_						: self.bpe_ranks.get(snake_case_ ,       float('''inf'''						)						)						)
											if bigram not in self.bpe_ranks:
														break
											__lowercase					, __lowercase 					=   bigram
											__lowercase 					=   []
											__lowercase 					=   0
											while i < len(snake_case_						):
														try:
																	__lowercase 					=   word.index(snake_case_ ,       snake_case_						)
														except ValueError:
																	new_word.extend(word[i:]						)
																	break
														else:
																	new_word.extend(word[i:j]						)
																	__lowercase 					=   j

														if word[i] == first and i < len(snake_case_						) - 1 and word[i + 1] == second:
																	new_word.append(first + second						)
																	i += 2
														else:
																	new_word.append(word[i]						)
																	i += 1
											__lowercase 					=   tuple(snake_case_						)
											__lowercase 					=   new_word
											if len(snake_case_						) == 1:
														break
											else:
														__lowercase 					=   get_pairs(snake_case_						)
								__lowercase 					=   ''' '''.join(snake_case_						)
								__lowercase 					=   word
								return word






					def A      (				self ,       snake_case_						)   ->			Tuple:







								'''simple docstring'''

								__lowercase 					=   []
								for token in re.findall(self.pat ,       snake_case_						):
											__lowercase 					=   ''''''.join(
											    self.byte_encoder[b] for b in token.encode('''utf-8'''						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
											bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case_						).split(''' '''						)						)
								return bpe_tokens






					def A      (				self ,       snake_case_						)   ->			List[Any]:







								'''simple docstring'''

								return self.encoder.get(snake_case_ ,       self.encoder.get(self.unk_token						)						)






					def A      (				self ,       snake_case_						)   ->			Tuple:







								'''simple docstring'''

								return self.decoder.get(snake_case_						)






					def A      (				self ,       snake_case_						)   ->			Any:







								'''simple docstring'''

								__lowercase 					=   ''''''.join(snake_case_						)
								__lowercase 					=   bytearray([self.byte_decoder[c] for c in text]						).decode('''utf-8''' ,       errors=self.errors						)
								return text






					def A      (				self ,       snake_case_ ,       snake_case_ = None						)   ->			Tuple[str]:







								'''simple docstring'''

								if not os.path.isdir(snake_case_						):
											logger.error(F'Vocabulary path ({save_directory}) should be a directory'						)
											return
								__lowercase 					=   os.path.join(
								    snake_case_ ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)
								__lowercase 					=   os.path.join(
								    snake_case_ ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']						)

								with open(snake_case_ ,       '''w''' ,       encoding='''utf-8'''						) as f:
											f.write(json.dumps(self.encoder ,       indent=2 ,       sort_keys=snake_case_ ,       ensure_ascii=snake_case_						) + '''\n'''						)

								__lowercase 					=   0
								with open(snake_case_ ,       '''w''' ,       encoding='''utf-8'''						) as writer:
											writer.write('''#version: 0.2\n'''						)
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,       key=lambda snake_case_						: kv[1]						):
														if index != token_index:
																	logger.warning(
																	    F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																	    ''' Please check that the tokenizer is not corrupted!'''						)
																	__lowercase 					=   token_index
														writer.write(''' '''.join(snake_case_						) + '''\n'''						)
														index += 1

								return vocab_file, merge_file






					def A      (				self ,       snake_case_ ,       snake_case_ = None						)   ->			List[int]:







								'''simple docstring'''

								if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								__lowercase 					=   [self.cls_token_id]
								__lowercase 					=   [self.sep_token_id]
								return cls + token_ids_a + sep + sep + token_ids_a + sep






					def A      (				self ,       snake_case_ ,       snake_case_ = None ,       snake_case_ = False						)   ->			List[int]:







								'''simple docstring'''

								if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=snake_case_ ,       token_ids_a=snake_case_ ,       already_has_special_tokens=snake_case_						)

								if token_ids_a is None:
											return [1] + ([0] * len(snake_case_						)) + [1]
								return [1] + ([0] * len(snake_case_						)) + [1, 1] + ([0] * len(snake_case_						)) + [1]






					def A      (				self ,       snake_case_ ,       snake_case_ = None						)   ->			List[int]:







								'''simple docstring'''

								__lowercase 					=   [self.sep_token_id]
								__lowercase 					=   [self.cls_token_id]

								if token_ids_a is None:
											return len(cls + token_ids_a + sep						) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]






					def A      (				self ,       snake_case_ ,       snake_case_=False ,       **snake_case_						)   ->			Any:







								'''simple docstring'''

								__lowercase 					=   kwargs.pop('''add_prefix_space''' ,       self.add_prefix_space						)
								if (is_split_into_words or add_prefix_space) and (len(snake_case_						) > 0 and not text[0].isspace()):
											__lowercase 					=   ''' ''' + text
								return (text, kwargs)






					def A      (				self ,       snake_case_ ,       snake_case_ = None ,       snake_case_ = PaddingStrategy.DO_NOT_PAD ,       snake_case_ = None ,       snake_case_ = None ,       )   ->			dict:







								'''simple docstring'''

								__lowercase 					=   super()._pad(
								    encoded_inputs=snake_case_ ,       max_length=snake_case_ ,       padding_strategy=snake_case_ ,       pad_to_multiple_of=snake_case_ ,       return_attention_mask=snake_case_ ,       )

								# Load from model defaults
								if return_attention_mask is None:
											__lowercase 					=   '''attention_mask''' in self.model_input_names

								if return_attention_mask and "global_attention_mask" in encoded_inputs:
											__lowercase 					=   encoded_inputs[self.model_input_names[0]]
											# `global_attention_mask` need to have the same length as other (sequential) inputs.
											__lowercase 					=   len(encoded_inputs['''global_attention_mask''']						) != len(snake_case_						)

											if needs_to_be_padded:
														__lowercase 					=   len(snake_case_						) - len(encoded_inputs['''global_attention_mask''']						)

														if self.padding_side == "right":
																	# Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
																	__lowercase 					=   (
																	    encoded_inputs['''global_attention_mask'''] + [-1] * difference
																	)
														elif self.padding_side == "left":
																	__lowercase 					=   [-1] * difference + encoded_inputs[
																	    '''global_attention_mask'''
																	]
														else:
																	raise ValueError('''Invalid padding strategy:''' + str(self.padding_side						)						)

								return encoded_inputs


