


'''simple docstring'''
import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


__snake_case				 =	logging.get_logger(__name__)


__snake_case				 =	{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

# See all LED models at https://huggingface.co/models?filter=LED
__snake_case				 =	{
    '''vocab_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json''',
    },
}

__snake_case				 =	{
    '''allenai/led-base-16384''': 16384,
}


@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def      a (     )    ->      Any:



					'''simple docstring'''






					UpperCamelCase__  :Union[str, Any]				   =					(
					    list(range(ord('''!'''  )					,   ord('''~'''  ) + 1  )  ) + list(range(ord('''¡'''  )					,   ord('''¬'''  ) + 1  )  ) + list(range(ord('''®'''  )					,   ord('''ÿ'''  ) + 1  )  )
					)
					UpperCamelCase__  :Tuple				   =					bs[:]
					UpperCamelCase__  :Tuple				   =					0
					for b in range(2**8  ):
										if b not in bs:
															bs.append(__a  )
															cs.append(2**8 + n  )
															n += 1
					UpperCamelCase__  :List[Any]				   =					[chr(__a  ) for n in cs]
					return dict(zip(__a					,   __a  )  )


def      a (     __a  )    ->      str:



					'''simple docstring'''






					UpperCamelCase__  :Union[str, Any]				   =					set()
					UpperCamelCase__  :Union[str, Any]				   =					word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)  )
										UpperCamelCase__  :List[str]				   =					char
					return pairs







class   lowercase      (				A__   ):

		"""simple docstring"""

		_a       					=							VOCAB_FILES_NAMES
		_a       					=							PRETRAINED_VOCAB_FILES_MAP
		_a       					=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_a       					=							['input_ids', 'attention_mask']




		def __init__(						self       ,  UpperCamelCase_       ,  UpperCamelCase_       ,  UpperCamelCase_="replace"       ,  UpperCamelCase_="<s>"       ,  UpperCamelCase_="</s>"       ,  UpperCamelCase_="</s>"       ,  UpperCamelCase_="<s>"       ,  UpperCamelCase_="<unk>"       ,  UpperCamelCase_="<pad>"       ,  UpperCamelCase_="<mask>"       ,  UpperCamelCase_=False       ,  **UpperCamelCase_       ,  ):

							'''simple docstring'''


							UpperCamelCase__  :Tuple				   =					AddedToken(UpperCamelCase_       ,  lstrip=UpperCamelCase_       ,  rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_       ,  UpperCamelCase_				) else bos_token
							UpperCamelCase__  :str				   =					AddedToken(UpperCamelCase_       ,  lstrip=UpperCamelCase_       ,  rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_       ,  UpperCamelCase_				) else eos_token
							UpperCamelCase__  :List[Any]				   =					AddedToken(UpperCamelCase_       ,  lstrip=UpperCamelCase_       ,  rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_       ,  UpperCamelCase_				) else sep_token
							UpperCamelCase__  :Tuple				   =					AddedToken(UpperCamelCase_       ,  lstrip=UpperCamelCase_       ,  rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_       ,  UpperCamelCase_				) else cls_token
							UpperCamelCase__  :List[str]				   =					AddedToken(UpperCamelCase_       ,  lstrip=UpperCamelCase_       ,  rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_       ,  UpperCamelCase_				) else unk_token
							UpperCamelCase__  :List[str]				   =					AddedToken(UpperCamelCase_       ,  lstrip=UpperCamelCase_       ,  rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_       ,  UpperCamelCase_				) else pad_token

							# Mask token behave like a normal word, i.e. include the space before it
							UpperCamelCase__  :Union[str, Any]				   =					AddedToken(UpperCamelCase_       ,  lstrip=UpperCamelCase_       ,  rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_       ,  UpperCamelCase_				) else mask_token

							super().__init__(
							    errors=UpperCamelCase_       ,  bos_token=UpperCamelCase_       ,  eos_token=UpperCamelCase_       ,  unk_token=UpperCamelCase_       ,  sep_token=UpperCamelCase_       ,  cls_token=UpperCamelCase_       ,  pad_token=UpperCamelCase_       ,  mask_token=UpperCamelCase_       ,  add_prefix_space=UpperCamelCase_       ,  **UpperCamelCase_       ,  )

							with open(UpperCamelCase_       ,  encoding='''utf-8'''				) as vocab_handle:
												UpperCamelCase__  :Dict				   =					json.load(UpperCamelCase_				)
							UpperCamelCase__  :int				   =					{v: k for k, v in self.encoder.items()}
							UpperCamelCase__  :List[Any]				   =					errors  # how to handle errors in decoding
							UpperCamelCase__  :int				   =					bytes_to_unicode()
							UpperCamelCase__  :List[str]				   =					{v: k for k, v in self.byte_encoder.items()}
							with open(UpperCamelCase_       ,  encoding='''utf-8'''				) as merges_handle:
												UpperCamelCase__  :List[str]				   =					merges_handle.read().split('''\n'''				)[1:-1]
							UpperCamelCase__  :List[Any]				   =					[tuple(merge.split()				) for merge in bpe_merges]
							UpperCamelCase__  :List[str]				   =					dict(zip(UpperCamelCase_       ,  range(len(UpperCamelCase_				)				)				)				)
							UpperCamelCase__  :Optional[int]				   =					{}
							UpperCamelCase__  :Optional[int]				   =					add_prefix_space

							# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
							UpperCamelCase__  :List[str]				   =					re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''				)




		@property
		# Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							return len(self.encoder				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							return dict(self.encoder       ,  **self.added_tokens_encoder				)




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_				):

							'''simple docstring'''


							if token in self.cache:
												return self.cache[token]
							UpperCamelCase__  :Union[str, Any]				   =					tuple(UpperCamelCase_				)
							UpperCamelCase__  :Any				   =					get_pairs(UpperCamelCase_				)

							if not pairs:
												return token

							while True:
												UpperCamelCase__  :List[Any]				   =					min(UpperCamelCase_       ,  key=lambda UpperCamelCase_				: self.bpe_ranks.get(UpperCamelCase_       ,  float('''inf'''				)				)				)
												if bigram not in self.bpe_ranks:
																	break
												UpperCamelCase__   ,							UpperCamelCase__  :Dict				   =					bigram
												UpperCamelCase__  :Optional[Any]				   =					[]
												UpperCamelCase__  :Union[str, Any]				   =					0
												while i < len(UpperCamelCase_				):
																	try:
																						UpperCamelCase__  :Optional[Any]				   =					word.index(UpperCamelCase_       ,  UpperCamelCase_				)
																	except ValueError:
																						new_word.extend(word[i:]				)
																						break
																	else:
																						new_word.extend(word[i:j]				)
																						UpperCamelCase__  :List[Any]				   =					j

																	if word[i] == first and i < len(UpperCamelCase_				) - 1 and word[i + 1] == second:
																						new_word.append(first + second				)
																						i += 2
																	else:
																						new_word.append(word[i]				)
																						i += 1
												UpperCamelCase__  :List[Any]				   =					tuple(UpperCamelCase_				)
												UpperCamelCase__  :Tuple				   =					new_word
												if len(UpperCamelCase_				) == 1:
																	break
												else:
																	UpperCamelCase__  :List[str]				   =					get_pairs(UpperCamelCase_				)
							UpperCamelCase__  :Dict				   =					''' '''.join(UpperCamelCase_				)
							UpperCamelCase__  :Dict				   =					word
							return word




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_				):

							'''simple docstring'''


							UpperCamelCase__  :str				   =					[]
							for token in re.findall(self.pat       ,  UpperCamelCase_				):
												UpperCamelCase__  :Tuple				   =					''''''.join(
												    self.byte_encoder[b] for b in token.encode('''utf-8'''				)				)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
												bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCamelCase_				).split(''' '''				)				)
							return bpe_tokens




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_				):

							'''simple docstring'''


							return self.encoder.get(UpperCamelCase_       ,  self.encoder.get(self.unk_token				)				)




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_				):

							'''simple docstring'''


							return self.decoder.get(UpperCamelCase_				)




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_				):

							'''simple docstring'''


							UpperCamelCase__  :Tuple				   =					''''''.join(UpperCamelCase_				)
							UpperCamelCase__  :int				   =					bytearray([self.byte_decoder[c] for c in text]				).decode('''utf-8'''       ,  errors=self.errors				)
							return text




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_       ,  UpperCamelCase_ = None				):

							'''simple docstring'''


							if not os.path.isdir(UpperCamelCase_				):
												logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''				)
												return
							UpperCamelCase__  :Union[str, Any]				   =					os.path.join(
							    UpperCamelCase_       ,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)
							UpperCamelCase__  :List[Any]				   =					os.path.join(
							    UpperCamelCase_       ,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']				)

							with open(UpperCamelCase_       ,  '''w'''       ,  encoding='''utf-8'''				) as f:
												f.write(json.dumps(self.encoder       ,  indent=2       ,  sort_keys=UpperCamelCase_       ,  ensure_ascii=UpperCamelCase_				) + '''\n'''				)

							UpperCamelCase__  :Tuple				   =					0
							with open(UpperCamelCase_       ,  '''w'''       ,  encoding='''utf-8'''				) as writer:
												writer.write('''#version: 0.2\n'''				)
												for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,  key=lambda UpperCamelCase_				: kv[1]				):
																	if index != token_index:
																						logger.warning(
																						    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																						    ''' Please check that the tokenizer is not corrupted!'''				)
																						UpperCamelCase__  :Optional[int]				   =					token_index
																	writer.write(''' '''.join(UpperCamelCase_				) + '''\n'''				)
																	index += 1

							return vocab_file, merge_file




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_       ,  UpperCamelCase_ = None				):

							'''simple docstring'''


							if token_ids_a is None:
												return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							UpperCamelCase__  :Tuple				   =					[self.cls_token_id]
							UpperCamelCase__  :Dict				   =					[self.sep_token_id]
							return cls + token_ids_a + sep + sep + token_ids_a + sep




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_       ,  UpperCamelCase_ = None       ,  UpperCamelCase_ = False				):

							'''simple docstring'''


							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=UpperCamelCase_       ,  token_ids_a=UpperCamelCase_       ,  already_has_special_tokens=UpperCamelCase_				)

							if token_ids_a is None:
												return [1] + ([0] * len(UpperCamelCase_				)) + [1]
							return [1] + ([0] * len(UpperCamelCase_				)) + [1, 1] + ([0] * len(UpperCamelCase_				)) + [1]




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_       ,  UpperCamelCase_ = None				):

							'''simple docstring'''


							UpperCamelCase__  :Any				   =					[self.sep_token_id]
							UpperCamelCase__  :Any				   =					[self.cls_token_id]

							if token_ids_a is None:
												return len(cls + token_ids_a + sep				) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_       ,  UpperCamelCase_=False       ,  **UpperCamelCase_				):

							'''simple docstring'''


							UpperCamelCase__  :Union[str, Any]				   =					kwargs.pop('''add_prefix_space'''       ,  self.add_prefix_space				)
							if (is_split_into_words or add_prefix_space) and (len(UpperCamelCase_				) > 0 and not text[0].isspace()):
												UpperCamelCase__  :List[Any]				   =					''' ''' + text
							return (text, kwargs)




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_       ,  UpperCamelCase_ = None       ,  UpperCamelCase_ = PaddingStrategy.DO_NOT_PAD       ,  UpperCamelCase_ = None       ,  UpperCamelCase_ = None       ,  ):

							'''simple docstring'''


							UpperCamelCase__  :int				   =					super()._pad(
							    encoded_inputs=UpperCamelCase_       ,  max_length=UpperCamelCase_       ,  padding_strategy=UpperCamelCase_       ,  pad_to_multiple_of=UpperCamelCase_       ,  return_attention_mask=UpperCamelCase_       ,  )

							# Load from model defaults
							if return_attention_mask is None:
												UpperCamelCase__  :Dict				   =					'''attention_mask''' in self.model_input_names

							if return_attention_mask and "global_attention_mask" in encoded_inputs:
												UpperCamelCase__  :Optional[int]				   =					encoded_inputs[self.model_input_names[0]]
												# `global_attention_mask` need to have the same length as other (sequential) inputs.
												UpperCamelCase__  :Union[str, Any]				   =					len(encoded_inputs['''global_attention_mask''']				) != len(UpperCamelCase_				)

												if needs_to_be_padded:
																	UpperCamelCase__  :str				   =					len(UpperCamelCase_				) - len(encoded_inputs['''global_attention_mask''']				)

																	if self.padding_side == "right":
																						# Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
																						UpperCamelCase__  :Optional[int]				   =					(
																						    encoded_inputs['''global_attention_mask'''] + [-1] * difference
																						)
																	elif self.padding_side == "left":
																						UpperCamelCase__  :int				   =					[-1] * difference + encoded_inputs[
																						    '''global_attention_mask'''
																						]
																	else:
																						raise ValueError('''Invalid padding strategy:''' + str(self.padding_side				)				)

							return encoded_inputs



'''simple docstring'''
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__snake_case				 =	{
    '''configuration_convbert''': ['''CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''ConvBertConfig''', '''ConvBertOnnxConfig'''],
    '''tokenization_convbert''': ['''ConvBertTokenizer'''],
}

try:
		if not is_tokenizers_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__snake_case				 =	['''ConvBertTokenizerFast''']

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__snake_case				 =	[
		    '''CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''ConvBertForMaskedLM''',
		    '''ConvBertForMultipleChoice''',
		    '''ConvBertForQuestionAnswering''',
		    '''ConvBertForSequenceClassification''',
		    '''ConvBertForTokenClassification''',
		    '''ConvBertLayer''',
		    '''ConvBertModel''',
		    '''ConvBertPreTrainedModel''',
		    '''load_tf_weights_in_convbert''',
		]


try:
		if not is_tf_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__snake_case				 =	[
		    '''TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''TFConvBertForMaskedLM''',
		    '''TFConvBertForMultipleChoice''',
		    '''TFConvBertForQuestionAnswering''',
		    '''TFConvBertForSequenceClassification''',
		    '''TFConvBertForTokenClassification''',
		    '''TFConvBertLayer''',
		    '''TFConvBertModel''',
		    '''TFConvBertPreTrainedModel''',
		]


if TYPE_CHECKING:
		from .configuration_convbert import CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvBertConfig, ConvBertOnnxConfig
		from .tokenization_convbert import ConvBertTokenizer

		try:
				if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_convbert_fast import ConvBertTokenizerFast

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_convbert import (
				    CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    ConvBertForMaskedLM,
				    ConvBertForMultipleChoice,
				    ConvBertForQuestionAnswering,
				    ConvBertForSequenceClassification,
				    ConvBertForTokenClassification,
				    ConvBertLayer,
				    ConvBertModel,
				    ConvBertPreTrainedModel,
				    load_tf_weights_in_convbert,
				)

		try:
				if not is_tf_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_tf_convbert import (
				    TF_CONVBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFConvBertForMaskedLM,
				    TFConvBertForMultipleChoice,
				    TFConvBertForQuestionAnswering,
				    TFConvBertForSequenceClassification,
				    TFConvBertForTokenClassification,
				    TFConvBertLayer,
				    TFConvBertModel,
				    TFConvBertPreTrainedModel,
				)


else:
		import sys

		__snake_case				 =	_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)