





import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


UpperCAmelCase_  							=				logging.get_logger(__name__)


UpperCAmelCase_  							=				{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

# See all LED models at https://huggingface.co/models?filter=LED
UpperCAmelCase_  							=				{
    'vocab_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json',
    },
    'merges_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json',
    },
}

UpperCAmelCase_  							=				{
    'allenai/led-base-16384': 1_6384,
}




@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def 	lowerCAmelCase_					(						)   ->			str:
				UpperCamelCase__							:       Optional[int]      =   (
				    list(range(ord('''!'''   )	,     ord('''~'''   ) + 1   )   ) + list(range(ord('''¡'''   )	,     ord('''¬'''   ) + 1   )   ) + list(range(ord('''®'''   )	,     ord('''ÿ'''   ) + 1   )   )
				)
				UpperCamelCase__							:       List[str]      =   bs[:]
				UpperCamelCase__							:       Dict      =   0
				for b in range(2**8   ):
								if b not in bs:
												bs.append(__UpperCAmelCase   )
												cs.append(2**8 + n   )
												n += 1
				UpperCamelCase__							:       str      =   [chr(__UpperCAmelCase   ) for n in cs]
				return dict(zip(__UpperCAmelCase	,     __UpperCAmelCase   )   )




def 	lowerCAmelCase_					(						__UpperCAmelCase:    Union[str, Any]   )   ->			str:
				UpperCamelCase__							:       Optional[int]      =   set()
				UpperCamelCase__							:       Any      =   word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)   )
								UpperCamelCase__							:       Optional[Any]      =   char
				return pairs






class 					lowercase__ (			__lowerCamelCase       ):

			'''simple docstring'''


			a		:							Union[str, Any]     	=	VOCAB_FILES_NAMES
			a		:							List[Any]     	=	PRETRAINED_VOCAB_FILES_MAP
			a		:							Any     	=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			a		:							Tuple     	=	["input_ids", "attention_mask"]




			def __init__(					self,	__magic_name__,	__magic_name__,	__magic_name__="replace",	__magic_name__="<s>",	__magic_name__="</s>",	__magic_name__="</s>",	__magic_name__="<s>",	__magic_name__="<unk>",	__magic_name__="<pad>",	__magic_name__="<mask>",	__magic_name__=False,	**__magic_name__,	)				->   int:






							"""simple docstring"""
							UpperCamelCase__							:       List[Any]      =   AddedToken(__magic_name__,	lstrip=__magic_name__,	rstrip=__magic_name__							) if isinstance(__magic_name__,	__magic_name__							) else bos_token
							UpperCamelCase__							:       str      =   AddedToken(__magic_name__,	lstrip=__magic_name__,	rstrip=__magic_name__							) if isinstance(__magic_name__,	__magic_name__							) else eos_token
							UpperCamelCase__							:       Dict      =   AddedToken(__magic_name__,	lstrip=__magic_name__,	rstrip=__magic_name__							) if isinstance(__magic_name__,	__magic_name__							) else sep_token
							UpperCamelCase__							:       Any      =   AddedToken(__magic_name__,	lstrip=__magic_name__,	rstrip=__magic_name__							) if isinstance(__magic_name__,	__magic_name__							) else cls_token
							UpperCamelCase__							:       Dict      =   AddedToken(__magic_name__,	lstrip=__magic_name__,	rstrip=__magic_name__							) if isinstance(__magic_name__,	__magic_name__							) else unk_token
							UpperCamelCase__							:       Any      =   AddedToken(__magic_name__,	lstrip=__magic_name__,	rstrip=__magic_name__							) if isinstance(__magic_name__,	__magic_name__							) else pad_token

							# Mask token behave like a normal word, i.e. include the space before it
							UpperCamelCase__							:       Union[str, Any]      =   AddedToken(__magic_name__,	lstrip=__magic_name__,	rstrip=__magic_name__							) if isinstance(__magic_name__,	__magic_name__							) else mask_token

							super().__init__(
							    errors=__magic_name__,	bos_token=__magic_name__,	eos_token=__magic_name__,	unk_token=__magic_name__,	sep_token=__magic_name__,	cls_token=__magic_name__,	pad_token=__magic_name__,	mask_token=__magic_name__,	add_prefix_space=__magic_name__,	**__magic_name__,	)

							with open(__magic_name__,	encoding='''utf-8'''							) as vocab_handle:
											UpperCamelCase__							:       int      =   json.load(__magic_name__							)
							UpperCamelCase__							:       Dict      =   {v: k for k, v in self.encoder.items()}
							UpperCamelCase__							:       str      =   errors  # how to handle errors in decoding
							UpperCamelCase__							:       Tuple      =   bytes_to_unicode()
							UpperCamelCase__							:       Any      =   {v: k for k, v in self.byte_encoder.items()}
							with open(__magic_name__,	encoding='''utf-8'''							) as merges_handle:
											UpperCamelCase__							:       str      =   merges_handle.read().split('''\n'''							)[1:-1]
							UpperCamelCase__							:       Dict      =   [tuple(merge.split()							) for merge in bpe_merges]
							UpperCamelCase__							:       Tuple      =   dict(zip(__magic_name__,	range(len(__magic_name__							)							)							)							)
							UpperCamelCase__							:       int      =   {}
							UpperCamelCase__							:       str      =   add_prefix_space

							# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
							UpperCamelCase__							:       Optional[int]      =   re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''							)




			@property
			# Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
			def    UpperCamelCase__						(					self							)				->   Any:






							"""simple docstring"""
							return len(self.encoder							)




			def    UpperCamelCase__						(					self							)				->   Optional[int]:






							"""simple docstring"""
							return dict(self.encoder,	**self.added_tokens_encoder							)




			def    UpperCamelCase__						(					self,	__magic_name__							)				->   List[Any]:






							"""simple docstring"""
							if token in self.cache:
											return self.cache[token]
							UpperCamelCase__							:       Optional[Any]      =   tuple(__magic_name__							)
							UpperCamelCase__							:       List[Any]      =   get_pairs(__magic_name__							)

							if not pairs:
											return token

							while True:
											UpperCamelCase__							:       str      =   min(__magic_name__,	key=lambda __magic_name__							: self.bpe_ranks.get(__magic_name__,	float('''inf'''							)							)							)
											if bigram not in self.bpe_ranks:
															break
											UpperCamelCase__     ,UpperCamelCase__							:       List[Any]      =   bigram
											UpperCamelCase__							:       Optional[int]      =   []
											UpperCamelCase__							:       Dict      =   0
											while i < len(__magic_name__							):
															try:
																			UpperCamelCase__							:       Optional[Any]      =   word.index(__magic_name__,	__magic_name__							)
															except ValueError:
																			new_word.extend(word[i:]							)
																			break
															else:
																			new_word.extend(word[i:j]							)
																			UpperCamelCase__							:       Optional[Any]      =   j

															if word[i] == first and i < len(__magic_name__							) - 1 and word[i + 1] == second:
																			new_word.append(first + second							)
																			i += 2
															else:
																			new_word.append(word[i]							)
																			i += 1
											UpperCamelCase__							:       List[Any]      =   tuple(__magic_name__							)
											UpperCamelCase__							:       Tuple      =   new_word
											if len(__magic_name__							) == 1:
															break
											else:
															UpperCamelCase__							:       Tuple      =   get_pairs(__magic_name__							)
							UpperCamelCase__							:       int      =   ''' '''.join(__magic_name__							)
							UpperCamelCase__							:       Optional[Any]      =   word
							return word




			def    UpperCamelCase__						(					self,	__magic_name__							)				->   int:






							"""simple docstring"""
							UpperCamelCase__							:       Any      =   []
							for token in re.findall(self.pat,	__magic_name__							):
											UpperCamelCase__							:       Optional[Any]      =   ''''''.join(
											    self.byte_encoder[b] for b in token.encode('''utf-8'''							)							)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
											bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__magic_name__							).split(''' '''							)							)
							return bpe_tokens




			def    UpperCamelCase__						(					self,	__magic_name__							)				->   Tuple:






							"""simple docstring"""
							return self.encoder.get(__magic_name__,	self.encoder.get(self.unk_token							)							)




			def    UpperCamelCase__						(					self,	__magic_name__							)				->   Dict:






							"""simple docstring"""
							return self.decoder.get(__magic_name__							)




			def    UpperCamelCase__						(					self,	__magic_name__							)				->   Optional[int]:






							"""simple docstring"""
							UpperCamelCase__							:       List[Any]      =   ''''''.join(__magic_name__							)
							UpperCamelCase__							:       List[str]      =   bytearray([self.byte_decoder[c] for c in text]							).decode('''utf-8''',	errors=self.errors							)
							return text




			def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__ = None							)				->   Tuple[str]:






							"""simple docstring"""
							if not os.path.isdir(__magic_name__							):
											logger.error(f"Vocabulary path ({save_directory}) should be a directory"							)
											return
							UpperCamelCase__							:       List[str]      =   os.path.join(
							    __magic_name__,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)
							UpperCamelCase__							:       int      =   os.path.join(
							    __magic_name__,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']							)

							with open(__magic_name__,	'''w''',	encoding='''utf-8'''							) as f:
											f.write(json.dumps(self.encoder,	indent=2,	sort_keys=__magic_name__,	ensure_ascii=__magic_name__							) + '''\n'''							)

							UpperCamelCase__							:       List[str]      =   0
							with open(__magic_name__,	'''w''',	encoding='''utf-8'''							) as writer:
											writer.write('''#version: 0.2\n'''							)
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),	key=lambda __magic_name__							: kv[1]							):
															if index != token_index:
																			logger.warning(
																			    f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
																			    ''' Please check that the tokenizer is not corrupted!'''							)
																			UpperCamelCase__							:       Dict      =   token_index
															writer.write(''' '''.join(__magic_name__							) + '''\n'''							)
															index += 1

							return vocab_file, merge_file




			def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__ = None							)				->   List[int]:






							"""simple docstring"""
							if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							UpperCamelCase__							:       Optional[Any]      =   [self.cls_token_id]
							UpperCamelCase__							:       Dict      =   [self.sep_token_id]
							return cls + token_ids_a + sep + sep + token_ids_a + sep




			def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__ = None,	__magic_name__ = False							)				->   List[int]:






							"""simple docstring"""
							if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=__magic_name__,	token_ids_a=__magic_name__,	already_has_special_tokens=__magic_name__							)

							if token_ids_a is None:
											return [1] + ([0] * len(__magic_name__							)) + [1]
							return [1] + ([0] * len(__magic_name__							)) + [1, 1] + ([0] * len(__magic_name__							)) + [1]




			def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__ = None							)				->   List[int]:






							"""simple docstring"""
							UpperCamelCase__							:       List[Any]      =   [self.sep_token_id]
							UpperCamelCase__							:       Dict      =   [self.cls_token_id]

							if token_ids_a is None:
											return len(cls + token_ids_a + sep							) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]




			def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__=False,	**__magic_name__							)				->   List[Any]:






							"""simple docstring"""
							UpperCamelCase__							:       List[str]      =   kwargs.pop('''add_prefix_space''',	self.add_prefix_space							)
							if (is_split_into_words or add_prefix_space) and (len(__magic_name__							) > 0 and not text[0].isspace()):
											UpperCamelCase__							:       List[Any]      =   ''' ''' + text
							return (text, kwargs)




			def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__ = None,	__magic_name__ = PaddingStrategy.DO_NOT_PAD,	__magic_name__ = None,	__magic_name__ = None,	)				->   dict:






							"""simple docstring"""
							UpperCamelCase__							:       str      =   super()._pad(
							    encoded_inputs=__magic_name__,	max_length=__magic_name__,	padding_strategy=__magic_name__,	pad_to_multiple_of=__magic_name__,	return_attention_mask=__magic_name__,	)

							# Load from model defaults
							if return_attention_mask is None:
											UpperCamelCase__							:       Optional[Any]      =   '''attention_mask''' in self.model_input_names

							if return_attention_mask and "global_attention_mask" in encoded_inputs:
											UpperCamelCase__							:       int      =   encoded_inputs[self.model_input_names[0]]
											# `global_attention_mask` need to have the same length as other (sequential) inputs.
											UpperCamelCase__							:       str      =   len(encoded_inputs['''global_attention_mask''']							) != len(__magic_name__							)

											if needs_to_be_padded:
															UpperCamelCase__							:       Union[str, Any]      =   len(__magic_name__							) - len(encoded_inputs['''global_attention_mask''']							)

															if self.padding_side == "right":
																			# Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
																			UpperCamelCase__							:       str      =   (
																			    encoded_inputs['''global_attention_mask'''] + [-1] * difference
																			)
															elif self.padding_side == "left":
																			UpperCamelCase__							:       Tuple      =   [-1] * difference + encoded_inputs[
																			    '''global_attention_mask'''
																			]
															else:
																			raise ValueError('''Invalid padding strategy:''' + str(self.padding_side							)							)

							return encoded_inputs








import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
  if is_torch_available():
    import torch
  if is_tf_available():
    import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


UpperCAmelCase_  							=				logging.get_logger(__name__)

UpperCAmelCase_  							=				{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase_  							=				{
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'Salesforce/codegen-350M-mono': (
            'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'
        ),
    },
}

UpperCAmelCase_  							=				{
    'Salesforce/codegen-350M-mono': 2048,
}






class 					lowercase__ (			__lowerCamelCase       ):

   '''simple docstring'''


   a		:							Union[str, Any]     	=	VOCAB_FILES_NAMES
   a		:							List[str]     	=	PRETRAINED_VOCAB_FILES_MAP
   a		:							Optional[int]     	=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   a		:							List[Any]     	=	["input_ids", "attention_mask"]
   a		:							Any     	=	CodeGenTokenizer




   def __init__(					self,	__magic_name__=None,	__magic_name__=None,	__magic_name__=None,	__magic_name__="<|endoftext|>",	__magic_name__="<|endoftext|>",	__magic_name__="<|endoftext|>",	__magic_name__=False,	**__magic_name__,	)				->   Optional[int]:






       """simple docstring"""
       super().__init__(
           __magic_name__,	__magic_name__,	tokenizer_file=__magic_name__,	unk_token=__magic_name__,	bos_token=__magic_name__,	eos_token=__magic_name__,	add_prefix_space=__magic_name__,	**__magic_name__,	)

       if kwargs.pop('''add_bos_token''',	__magic_name__							):
           UpperCamelCase__							:       List[Any]      =   kwargs.pop('''name_or_path''',	''''''							)
           raise ValueError(
               '''Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'''
               '''Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'''
               f"`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"
               f"`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"
               '''This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'''
               ''' so that the fast tokenizer works correctly.'''							)

       UpperCamelCase__							:       Dict      =   json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
       if pre_tok_state.get('''add_prefix_space''',	__magic_name__							) != add_prefix_space:
           UpperCamelCase__							:       List[str]      =   getattr(__magic_name__,	pre_tok_state.pop('''type'''							)							)
           UpperCamelCase__							:       int      =   add_prefix_space
           UpperCamelCase__							:       List[str]      =   pre_tok_class(**__magic_name__							)

       UpperCamelCase__							:       List[Any]      =   add_prefix_space




   def    UpperCamelCase__						(					self,	*__magic_name__,	**__magic_name__							)				->   BatchEncoding:






       """simple docstring"""
       UpperCamelCase__							:       List[Any]      =   kwargs.get('''is_split_into_words''',	__magic_name__							)
       assert self.add_prefix_space or not is_split_into_words, (
           f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
           "to use it with pretokenized inputs."
       )

       return super()._batch_encode_plus(*__magic_name__,	**__magic_name__							)




   def    UpperCamelCase__						(					self,	*__magic_name__,	**__magic_name__							)				->   BatchEncoding:






       """simple docstring"""
       UpperCamelCase__							:       Dict      =   kwargs.get('''is_split_into_words''',	__magic_name__							)

       assert self.add_prefix_space or not is_split_into_words, (
           f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
           "to use it with pretokenized inputs."
       )

       return super()._encode_plus(*__magic_name__,	**__magic_name__							)




   def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__ = None							)				->   Tuple[str]:






       """simple docstring"""
       UpperCamelCase__							:       List[Any]      =   self._tokenizer.model.save(__magic_name__,	name=__magic_name__							)
       return tuple(__magic_name__							)




   def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__ = False,	__magic_name__ = None,	__magic_name__ = None,	**__magic_name__,	)				->   str:






       """simple docstring"""
       UpperCamelCase__							:       Dict      =   super().decode(
           token_ids=__magic_name__,	skip_special_tokens=__magic_name__,	clean_up_tokenization_spaces=__magic_name__,	**__magic_name__,	)

       if truncate_before_pattern is not None and len(__magic_name__							) > 0:
           UpperCamelCase__							:       str      =   self.truncate(__magic_name__,	__magic_name__							)

       return decoded_text




   def    UpperCamelCase__						(					self,	__magic_name__,	__magic_name__							)				->   List[str]:






       """simple docstring"""
       def find_re(__magic_name__,	__magic_name__,	__magic_name__							):
           UpperCamelCase__							:       Dict      =   pattern.search(__magic_name__,	__magic_name__							)
           return m.start() if m else -1

       UpperCamelCase__							:       Union[str, Any]      =   [re.compile(__magic_name__,	re.MULTILINE							) for pattern in truncate_before_pattern]

       UpperCamelCase__							:       List[Any]      =   list(re.finditer('''^print''',	__magic_name__,	re.MULTILINE							)							)

       if len(__magic_name__							) > 1:
           UpperCamelCase__							:       Tuple      =   completion[: prints[1].start()]

       UpperCamelCase__							:       Optional[int]      =   list(re.finditer('''^def''',	__magic_name__,	re.MULTILINE							)							)

       if len(__magic_name__							) > 1:
           UpperCamelCase__							:       Optional[Any]      =   completion[: defs[1].start()]

       UpperCamelCase__							:       str      =   0

       UpperCamelCase__							:       Tuple      =   [
           pos for pos in [find_re(__magic_name__,	__magic_name__,	__magic_name__							) for terminal in terminals] if pos != -1
       ]

       if len(__magic_name__							) > 0:
           return completion[: min(__magic_name__							)]
       else:
           return completion

