





import json
from typing import Dict, List, Optional, Tuple, Union

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding, EncodedInput
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_led import LEDTokenizer


_lowercase		:  Union[str, Any]			       =logging.get_logger(__name__)


_lowercase		:  Dict			       ={"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

_lowercase		:  Union[str, Any]			       ={
    """vocab_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json""",
    },
}

_lowercase		:  Tuple			       ={
    """allenai/led-base-16384""": 1_6384,
}






class 						UpperCamelCase_  (  snake_case__							):
					_a      :						Optional[int]						    =					VOCAB_FILES_NAMES
					_a      :						Optional[int]						    =					PRETRAINED_VOCAB_FILES_MAP
					_a      :						Optional[Any]						    =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					_a      :						List[str]						    =					LEDTokenizer
					_a      :						int						    =					['input_ids', 'attention_mask']



					def __init__(		self     :     str       ,      lowerCamelCase     :     Optional[int]=None       ,      lowerCamelCase     :     List[str]=None       ,      lowerCamelCase     :     Union[str, Any]=None       ,      lowerCamelCase     :     Union[str, Any]="replace"       ,      lowerCamelCase     :     Union[str, Any]="<s>"       ,      lowerCamelCase     :     List[str]="</s>"       ,      lowerCamelCase     :     List[Any]="</s>"       ,      lowerCamelCase     :     int="<s>"       ,      lowerCamelCase     :     Tuple="<unk>"       ,      lowerCamelCase     :     List[str]="<pad>"       ,      lowerCamelCase     :     Optional[Any]="<mask>"       ,      lowerCamelCase     :     Dict=False       ,      lowerCamelCase     :     List[str]=True       ,      **lowerCamelCase     :     Optional[int]       ,      ):
							super().__init__(
							    lowerCamelCase       ,      lowerCamelCase       ,      tokenizer_file=lowerCamelCase       ,      errors=lowerCamelCase       ,      bos_token=lowerCamelCase       ,      eos_token=lowerCamelCase       ,      sep_token=lowerCamelCase       ,      cls_token=lowerCamelCase       ,      unk_token=lowerCamelCase       ,      pad_token=lowerCamelCase       ,      mask_token=lowerCamelCase       ,      add_prefix_space=lowerCamelCase       ,      trim_offsets=lowerCamelCase       ,      **lowerCamelCase       ,      )

							lowerCamelCase_	:     Tuple 						=      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
							if pre_tok_state.get('add_prefix_space'       ,      lowerCamelCase						) != add_prefix_space:
									lowerCamelCase_	:     Optional[Any] 						=      getattr(lowerCamelCase       ,      pre_tok_state.pop('type'						)						)
									lowerCamelCase_	:     Union[str, Any] 						=      add_prefix_space
									lowerCamelCase_	:     str 						=      pre_tok_class(**lowerCamelCase						)

							lowerCamelCase_	:     Optional[int] 						=      add_prefix_space

							# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
							lowerCamelCase_	:     List[Any] 						=      'post_processor'
							lowerCamelCase_	:     Dict 						=      getattr(self.backend_tokenizer       ,      lowerCamelCase       ,      lowerCamelCase						)
							if tokenizer_component_instance:
									lowerCamelCase_	:     str 						=      json.loads(tokenizer_component_instance.__getstate__()						)

									# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
									if "sep" in state:
											lowerCamelCase_	:     Tuple 						=      tuple(state['sep']						)
									if "cls" in state:
											lowerCamelCase_	:     Optional[int] 						=      tuple(state['cls']						)

									lowerCamelCase_	:     Tuple 						=      False

									if state.get('add_prefix_space'       ,      lowerCamelCase						) != add_prefix_space:
											lowerCamelCase_	:     Dict 						=      add_prefix_space
											lowerCamelCase_	:     Dict 						=      True

									if state.get('trim_offsets'       ,      lowerCamelCase						) != trim_offsets:
											lowerCamelCase_	:     List[str] 						=      trim_offsets
											lowerCamelCase_	:     Tuple 						=      True

									if changes_to_apply:
											lowerCamelCase_	:     Union[str, Any] 						=      getattr(lowerCamelCase       ,      state.pop('type'						)						)
											lowerCamelCase_	:     str 						=      component_class(**lowerCamelCase						)
											setattr(self.backend_tokenizer       ,      lowerCamelCase       ,      lowerCamelCase						)



					@property
					# Copied from transformers.models.bart.tokenization_bart_fast.BartTokenizerFast.mask_token with BART->LED
					def   __a     (		self     :     Optional[int]						):
							if self._mask_token is None:
									if self.verbose:
											logger.error('Using mask_token, but it is not set yet.'						)
									return None
							return str(self._mask_token						)



					@mask_token.setter
					def   __a     (		self     :     int       ,      lowerCamelCase     :     Any						):
							lowerCamelCase_	:     Optional[int] 						=      AddedToken(lowerCamelCase       ,      lstrip=lowerCamelCase       ,      rstrip=lowerCamelCase						) if isinstance(lowerCamelCase       ,      lowerCamelCase						) else value
							lowerCamelCase_	:     Any 						=      value



					def   __a     (		self     :     Optional[int]       ,      *lowerCamelCase     :     Optional[int]       ,      **lowerCamelCase     :     int						):
							lowerCamelCase_	:     List[Any] 						=      kwargs.get('is_split_into_words'       ,      lowerCamelCase						)

							if is_split_into_words and not self.add_prefix_space:
									raise ValueError(
									    F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
									    'to use it with pretokenized inputs.'						)

							return super()._batch_encode_plus(*lowerCamelCase       ,      **lowerCamelCase						)



					def   __a     (		self     :     Optional[Any]       ,      *lowerCamelCase     :     int       ,      **lowerCamelCase     :     str						):
							lowerCamelCase_	:     Tuple 						=      kwargs.get('is_split_into_words'       ,      lowerCamelCase						)

							if is_split_into_words and not self.add_prefix_space:
									raise ValueError(
									    F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
									    'to use it with pretokenized inputs.'						)

							return super()._encode_plus(*lowerCamelCase       ,      **lowerCamelCase						)



					def   __a     (		self     :     List[str]       ,      lowerCamelCase     :     str       ,      lowerCamelCase     :     Optional[str] = None						):
							lowerCamelCase_	:     List[str] 						=      self._tokenizer.model.save(lowerCamelCase       ,      name=lowerCamelCase						)
							return tuple(lowerCamelCase						)



					def   __a     (		self     :     Union[str, Any]       ,      lowerCamelCase     :     Union[str, Any]       ,      lowerCamelCase     :     Tuple=None						):
							lowerCamelCase_	:     Any 						=      [self.bos_token_id] + token_ids_a + [self.eos_token_id]
							if token_ids_a is None:
									return output

							return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]



					def   __a     (		self     :     Any       ,      lowerCamelCase     :     List[int]       ,      lowerCamelCase     :     Optional[List[int]] = None						):
							lowerCamelCase_	:     Any 						=      [self.sep_token_id]
							lowerCamelCase_	:     List[Any] 						=      [self.cls_token_id]

							if token_ids_a is None:
									return len(cls + token_ids_a + sep						) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]





					def   __a     (		self     :     List[str]       ,      lowerCamelCase     :     Union[Dict[str, EncodedInput], BatchEncoding]       ,      lowerCamelCase     :     Optional[int] = None       ,      lowerCamelCase     :     PaddingStrategy = PaddingStrategy.DO_NOT_PAD       ,      lowerCamelCase     :     Optional[int] = None       ,      lowerCamelCase     :     Optional[bool] = None       ,      ):
							lowerCamelCase_	:     Tuple 						=      super()._pad(
							    encoded_inputs=lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding_strategy=lowerCamelCase       ,      pad_to_multiple_of=lowerCamelCase       ,      return_attention_mask=lowerCamelCase       ,      )

							# Load from model defaults
							if return_attention_mask is None:
									lowerCamelCase_	:     List[str] 						=      'attention_mask' in self.model_input_names

							if return_attention_mask and "global_attention_mask" in encoded_inputs:
									lowerCamelCase_	:     Any 						=      encoded_inputs[self.model_input_names[0]]
									# `global_attention_mask` need to have the same length as other (sequential) inputs.
									lowerCamelCase_	:     Any 						=      len(encoded_inputs['global_attention_mask']						) != len(lowerCamelCase						)

									if needs_to_be_padded:
											lowerCamelCase_	:     Optional[Any] 						=      len(lowerCamelCase						) - len(encoded_inputs['global_attention_mask']						)

											if self.padding_side == "right":
													# Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
													lowerCamelCase_	:     int 						=      (
													    encoded_inputs['global_attention_mask'] + [-1] * difference
													)
											elif self.padding_side == "left":
													lowerCamelCase_	:     str 						=      [-1] * difference + encoded_inputs[
													    'global_attention_mask'
													]
											else:
													raise ValueError('Invalid padding strategy:' + str(self.padding_side						)						)

							return encoded_inputs







import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 						UpperCamelCase_  (  snake_case__  ,     unittest.TestCase							):
					_a      :						Optional[Any]						    =					CodeGenTokenizer
					_a      :						Optional[Any]						    =					CodeGenTokenizerFast
					_a      :						Union[str, Any]						    =					True
					_a      :						Dict						    =					{'add_prefix_space': True}
					_a      :						Dict						    =					False



					def   __a     (		self     :     Union[str, Any]						):
							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							lowerCamelCase_	:     Optional[Any] 						=      [
							    'l',
							    'o',
							    'w',
							    'e',
							    'r',
							    's',
							    't',
							    'i',
							    'd',
							    'n',
							    '\u0120',
							    '\u0120l',
							    '\u0120n',
							    '\u0120lo',
							    '\u0120low',
							    'er',
							    '\u0120lowest',
							    '\u0120newer',
							    '\u0120wider',
							    '<unk>',
							    '<|endoftext|>',
							]
							lowerCamelCase_	:     List[Any] 						=      dict(zip(lowerCamelCase       ,      range(len(lowerCamelCase						)						)						)						)
							lowerCamelCase_	:     Tuple 						=      ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
							lowerCamelCase_	:     Optional[int] 						=      {'unk_token': '<unk>'}

							lowerCamelCase_	:     Optional[Any] 						=      os.path.join(self.tmpdirname       ,      VOCAB_FILES_NAMES['vocab_file']						)
							lowerCamelCase_	:     Tuple 						=      os.path.join(self.tmpdirname       ,      VOCAB_FILES_NAMES['merges_file']						)
							with open(self.vocab_file       ,      'w'       ,      encoding='utf-8'						) as fp:
									fp.write(json.dumps(lowerCamelCase						) + '\n'						)
							with open(self.merges_file       ,      'w'       ,      encoding='utf-8'						) as fp:
									fp.write('\n'.join(lowerCamelCase						)						)



					def   __a     (		self     :     Optional[int]       ,      **lowerCamelCase     :     Dict						):
							kwargs.update(self.special_tokens_map						)
							return CodeGenTokenizer.from_pretrained(self.tmpdirname       ,      **lowerCamelCase						)



					def   __a     (		self     :     List[Any]       ,      **lowerCamelCase     :     int						):
							kwargs.update(self.special_tokens_map						)
							return CodeGenTokenizerFast.from_pretrained(self.tmpdirname       ,      **lowerCamelCase						)



					def   __a     (		self     :     Tuple       ,      lowerCamelCase     :     List[str]						):
							lowerCamelCase_	:     List[str] 						=      'lower newer'
							lowerCamelCase_	:     str 						=      'lower newer'
							return input_text, output_text



					def   __a     (		self     :     Optional[int]						):
							lowerCamelCase_	:     List[Any] 						=      CodeGenTokenizer(self.vocab_file       ,      self.merges_file       ,      **self.special_tokens_map						)
							lowerCamelCase_	:     Union[str, Any] 						=      'lower newer'
							lowerCamelCase_	:     List[str] 						=      ['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
							lowerCamelCase_	:     Dict 						=      tokenizer.tokenize(lowerCamelCase       ,      add_prefix_space=lowerCamelCase						)
							self.assertListEqual(lowerCamelCase       ,      lowerCamelCase						)

							lowerCamelCase_	:     Dict 						=      tokens + [tokenizer.unk_token]
							lowerCamelCase_	:     Dict 						=      [14, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase						)       ,      lowerCamelCase						)



					def   __a     (		self     :     str						):
							if not self.test_rust_tokenizer:
									return

							lowerCamelCase_	:     Dict 						=      self.get_tokenizer()
							lowerCamelCase_	:     Optional[Any] 						=      self.get_rust_tokenizer(add_prefix_space=lowerCamelCase						)

							lowerCamelCase_	:     int 						=      'lower newer'

							# Testing tokenization
							lowerCamelCase_	:     List[str] 						=      tokenizer.tokenize(lowerCamelCase       ,      add_prefix_space=lowerCamelCase						)
							lowerCamelCase_	:     List[Any] 						=      rust_tokenizer.tokenize(lowerCamelCase						)
							self.assertListEqual(lowerCamelCase       ,      lowerCamelCase						)

							# Testing conversion to ids without special tokens
							lowerCamelCase_	:     List[str] 						=      tokenizer.encode(lowerCamelCase       ,      add_special_tokens=lowerCamelCase       ,      add_prefix_space=lowerCamelCase						)
							lowerCamelCase_	:     str 						=      rust_tokenizer.encode(lowerCamelCase       ,      add_special_tokens=lowerCamelCase						)
							self.assertListEqual(lowerCamelCase       ,      lowerCamelCase						)

							# Testing conversion to ids with special tokens
							lowerCamelCase_	:     Dict 						=      self.get_rust_tokenizer(add_prefix_space=lowerCamelCase						)
							lowerCamelCase_	:     Tuple 						=      tokenizer.encode(lowerCamelCase       ,      add_prefix_space=lowerCamelCase						)
							lowerCamelCase_	:     Union[str, Any] 						=      rust_tokenizer.encode(lowerCamelCase						)
							self.assertListEqual(lowerCamelCase       ,      lowerCamelCase						)

							# Testing the unknown token
							lowerCamelCase_	:     Optional[int] 						=      tokens + [rust_tokenizer.unk_token]
							lowerCamelCase_	:     int 						=      [14, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowerCamelCase						)       ,      lowerCamelCase						)



					def   __a     (		self     :     Tuple       ,      *lowerCamelCase     :     Dict       ,      **lowerCamelCase     :     Tuple						):
							# It's very difficult to mix/test pretokenization with byte-level
							# And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
							pass



					def   __a     (		self     :     Optional[int]       ,      lowerCamelCase     :     Any=15						):
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"						):
											lowerCamelCase_	:     Optional[int] 						=      self.rust_tokenizer_class.from_pretrained(lowerCamelCase       ,      **lowerCamelCase						)

											# Simple input
											lowerCamelCase_	:     Any 						=      'This is a simple input'
											lowerCamelCase_	:     Optional[int] 						=      ['This is a simple input 1', 'This is a simple input 2']
											lowerCamelCase_	:     Optional[Any] 						=      ('This is a simple input', 'This is a pair')
											lowerCamelCase_	:     str 						=      [
											    ('This is a simple input 1', 'This is a simple input 2'),
											    ('This is a simple pair 1', 'This is a simple pair 2'),
											]

											# Simple input tests
											self.assertRaises(lowerCamelCase       ,      tokenizer_r.encode       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding='max_length'						)

											# Simple input
											self.assertRaises(lowerCamelCase       ,      tokenizer_r.encode_plus       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding='max_length'						)

											# Simple input
											self.assertRaises(
											    lowerCamelCase       ,      tokenizer_r.batch_encode_plus       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding='max_length'       ,      )

											# Pair input
											self.assertRaises(lowerCamelCase       ,      tokenizer_r.encode       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding='max_length'						)

											# Pair input
											self.assertRaises(lowerCamelCase       ,      tokenizer_r.encode_plus       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding='max_length'						)

											# Pair input
											self.assertRaises(
											    lowerCamelCase       ,      tokenizer_r.batch_encode_plus       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding='max_length'       ,      )



					def   __a     (		self     :     Optional[Any]						):
							lowerCamelCase_	:     Optional[int] 						=      CodeGenTokenizer.from_pretrained(self.tmpdirname       ,      pad_token='<pad>'						)

							# Simple input
							lowerCamelCase_	:     Tuple 						=      'This is a simple input'
							lowerCamelCase_	:     Tuple 						=      ['This is a simple input looooooooong', 'This is a simple input']
							lowerCamelCase_	:     Any 						=      ('This is a simple input', 'This is a pair')
							lowerCamelCase_	:     Tuple 						=      [
							    ('This is a simple input loooooong', 'This is a simple input'),
							    ('This is a simple pair loooooong', 'This is a simple pair'),
							]

							lowerCamelCase_	:     str 						=      tokenizer.pad_token_id

							lowerCamelCase_	:     int 						=      tokenizer(lowerCamelCase       ,      padding='max_length'       ,      max_length=30       ,      return_tensors='np'						)
							lowerCamelCase_	:     List[Any] 						=      tokenizer(lowerCamelCase       ,      padding=lowerCamelCase       ,      truncate=lowerCamelCase       ,      return_tensors='np'						)
							lowerCamelCase_	:     Tuple 						=      tokenizer(*lowerCamelCase       ,      padding='max_length'       ,      max_length=60       ,      return_tensors='np'						)
							lowerCamelCase_	:     Union[str, Any] 						=      tokenizer(lowerCamelCase       ,      padding=lowerCamelCase       ,      truncate=lowerCamelCase       ,      return_tensors='np'						)

							# s
							# test single string max_length padding
							self.assertEqual(out_s['input_ids'].shape[-1]       ,      30						)
							self.assertTrue(pad_token_id in out_s['input_ids']						)
							self.assertTrue(0 in out_s['attention_mask']						)

							# s2
							# test automatic padding
							self.assertEqual(out_sa['input_ids'].shape[-1]       ,      33						)
							# long slice doesn't have padding
							self.assertFalse(pad_token_id in out_sa['input_ids'][0]						)
							self.assertFalse(0 in out_sa['attention_mask'][0]						)
							# short slice does have padding
							self.assertTrue(pad_token_id in out_sa['input_ids'][1]						)
							self.assertTrue(0 in out_sa['attention_mask'][1]						)

							# p
							# test single pair max_length padding
							self.assertEqual(out_p['input_ids'].shape[-1]       ,      60						)
							self.assertTrue(pad_token_id in out_p['input_ids']						)
							self.assertTrue(0 in out_p['attention_mask']						)

							# p2
							# test automatic padding pair
							self.assertEqual(out_pa['input_ids'].shape[-1]       ,      52						)
							# long slice pair doesn't have padding
							self.assertFalse(pad_token_id in out_pa['input_ids'][0]						)
							self.assertFalse(0 in out_pa['attention_mask'][0]						)
							# short slice pair does have padding
							self.assertTrue(pad_token_id in out_pa['input_ids'][1]						)
							self.assertTrue(0 in out_pa['attention_mask'][1]						)



					def   __a     (		self     :     str						):
							lowerCamelCase_	:     List[str] 						=      '$$$'
							lowerCamelCase_	:     Dict 						=      CodeGenTokenizer.from_pretrained(self.tmpdirname       ,      bos_token=lowerCamelCase       ,      add_bos_token=lowerCamelCase						)

							lowerCamelCase_	:     Optional[Any] 						=      'This is a simple input'
							lowerCamelCase_	:     Optional[Any] 						=      ['This is a simple input 1', 'This is a simple input 2']

							lowerCamelCase_	:     List[str] 						=      tokenizer.bos_token_id

							lowerCamelCase_	:     Optional[Any] 						=      tokenizer(lowerCamelCase						)
							lowerCamelCase_	:     Optional[int] 						=      tokenizer(lowerCamelCase						)

							self.assertEqual(out_s.input_ids[0]       ,      lowerCamelCase						)
							self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids						)						)

							lowerCamelCase_	:     str 						=      tokenizer.decode(out_s.input_ids						)
							lowerCamelCase_	:     Any 						=      tokenizer.batch_decode(out_sa.input_ids						)

							self.assertEqual(decode_s.split()[0]       ,      lowerCamelCase						)
							self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa						)						)



					@slow
					def   __a     (		self     :     List[str]						):
							lowerCamelCase_	:     Optional[int] 						=      CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono'						)

							lowerCamelCase_	:     Tuple 						=      '\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
							lowerCamelCase_	:     Any 						=      '\nif len_a > len_b:      result = a\nelse:      result = b'

							lowerCamelCase_	:     int 						=      tokenizer.encode(lowerCamelCase						)
							lowerCamelCase_	:     Optional[int] 						=      ['^#', re.escape('<|endoftext|>'						), '^\'\'\'', '^"""', '\n\n\n']
							lowerCamelCase_	:     str 						=      tokenizer.decode(lowerCamelCase       ,      truncate_before_pattern=lowerCamelCase						)
							self.assertEqual(lowerCamelCase       ,      lowerCamelCase						)





					def   __a     (		self     :     Optional[Any]						):
							pass
