





from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowercase		:  Tuple			       =logging.get_logger(__name__)

_lowercase		:  Optional[Any]			       ={
    """google/canine-s""": """https://huggingface.co/google/canine-s/resolve/main/config.json""",
    # See all CANINE models at https://huggingface.co/models?filter=canine
}






class 						UpperCamelCase_  (  snake_case__							):
					_a      :						str						    =					'canine'



					def __init__(		self     :     Dict       ,      lowerCamelCase     :     Union[str, Any]=7_68       ,      lowerCamelCase     :     Optional[int]=12       ,      lowerCamelCase     :     int=12       ,      lowerCamelCase     :     str=30_72       ,      lowerCamelCase     :     Optional[Any]="gelu"       ,      lowerCamelCase     :     Optional[int]=0.1       ,      lowerCamelCase     :     int=0.1       ,      lowerCamelCase     :     Any=1_63_84       ,      lowerCamelCase     :     Union[str, Any]=16       ,      lowerCamelCase     :     Union[str, Any]=0.02       ,      lowerCamelCase     :     Optional[int]=1E-12       ,      lowerCamelCase     :     Optional[Any]=0       ,      lowerCamelCase     :     Optional[int]=0Xe_0_0_0       ,      lowerCamelCase     :     Optional[int]=0Xe_0_0_1       ,      lowerCamelCase     :     List[Any]=4       ,      lowerCamelCase     :     int=4       ,      lowerCamelCase     :     Any=8       ,      lowerCamelCase     :     Any=1_63_84       ,      lowerCamelCase     :     int=1_28       ,      **lowerCamelCase     :     Dict       ,      ):
							super().__init__(pad_token_id=lowerCamelCase       ,      bos_token_id=lowerCamelCase       ,      eos_token_id=lowerCamelCase       ,      **lowerCamelCase						)

							lowerCamelCase_	:     List[Any] 						=      max_position_embeddings
							lowerCamelCase_	:     Union[str, Any] 						=      hidden_size
							lowerCamelCase_	:     List[Any] 						=      num_hidden_layers
							lowerCamelCase_	:     Union[str, Any] 						=      num_attention_heads
							lowerCamelCase_	:     Optional[int] 						=      intermediate_size
							lowerCamelCase_	:     List[Any] 						=      hidden_act
							lowerCamelCase_	:     Optional[int] 						=      hidden_dropout_prob
							lowerCamelCase_	:     Optional[Any] 						=      attention_probs_dropout_prob
							lowerCamelCase_	:     Tuple 						=      initializer_range
							lowerCamelCase_	:     str 						=      type_vocab_size
							lowerCamelCase_	:     str 						=      layer_norm_eps

							# Character config:
							lowerCamelCase_	:     Union[str, Any] 						=      downsampling_rate
							lowerCamelCase_	:     int 						=      upsampling_kernel_size
							lowerCamelCase_	:     str 						=      num_hash_functions
							lowerCamelCase_	:     List[str] 						=      num_hash_buckets
							lowerCamelCase_	:     Dict 						=      local_transformer_stride







import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 						UpperCamelCase_  (  snake_case__  ,     unittest.TestCase							):
					_a      :						Optional[Any]						    =					CodeGenTokenizer
					_a      :						Optional[Any]						    =					CodeGenTokenizerFast
					_a      :						Union[str, Any]						    =					True
					_a      :						Dict						    =					{'add_prefix_space': True}
					_a      :						Dict						    =					False



					def   __a     (		self     :     Union[str, Any]						):
							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							lowerCamelCase_	:     Optional[Any] 						=      [
							    'l',
							    'o',
							    'w',
							    'e',
							    'r',
							    's',
							    't',
							    'i',
							    'd',
							    'n',
							    '\u0120',
							    '\u0120l',
							    '\u0120n',
							    '\u0120lo',
							    '\u0120low',
							    'er',
							    '\u0120lowest',
							    '\u0120newer',
							    '\u0120wider',
							    '<unk>',
							    '<|endoftext|>',
							]
							lowerCamelCase_	:     List[Any] 						=      dict(zip(lowerCamelCase       ,      range(len(lowerCamelCase						)						)						)						)
							lowerCamelCase_	:     Tuple 						=      ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
							lowerCamelCase_	:     Optional[int] 						=      {'unk_token': '<unk>'}

							lowerCamelCase_	:     Optional[Any] 						=      os.path.join(self.tmpdirname       ,      VOCAB_FILES_NAMES['vocab_file']						)
							lowerCamelCase_	:     Tuple 						=      os.path.join(self.tmpdirname       ,      VOCAB_FILES_NAMES['merges_file']						)
							with open(self.vocab_file       ,      'w'       ,      encoding='utf-8'						) as fp:
									fp.write(json.dumps(lowerCamelCase						) + '\n'						)
							with open(self.merges_file       ,      'w'       ,      encoding='utf-8'						) as fp:
									fp.write('\n'.join(lowerCamelCase						)						)



					def   __a     (		self     :     Optional[int]       ,      **lowerCamelCase     :     Dict						):
							kwargs.update(self.special_tokens_map						)
							return CodeGenTokenizer.from_pretrained(self.tmpdirname       ,      **lowerCamelCase						)



					def   __a     (		self     :     List[Any]       ,      **lowerCamelCase     :     int						):
							kwargs.update(self.special_tokens_map						)
							return CodeGenTokenizerFast.from_pretrained(self.tmpdirname       ,      **lowerCamelCase						)



					def   __a     (		self     :     Tuple       ,      lowerCamelCase     :     List[str]						):
							lowerCamelCase_	:     List[str] 						=      'lower newer'
							lowerCamelCase_	:     str 						=      'lower newer'
							return input_text, output_text



					def   __a     (		self     :     Optional[int]						):
							lowerCamelCase_	:     List[Any] 						=      CodeGenTokenizer(self.vocab_file       ,      self.merges_file       ,      **self.special_tokens_map						)
							lowerCamelCase_	:     Union[str, Any] 						=      'lower newer'
							lowerCamelCase_	:     List[str] 						=      ['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
							lowerCamelCase_	:     Dict 						=      tokenizer.tokenize(lowerCamelCase       ,      add_prefix_space=lowerCamelCase						)
							self.assertListEqual(lowerCamelCase       ,      lowerCamelCase						)

							lowerCamelCase_	:     Dict 						=      tokens + [tokenizer.unk_token]
							lowerCamelCase_	:     Dict 						=      [14, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase						)       ,      lowerCamelCase						)



					def   __a     (		self     :     str						):
							if not self.test_rust_tokenizer:
									return

							lowerCamelCase_	:     Dict 						=      self.get_tokenizer()
							lowerCamelCase_	:     Optional[Any] 						=      self.get_rust_tokenizer(add_prefix_space=lowerCamelCase						)

							lowerCamelCase_	:     int 						=      'lower newer'

							# Testing tokenization
							lowerCamelCase_	:     List[str] 						=      tokenizer.tokenize(lowerCamelCase       ,      add_prefix_space=lowerCamelCase						)
							lowerCamelCase_	:     List[Any] 						=      rust_tokenizer.tokenize(lowerCamelCase						)
							self.assertListEqual(lowerCamelCase       ,      lowerCamelCase						)

							# Testing conversion to ids without special tokens
							lowerCamelCase_	:     List[str] 						=      tokenizer.encode(lowerCamelCase       ,      add_special_tokens=lowerCamelCase       ,      add_prefix_space=lowerCamelCase						)
							lowerCamelCase_	:     str 						=      rust_tokenizer.encode(lowerCamelCase       ,      add_special_tokens=lowerCamelCase						)
							self.assertListEqual(lowerCamelCase       ,      lowerCamelCase						)

							# Testing conversion to ids with special tokens
							lowerCamelCase_	:     Dict 						=      self.get_rust_tokenizer(add_prefix_space=lowerCamelCase						)
							lowerCamelCase_	:     Tuple 						=      tokenizer.encode(lowerCamelCase       ,      add_prefix_space=lowerCamelCase						)
							lowerCamelCase_	:     Union[str, Any] 						=      rust_tokenizer.encode(lowerCamelCase						)
							self.assertListEqual(lowerCamelCase       ,      lowerCamelCase						)

							# Testing the unknown token
							lowerCamelCase_	:     Optional[int] 						=      tokens + [rust_tokenizer.unk_token]
							lowerCamelCase_	:     int 						=      [14, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowerCamelCase						)       ,      lowerCamelCase						)



					def   __a     (		self     :     Tuple       ,      *lowerCamelCase     :     Dict       ,      **lowerCamelCase     :     Tuple						):
							# It's very difficult to mix/test pretokenization with byte-level
							# And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
							pass



					def   __a     (		self     :     Optional[int]       ,      lowerCamelCase     :     Any=15						):
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"						):
											lowerCamelCase_	:     Optional[int] 						=      self.rust_tokenizer_class.from_pretrained(lowerCamelCase       ,      **lowerCamelCase						)

											# Simple input
											lowerCamelCase_	:     Any 						=      'This is a simple input'
											lowerCamelCase_	:     Optional[int] 						=      ['This is a simple input 1', 'This is a simple input 2']
											lowerCamelCase_	:     Optional[Any] 						=      ('This is a simple input', 'This is a pair')
											lowerCamelCase_	:     str 						=      [
											    ('This is a simple input 1', 'This is a simple input 2'),
											    ('This is a simple pair 1', 'This is a simple pair 2'),
											]

											# Simple input tests
											self.assertRaises(lowerCamelCase       ,      tokenizer_r.encode       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding='max_length'						)

											# Simple input
											self.assertRaises(lowerCamelCase       ,      tokenizer_r.encode_plus       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding='max_length'						)

											# Simple input
											self.assertRaises(
											    lowerCamelCase       ,      tokenizer_r.batch_encode_plus       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding='max_length'       ,      )

											# Pair input
											self.assertRaises(lowerCamelCase       ,      tokenizer_r.encode       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding='max_length'						)

											# Pair input
											self.assertRaises(lowerCamelCase       ,      tokenizer_r.encode_plus       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding='max_length'						)

											# Pair input
											self.assertRaises(
											    lowerCamelCase       ,      tokenizer_r.batch_encode_plus       ,      lowerCamelCase       ,      max_length=lowerCamelCase       ,      padding='max_length'       ,      )



					def   __a     (		self     :     Optional[Any]						):
							lowerCamelCase_	:     Optional[int] 						=      CodeGenTokenizer.from_pretrained(self.tmpdirname       ,      pad_token='<pad>'						)

							# Simple input
							lowerCamelCase_	:     Tuple 						=      'This is a simple input'
							lowerCamelCase_	:     Tuple 						=      ['This is a simple input looooooooong', 'This is a simple input']
							lowerCamelCase_	:     Any 						=      ('This is a simple input', 'This is a pair')
							lowerCamelCase_	:     Tuple 						=      [
							    ('This is a simple input loooooong', 'This is a simple input'),
							    ('This is a simple pair loooooong', 'This is a simple pair'),
							]

							lowerCamelCase_	:     str 						=      tokenizer.pad_token_id

							lowerCamelCase_	:     int 						=      tokenizer(lowerCamelCase       ,      padding='max_length'       ,      max_length=30       ,      return_tensors='np'						)
							lowerCamelCase_	:     List[Any] 						=      tokenizer(lowerCamelCase       ,      padding=lowerCamelCase       ,      truncate=lowerCamelCase       ,      return_tensors='np'						)
							lowerCamelCase_	:     Tuple 						=      tokenizer(*lowerCamelCase       ,      padding='max_length'       ,      max_length=60       ,      return_tensors='np'						)
							lowerCamelCase_	:     Union[str, Any] 						=      tokenizer(lowerCamelCase       ,      padding=lowerCamelCase       ,      truncate=lowerCamelCase       ,      return_tensors='np'						)

							# s
							# test single string max_length padding
							self.assertEqual(out_s['input_ids'].shape[-1]       ,      30						)
							self.assertTrue(pad_token_id in out_s['input_ids']						)
							self.assertTrue(0 in out_s['attention_mask']						)

							# s2
							# test automatic padding
							self.assertEqual(out_sa['input_ids'].shape[-1]       ,      33						)
							# long slice doesn't have padding
							self.assertFalse(pad_token_id in out_sa['input_ids'][0]						)
							self.assertFalse(0 in out_sa['attention_mask'][0]						)
							# short slice does have padding
							self.assertTrue(pad_token_id in out_sa['input_ids'][1]						)
							self.assertTrue(0 in out_sa['attention_mask'][1]						)

							# p
							# test single pair max_length padding
							self.assertEqual(out_p['input_ids'].shape[-1]       ,      60						)
							self.assertTrue(pad_token_id in out_p['input_ids']						)
							self.assertTrue(0 in out_p['attention_mask']						)

							# p2
							# test automatic padding pair
							self.assertEqual(out_pa['input_ids'].shape[-1]       ,      52						)
							# long slice pair doesn't have padding
							self.assertFalse(pad_token_id in out_pa['input_ids'][0]						)
							self.assertFalse(0 in out_pa['attention_mask'][0]						)
							# short slice pair does have padding
							self.assertTrue(pad_token_id in out_pa['input_ids'][1]						)
							self.assertTrue(0 in out_pa['attention_mask'][1]						)



					def   __a     (		self     :     str						):
							lowerCamelCase_	:     List[str] 						=      '$$$'
							lowerCamelCase_	:     Dict 						=      CodeGenTokenizer.from_pretrained(self.tmpdirname       ,      bos_token=lowerCamelCase       ,      add_bos_token=lowerCamelCase						)

							lowerCamelCase_	:     Optional[Any] 						=      'This is a simple input'
							lowerCamelCase_	:     Optional[Any] 						=      ['This is a simple input 1', 'This is a simple input 2']

							lowerCamelCase_	:     List[str] 						=      tokenizer.bos_token_id

							lowerCamelCase_	:     Optional[Any] 						=      tokenizer(lowerCamelCase						)
							lowerCamelCase_	:     Optional[int] 						=      tokenizer(lowerCamelCase						)

							self.assertEqual(out_s.input_ids[0]       ,      lowerCamelCase						)
							self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids						)						)

							lowerCamelCase_	:     str 						=      tokenizer.decode(out_s.input_ids						)
							lowerCamelCase_	:     Any 						=      tokenizer.batch_decode(out_sa.input_ids						)

							self.assertEqual(decode_s.split()[0]       ,      lowerCamelCase						)
							self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa						)						)



					@slow
					def   __a     (		self     :     List[str]						):
							lowerCamelCase_	:     Optional[int] 						=      CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono'						)

							lowerCamelCase_	:     Tuple 						=      '\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
							lowerCamelCase_	:     Any 						=      '\nif len_a > len_b:      result = a\nelse:      result = b'

							lowerCamelCase_	:     int 						=      tokenizer.encode(lowerCamelCase						)
							lowerCamelCase_	:     Optional[int] 						=      ['^#', re.escape('<|endoftext|>'						), '^\'\'\'', '^"""', '\n\n\n']
							lowerCamelCase_	:     str 						=      tokenizer.decode(lowerCamelCase       ,      truncate_before_pattern=lowerCamelCase						)
							self.assertEqual(lowerCamelCase       ,      lowerCamelCase						)





					def   __a     (		self     :     Optional[Any]						):
							pass
