
import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class    _snake_case (						UpperCAmelCase_	,    unittest.TestCase				):
 __lowerCAmelCase	:  Tuple	   =      CodeGenTokenizer
 __lowerCAmelCase	:  Optional[int]	   =      CodeGenTokenizerFast
 __lowerCAmelCase	:  Any	   =      True
 __lowerCAmelCase	:  Tuple	   =      {'add_prefix_space': True}
 __lowerCAmelCase	:  str	   =      False


 def   lowercase__  (       self):


   '''simple docstring'''




   super().setUp()

   # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
   lowercase__	: Union[str, Any]     		=   [
       """l""",
       """o""",
       """w""",
       """e""",
       """r""",
       """s""",
       """t""",
       """i""",
       """d""",
       """n""",
       """\u0120""",
       """\u0120l""",
       """\u0120n""",
       """\u0120lo""",
       """\u0120low""",
       """er""",
       """\u0120lowest""",
       """\u0120newer""",
       """\u0120wider""",
       """<unk>""",
       """<|endoftext|>""",
   ]
   lowercase__	: int     		=   dict(zip(SCREAMING_SNAKE_CASE_       ,	range(len(SCREAMING_SNAKE_CASE_))))
   lowercase__	: List[str]     		=   ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
   lowercase__	: Dict     		=   {"""unk_token""": """<unk>"""}

   lowercase__	: List[Any]     		=   os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES["""vocab_file"""])
   lowercase__	: str     		=   os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES["""merges_file"""])
   with open(self.vocab_file       ,	"""w"""       ,	encoding="""utf-8""") as fp:
     fp.write(json.dumps(SCREAMING_SNAKE_CASE_) + """\n""")
   with open(self.merges_file       ,	"""w"""       ,	encoding="""utf-8""") as fp:
     fp.write("""\n""".join(SCREAMING_SNAKE_CASE_))


 def   lowercase__  (       self       ,	**SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   kwargs.update(self.special_tokens_map)
   return CodeGenTokenizer.from_pretrained(self.tmpdirname       ,	**SCREAMING_SNAKE_CASE_)


 def   lowercase__  (       self       ,	**SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   kwargs.update(self.special_tokens_map)
   return CodeGenTokenizerFast.from_pretrained(self.tmpdirname       ,	**SCREAMING_SNAKE_CASE_)


 def   lowercase__  (       self       ,	SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   lowercase__	: int     		=   """lower newer"""
   lowercase__	: Optional[Any]     		=   """lower newer"""
   return input_text, output_text


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: List[str]     		=   CodeGenTokenizer(self.vocab_file       ,	self.merges_file       ,	**self.special_tokens_map)
   lowercase__	: List[Any]     		=   """lower newer"""
   lowercase__	: str     		=   ["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
   lowercase__	: Dict     		=   tokenizer.tokenize(SCREAMING_SNAKE_CASE_       ,	add_prefix_space=SCREAMING_SNAKE_CASE_)
   self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)

   lowercase__	: Optional[Any]     		=   tokens + [tokenizer.unk_token]
   lowercase__	: int     		=   [14, 15, 10, 9, 3, 2, 15, 19]
   self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_)       ,	SCREAMING_SNAKE_CASE_)


 def   lowercase__  (       self):


   '''simple docstring'''




   if not self.test_rust_tokenizer:
     return

   lowercase__	: str     		=   self.get_tokenizer()
   lowercase__	: List[str]     		=   self.get_rust_tokenizer(add_prefix_space=SCREAMING_SNAKE_CASE_)

   lowercase__	: Union[str, Any]     		=   """lower newer"""

   # Testing tokenization
   lowercase__	: int     		=   tokenizer.tokenize(SCREAMING_SNAKE_CASE_       ,	add_prefix_space=SCREAMING_SNAKE_CASE_)
   lowercase__	: int     		=   rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_)
   self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)

   # Testing conversion to ids without special tokens
   lowercase__	: str     		=   tokenizer.encode(SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_       ,	add_prefix_space=SCREAMING_SNAKE_CASE_)
   lowercase__	: Optional[Any]     		=   rust_tokenizer.encode(SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)
   self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)

   # Testing conversion to ids with special tokens
   lowercase__	: Optional[int]     		=   self.get_rust_tokenizer(add_prefix_space=SCREAMING_SNAKE_CASE_)
   lowercase__	: int     		=   tokenizer.encode(SCREAMING_SNAKE_CASE_       ,	add_prefix_space=SCREAMING_SNAKE_CASE_)
   lowercase__	: List[Any]     		=   rust_tokenizer.encode(SCREAMING_SNAKE_CASE_)
   self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)

   # Testing the unknown token
   lowercase__	: Optional[int]     		=   tokens + [rust_tokenizer.unk_token]
   lowercase__	: Optional[Any]     		=   [14, 15, 10, 9, 3, 2, 15, 19]
   self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_)       ,	SCREAMING_SNAKE_CASE_)


 def   lowercase__  (       self       ,	*SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_):


   '''simple docstring'''




   pass


 def   lowercase__  (       self       ,	SCREAMING_SNAKE_CASE_=15):


   '''simple docstring'''




   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'):
       lowercase__	: List[Any]     		=   self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)

       # Simple input
       lowercase__	: Optional[int]     		=   """This is a simple input"""
       lowercase__	: Any     		=   ["""This is a simple input 1""", """This is a simple input 2"""]
       lowercase__	: Optional[Any]     		=   ("""This is a simple input""", """This is a pair""")
       lowercase__	: int     		=   [
           ("""This is a simple input 1""", """This is a simple input 2"""),
           ("""This is a simple pair 1""", """This is a simple pair 2"""),
       ]

       # Simple input tests
       self.assertRaises(SCREAMING_SNAKE_CASE_       ,	tokenizer_r.encode       ,	SCREAMING_SNAKE_CASE_       ,	max_length=SCREAMING_SNAKE_CASE_       ,	padding="""max_length""")

       # Simple input
       self.assertRaises(SCREAMING_SNAKE_CASE_       ,	tokenizer_r.encode_plus       ,	SCREAMING_SNAKE_CASE_       ,	max_length=SCREAMING_SNAKE_CASE_       ,	padding="""max_length""")

       # Simple input
       self.assertRaises(
           SCREAMING_SNAKE_CASE_       ,	tokenizer_r.batch_encode_plus       ,	SCREAMING_SNAKE_CASE_       ,	max_length=SCREAMING_SNAKE_CASE_       ,	padding="""max_length"""       ,	)

       # Pair input
       self.assertRaises(SCREAMING_SNAKE_CASE_       ,	tokenizer_r.encode       ,	SCREAMING_SNAKE_CASE_       ,	max_length=SCREAMING_SNAKE_CASE_       ,	padding="""max_length""")

       # Pair input
       self.assertRaises(SCREAMING_SNAKE_CASE_       ,	tokenizer_r.encode_plus       ,	SCREAMING_SNAKE_CASE_       ,	max_length=SCREAMING_SNAKE_CASE_       ,	padding="""max_length""")

       # Pair input
       self.assertRaises(
           SCREAMING_SNAKE_CASE_       ,	tokenizer_r.batch_encode_plus       ,	SCREAMING_SNAKE_CASE_       ,	max_length=SCREAMING_SNAKE_CASE_       ,	padding="""max_length"""       ,	)


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Optional[Any]     		=   CodeGenTokenizer.from_pretrained(self.tmpdirname       ,	pad_token="""<pad>""")

   # Simple input
   lowercase__	: Dict     		=   """This is a simple input"""
   lowercase__	: Optional[int]     		=   ["""This is a simple input looooooooong""", """This is a simple input"""]
   lowercase__	: List[str]     		=   ("""This is a simple input""", """This is a pair""")
   lowercase__	: Any     		=   [
       ("""This is a simple input loooooong""", """This is a simple input"""),
       ("""This is a simple pair loooooong""", """This is a simple pair"""),
   ]

   lowercase__	: Union[str, Any]     		=   tokenizer.pad_token_id

   lowercase__	: Optional[int]     		=   tokenizer(SCREAMING_SNAKE_CASE_       ,	padding="""max_length"""       ,	max_length=30       ,	return_tensors="""np""")
   lowercase__	: Any     		=   tokenizer(SCREAMING_SNAKE_CASE_       ,	padding=SCREAMING_SNAKE_CASE_       ,	truncate=SCREAMING_SNAKE_CASE_       ,	return_tensors="""np""")
   lowercase__	: List[Any]     		=   tokenizer(*SCREAMING_SNAKE_CASE_       ,	padding="""max_length"""       ,	max_length=60       ,	return_tensors="""np""")
   lowercase__	: Union[str, Any]     		=   tokenizer(SCREAMING_SNAKE_CASE_       ,	padding=SCREAMING_SNAKE_CASE_       ,	truncate=SCREAMING_SNAKE_CASE_       ,	return_tensors="""np""")

   # s
   # test single string max_length padding
   self.assertEqual(out_s["""input_ids"""].shape[-1]       ,	30)
   self.assertTrue(pad_token_id in out_s["""input_ids"""])
   self.assertTrue(0 in out_s["""attention_mask"""])

   # s2
   # test automatic padding
   self.assertEqual(out_sa["""input_ids"""].shape[-1]       ,	33)
   # long slice doesn't have padding
   self.assertFalse(pad_token_id in out_sa["""input_ids"""][0])
   self.assertFalse(0 in out_sa["""attention_mask"""][0])
   # short slice does have padding
   self.assertTrue(pad_token_id in out_sa["""input_ids"""][1])
   self.assertTrue(0 in out_sa["""attention_mask"""][1])

   # p
   # test single pair max_length padding
   self.assertEqual(out_p["""input_ids"""].shape[-1]       ,	60)
   self.assertTrue(pad_token_id in out_p["""input_ids"""])
   self.assertTrue(0 in out_p["""attention_mask"""])

   # p2
   # test automatic padding pair
   self.assertEqual(out_pa["""input_ids"""].shape[-1]       ,	52)
   # long slice pair doesn't have padding
   self.assertFalse(pad_token_id in out_pa["""input_ids"""][0])
   self.assertFalse(0 in out_pa["""attention_mask"""][0])
   # short slice pair does have padding
   self.assertTrue(pad_token_id in out_pa["""input_ids"""][1])
   self.assertTrue(0 in out_pa["""attention_mask"""][1])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Optional[int]     		=   """$$$"""
   lowercase__	: str     		=   CodeGenTokenizer.from_pretrained(self.tmpdirname       ,	bos_token=SCREAMING_SNAKE_CASE_       ,	add_bos_token=SCREAMING_SNAKE_CASE_)

   lowercase__	: Any     		=   """This is a simple input"""
   lowercase__	: Dict     		=   ["""This is a simple input 1""", """This is a simple input 2"""]

   lowercase__	: Optional[Any]     		=   tokenizer.bos_token_id

   lowercase__	: Any     		=   tokenizer(SCREAMING_SNAKE_CASE_)
   lowercase__	: Any     		=   tokenizer(SCREAMING_SNAKE_CASE_)

   self.assertEqual(out_s.input_ids[0]       ,	SCREAMING_SNAKE_CASE_)
   self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids))

   lowercase__	: Any     		=   tokenizer.decode(out_s.input_ids)
   lowercase__	: int     		=   tokenizer.batch_decode(out_sa.input_ids)

   self.assertEqual(decode_s.split()[0]       ,	SCREAMING_SNAKE_CASE_)
   self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa))


 @slow
 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Tuple     		=   CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono""")

   lowercase__	: Optional[Any]     		=   """\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
   lowercase__	: List[Any]     		=   """\nif len_a > len_b:      result = a\nelse:      result = b"""

   lowercase__	: Tuple     		=   tokenizer.encode(SCREAMING_SNAKE_CASE_)
   lowercase__	: Optional[Any]     		=   ["""^#""", re.escape("""<|endoftext|>"""), """^'''""", """^\"\"\"""", """\n\n\n"""]
   lowercase__	: int     		=   tokenizer.decode(SCREAMING_SNAKE_CASE_       ,	truncate_before_pattern=SCREAMING_SNAKE_CASE_)
   self.assertEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)



 def   lowercase__  (       self):


   '''simple docstring'''




   pass








from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowerCamelCase__     :		List[Any]        = logging.get_logger(__name__)

lowerCamelCase__     :		Union[str, Any]        = {
    """YituTech/conv-bert-base""": """https://huggingface.co/YituTech/conv-bert-base/resolve/main/config.json""",
    """YituTech/conv-bert-medium-small""": (
        """https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/config.json"""
    ),
    """YituTech/conv-bert-small""": """https://huggingface.co/YituTech/conv-bert-small/resolve/main/config.json""",
    # See all ConvBERT models at https://huggingface.co/models?filter=convbert
}



class    _snake_case (						UpperCAmelCase_				):
 __lowerCAmelCase	:  Union[str, Any]	   =      'convbert'


 def __init__(       self       ,	SCREAMING_SNAKE_CASE_=3_05_22       ,	SCREAMING_SNAKE_CASE_=7_68       ,	SCREAMING_SNAKE_CASE_=12       ,	SCREAMING_SNAKE_CASE_=12       ,	SCREAMING_SNAKE_CASE_=30_72       ,	SCREAMING_SNAKE_CASE_="gelu"       ,	SCREAMING_SNAKE_CASE_=0.1       ,	SCREAMING_SNAKE_CASE_=0.1       ,	SCREAMING_SNAKE_CASE_=5_12       ,	SCREAMING_SNAKE_CASE_=2       ,	SCREAMING_SNAKE_CASE_=0.0_2       ,	SCREAMING_SNAKE_CASE_=1E-12       ,	SCREAMING_SNAKE_CASE_=1       ,	SCREAMING_SNAKE_CASE_=0       ,	SCREAMING_SNAKE_CASE_=2       ,	SCREAMING_SNAKE_CASE_=7_68       ,	SCREAMING_SNAKE_CASE_=2       ,	SCREAMING_SNAKE_CASE_=9       ,	SCREAMING_SNAKE_CASE_=1       ,	SCREAMING_SNAKE_CASE_=None       ,	**SCREAMING_SNAKE_CASE_       ,	):


   '''simple docstring'''




   super().__init__(
       pad_token_id=SCREAMING_SNAKE_CASE_       ,	bos_token_id=SCREAMING_SNAKE_CASE_       ,	eos_token_id=SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_       ,	)

   lowercase__	: Dict     		=   vocab_size
   lowercase__	: List[Any]     		=   hidden_size
   lowercase__	: Optional[Any]     		=   num_hidden_layers
   lowercase__	: Union[str, Any]     		=   num_attention_heads
   lowercase__	: List[str]     		=   intermediate_size
   lowercase__	: Optional[int]     		=   hidden_act
   lowercase__	: Tuple     		=   hidden_dropout_prob
   lowercase__	: List[str]     		=   attention_probs_dropout_prob
   lowercase__	: Tuple     		=   max_position_embeddings
   lowercase__	: Dict     		=   type_vocab_size
   lowercase__	: Union[str, Any]     		=   initializer_range
   lowercase__	: Dict     		=   layer_norm_eps
   lowercase__	: Tuple     		=   embedding_size
   lowercase__	: List[str]     		=   head_ratio
   lowercase__	: Dict     		=   conv_kernel_size
   lowercase__	: Dict     		=   num_groups
   lowercase__	: int     		=   classifier_dropout



class    _snake_case (						UpperCAmelCase_				):


 @property
 def   lowercase__  (       self):


   '''simple docstring'''




   if self.task == "multiple-choice":
     lowercase__	: Union[str, Any]     		=   {0: """batch""", 1: """choice""", 2: """sequence"""}
   else:
     lowercase__	: str     		=   {0: """batch""", 1: """sequence"""}
   return OrderedDict(
       [
           ("""input_ids""", dynamic_axis),
           ("""attention_mask""", dynamic_axis),
           ("""token_type_ids""", dynamic_axis),
       ])






