





import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 		SCREAMING_SNAKE_CASE_							(    snake_case__       ,  unittest.TestCase   ):
				"""simple docstring"""
				__snake_case   :       Dict	 =		CodeGenTokenizer
				__snake_case   :       Optional[Any]	 =		CodeGenTokenizerFast
				__snake_case   :       Any	 =		True
				__snake_case   :       Union[str, Any]	 =		{"""add_prefix_space""": True}
				__snake_case   :       Dict	 =		False




				def 					__lowercase  (     self  :Dict			):
											super().setUp()

											# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
											__lowerCamelCase      :      str         =[
											    '''l''',
											    '''o''',
											    '''w''',
											    '''e''',
											    '''r''',
											    '''s''',
											    '''t''',
											    '''i''',
											    '''d''',
											    '''n''',
											    '''\u0120''',
											    '''\u0120l''',
											    '''\u0120n''',
											    '''\u0120lo''',
											    '''\u0120low''',
											    '''er''',
											    '''\u0120lowest''',
											    '''\u0120newer''',
											    '''\u0120wider''',
											    '''<unk>''',
											    '''<|endoftext|>''',
											]
											__lowerCamelCase      :      int         =dict(zip(__lowercase						,    range(len(__lowercase			)			)			)			)
											__lowerCamelCase      :      Dict         =['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
											__lowerCamelCase      :      Any         ={'''unk_token''': '''<unk>'''}

											__lowerCamelCase      :      List[Any]         =os.path.join(self.tmpdirname						,    VOCAB_FILES_NAMES['''vocab_file''']			)
											__lowerCamelCase      :      Optional[int]         =os.path.join(self.tmpdirname						,    VOCAB_FILES_NAMES['''merges_file''']			)
											with open(self.vocab_file						,    '''w'''						,    encoding='''utf-8'''			) as fp:
																		fp.write(json.dumps(__lowercase			) + '''\n'''			)
											with open(self.merges_file						,    '''w'''						,    encoding='''utf-8'''			) as fp:
																		fp.write('''\n'''.join(__lowercase			)			)




				def 					__lowercase  (     self  :Optional[int]						,    **__lowercase  :Tuple			):
											kwargs.update(self.special_tokens_map			)
											return CodeGenTokenizer.from_pretrained(self.tmpdirname						,    **__lowercase			)




				def 					__lowercase  (     self  :Any						,    **__lowercase  :Tuple			):
											kwargs.update(self.special_tokens_map			)
											return CodeGenTokenizerFast.from_pretrained(self.tmpdirname						,    **__lowercase			)




				def 					__lowercase  (     self  :List[str]						,    __lowercase  :Union[str, Any]			):
											__lowerCamelCase      :      List[str]         ='''lower newer'''
											__lowerCamelCase      :      Optional[Any]         ='''lower newer'''
											return input_text, output_text




				def 					__lowercase  (     self  :Dict			):
											__lowerCamelCase      :      str         =CodeGenTokenizer(self.vocab_file						,    self.merges_file						,    **self.special_tokens_map			)
											__lowerCamelCase      :      str         ='''lower newer'''
											__lowerCamelCase      :      int         =['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
											__lowerCamelCase      :      Tuple         =tokenizer.tokenize(__lowercase						,    add_prefix_space=__lowercase			)
											self.assertListEqual(__lowercase						,    __lowercase			)

											__lowerCamelCase      :      Tuple         =tokens + [tokenizer.unk_token]
											__lowerCamelCase      :      str         =[14, 15, 10, 9, 3, 2, 15, 19]
											self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase			)						,    __lowercase			)




				def 					__lowercase  (     self  :Union[str, Any]			):
											if not self.test_rust_tokenizer:
																		return

											__lowerCamelCase      :      str         =self.get_tokenizer()
											__lowerCamelCase      :      str         =self.get_rust_tokenizer(add_prefix_space=__lowercase			)

											__lowerCamelCase      :      Any         ='''lower newer'''

											# Testing tokenization
											__lowerCamelCase      :      Any         =tokenizer.tokenize(__lowercase						,    add_prefix_space=__lowercase			)
											__lowerCamelCase      :      List[Any]         =rust_tokenizer.tokenize(__lowercase			)
											self.assertListEqual(__lowercase						,    __lowercase			)

											# Testing conversion to ids without special tokens
											__lowerCamelCase      :      Optional[int]         =tokenizer.encode(__lowercase						,    add_special_tokens=__lowercase						,    add_prefix_space=__lowercase			)
											__lowerCamelCase      :      List[str]         =rust_tokenizer.encode(__lowercase						,    add_special_tokens=__lowercase			)
											self.assertListEqual(__lowercase						,    __lowercase			)

											# Testing conversion to ids with special tokens
											__lowerCamelCase      :      Dict         =self.get_rust_tokenizer(add_prefix_space=__lowercase			)
											__lowerCamelCase      :      List[str]         =tokenizer.encode(__lowercase						,    add_prefix_space=__lowercase			)
											__lowerCamelCase      :      List[str]         =rust_tokenizer.encode(__lowercase			)
											self.assertListEqual(__lowercase						,    __lowercase			)

											# Testing the unknown token
											__lowerCamelCase      :      Optional[int]         =tokens + [rust_tokenizer.unk_token]
											__lowerCamelCase      :      Union[str, Any]         =[14, 15, 10, 9, 3, 2, 15, 19]
											self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__lowercase			)						,    __lowercase			)




				def 					__lowercase  (     self  :Any						,    *__lowercase  :List[Any]						,    **__lowercase  :Any			):
											# It's very difficult to mix/test pretokenization with byte-level
											# And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
											pass




				def 					__lowercase  (     self  :List[str]						,    __lowercase  :Tuple=15			):
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'			):
																									__lowerCamelCase      :      Optional[Any]         =self.rust_tokenizer_class.from_pretrained(__lowercase						,    **__lowercase			)

																									# Simple input
																									__lowerCamelCase      :      Optional[int]         ='''This is a simple input'''
																									__lowerCamelCase      :      Dict         =['''This is a simple input 1''', '''This is a simple input 2''']
																									__lowerCamelCase      :      Any         =('''This is a simple input''', '''This is a pair''')
																									__lowerCamelCase      :      Dict         =[
																									    ('''This is a simple input 1''', '''This is a simple input 2'''),
																									    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
																									]

																									# Simple input tests
																									self.assertRaises(__lowercase						,    tokenizer_r.encode						,    __lowercase						,    max_length=__lowercase						,    padding='''max_length'''			)

																									# Simple input
																									self.assertRaises(__lowercase						,    tokenizer_r.encode_plus						,    __lowercase						,    max_length=__lowercase						,    padding='''max_length'''			)

																									# Simple input
																									self.assertRaises(
																									    __lowercase						,    tokenizer_r.batch_encode_plus						,    __lowercase						,    max_length=__lowercase						,    padding='''max_length'''						,    )

																									# Pair input
																									self.assertRaises(__lowercase						,    tokenizer_r.encode						,    __lowercase						,    max_length=__lowercase						,    padding='''max_length'''			)

																									# Pair input
																									self.assertRaises(__lowercase						,    tokenizer_r.encode_plus						,    __lowercase						,    max_length=__lowercase						,    padding='''max_length'''			)

																									# Pair input
																									self.assertRaises(
																									    __lowercase						,    tokenizer_r.batch_encode_plus						,    __lowercase						,    max_length=__lowercase						,    padding='''max_length'''						,    )




				def 					__lowercase  (     self  :Tuple			):
											__lowerCamelCase      :      Any         =CodeGenTokenizer.from_pretrained(self.tmpdirname						,    pad_token='''<pad>'''			)

											# Simple input
											__lowerCamelCase      :      Optional[int]         ='''This is a simple input'''
											__lowerCamelCase      :      Dict         =['''This is a simple input looooooooong''', '''This is a simple input''']
											__lowerCamelCase      :      int         =('''This is a simple input''', '''This is a pair''')
											__lowerCamelCase      :      int         =[
											    ('''This is a simple input loooooong''', '''This is a simple input'''),
											    ('''This is a simple pair loooooong''', '''This is a simple pair'''),
											]

											__lowerCamelCase      :      int         =tokenizer.pad_token_id

											__lowerCamelCase      :      List[str]         =tokenizer(__lowercase						,    padding='''max_length'''						,    max_length=30						,    return_tensors='''np'''			)
											__lowerCamelCase      :      Optional[Any]         =tokenizer(__lowercase						,    padding=__lowercase						,    truncate=__lowercase						,    return_tensors='''np'''			)
											__lowerCamelCase      :      str         =tokenizer(*__lowercase						,    padding='''max_length'''						,    max_length=60						,    return_tensors='''np'''			)
											__lowerCamelCase      :      Optional[Any]         =tokenizer(__lowercase						,    padding=__lowercase						,    truncate=__lowercase						,    return_tensors='''np'''			)

											# s
											# test single string max_length padding
											self.assertEqual(out_s['''input_ids'''].shape[-1]						,    30			)
											self.assertTrue(pad_token_id in out_s['''input_ids''']			)
											self.assertTrue(0 in out_s['''attention_mask''']			)

											# s2
											# test automatic padding
											self.assertEqual(out_sa['''input_ids'''].shape[-1]						,    33			)
											# long slice doesn't have padding
											self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]			)
											self.assertFalse(0 in out_sa['''attention_mask'''][0]			)
											# short slice does have padding
											self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]			)
											self.assertTrue(0 in out_sa['''attention_mask'''][1]			)

											# p
											# test single pair max_length padding
											self.assertEqual(out_p['''input_ids'''].shape[-1]						,    60			)
											self.assertTrue(pad_token_id in out_p['''input_ids''']			)
											self.assertTrue(0 in out_p['''attention_mask''']			)

											# p2
											# test automatic padding pair
											self.assertEqual(out_pa['''input_ids'''].shape[-1]						,    52			)
											# long slice pair doesn't have padding
											self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]			)
											self.assertFalse(0 in out_pa['''attention_mask'''][0]			)
											# short slice pair does have padding
											self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]			)
											self.assertTrue(0 in out_pa['''attention_mask'''][1]			)




				def 					__lowercase  (     self  :str			):
											__lowerCamelCase      :      Union[str, Any]         ='''$$$'''
											__lowerCamelCase      :      Any         =CodeGenTokenizer.from_pretrained(self.tmpdirname						,    bos_token=__lowercase						,    add_bos_token=__lowercase			)

											__lowerCamelCase      :      Tuple         ='''This is a simple input'''
											__lowerCamelCase      :      Optional[int]         =['''This is a simple input 1''', '''This is a simple input 2''']

											__lowerCamelCase      :      Optional[int]         =tokenizer.bos_token_id

											__lowerCamelCase      :      Dict         =tokenizer(__lowercase			)
											__lowerCamelCase      :      List[Any]         =tokenizer(__lowercase			)

											self.assertEqual(out_s.input_ids[0]						,    __lowercase			)
											self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids			)			)

											__lowerCamelCase      :      List[Any]         =tokenizer.decode(out_s.input_ids			)
											__lowerCamelCase      :      Dict         =tokenizer.batch_decode(out_sa.input_ids			)

											self.assertEqual(decode_s.split()[0]						,    __lowercase			)
											self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa			)			)




				@slow
				def 					__lowercase  (     self  :Optional[Any]			):
											__lowerCamelCase      :      Union[str, Any]         =CodeGenTokenizer.from_pretrained('''Salesforce/codegen-350M-mono'''			)

											__lowerCamelCase      :      List[Any]         ='''\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'''
											__lowerCamelCase      :      Tuple         ='''\nif len_a > len_b:      result = a\nelse:      result = b'''

											__lowerCamelCase      :      List[str]         =tokenizer.encode(__lowercase			)
											__lowerCamelCase      :      Tuple         =['''^#''', re.escape('''<|endoftext|>'''			), '''^\'\'\'''', '''^"""''', '''\n\n\n''']
											__lowerCamelCase      :      Optional[Any]         =tokenizer.decode(__lowercase						,    truncate_before_pattern=__lowercase			)
											self.assertEqual(__lowercase						,    __lowercase			)




				def 					__lowercase  (     self  :List[str]			):
											pass











"""simple docstring"""





import unittest

from .lib import (
    Matrix,
    Vector,
    axpy,
    square_zero_matrix,
    unit_basis_vector,
    zero_vector,
)
class 		SCREAMING_SNAKE_CASE_							(    unittest.TestCase   ):
				"""simple docstring"""




				def 					__lowercase  (     self  :Tuple			):
											__lowerCamelCase      :      Any         =Vector([1, 2, 3]			)
											self.assertEqual(x.component(0			)						,    1			)
											self.assertEqual(x.component(2			)						,    3			)
											__lowerCamelCase      :      Any         =Vector()




				def 					__lowercase  (     self  :Dict			):
											__lowerCamelCase      :      Tuple         =Vector([0, 0, 0, 0, 0, 1]			)
											self.assertEqual(str(__lowercase			)						,    '''(0,0,0,0,0,1)'''			)




				def 					__lowercase  (     self  :Dict			):
											__lowerCamelCase      :      int         =Vector([1, 2, 3, 4]			)
											self.assertEqual(len(__lowercase			)						,    4			)




				def 					__lowercase  (     self  :Dict			):
											__lowerCamelCase      :      Optional[Any]         =Vector([1, 2]			)
											__lowerCamelCase      :      Dict         =Vector([1, 2, 3, 4, 5]			)
											__lowerCamelCase      :      List[Any]         =Vector([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]			)
											__lowerCamelCase      :      int         =Vector([1, -1, 1, -1, 2, -3, 4, -5]			)
											self.assertAlmostEqual(x.euclidean_length()						,    2.236						,    3			)
											self.assertAlmostEqual(y.euclidean_length()						,    7.416						,    3			)
											self.assertEqual(z.euclidean_length()						,    0			)
											self.assertAlmostEqual(w.euclidean_length()						,    7.616						,    3			)




				def 					__lowercase  (     self  :Optional[int]			):
											__lowerCamelCase      :      Tuple         =Vector([1, 2, 3]			)
											__lowerCamelCase      :      Any         =Vector([1, 1, 1]			)
											self.assertEqual((x + y).component(0			)						,    2			)
											self.assertEqual((x + y).component(1			)						,    3			)
											self.assertEqual((x + y).component(2			)						,    4			)




				def 					__lowercase  (     self  :str			):
											__lowerCamelCase      :      Union[str, Any]         =Vector([1, 2, 3]			)
											__lowerCamelCase      :      int         =Vector([1, 1, 1]			)
											self.assertEqual((x - y).component(0			)						,    0			)
											self.assertEqual((x - y).component(1			)						,    1			)
											self.assertEqual((x - y).component(2			)						,    2			)




				def 					__lowercase  (     self  :int			):
											__lowerCamelCase      :      List[Any]         =Vector([1, 2, 3]			)
											__lowerCamelCase      :      List[Any]         =Vector([2, -1, 4]			)  # for test of dot product
											__lowerCamelCase      :      Any         =Vector([1, -2, -1]			)
											self.assertEqual(str(x * 3.0			)						,    '''(3.0,6.0,9.0)'''			)
											self.assertEqual((a * b)						,    0			)




				def 					__lowercase  (     self  :List[Any]			):
											self.assertEqual(str(zero_vector(10			)			).count('''0'''			)						,    10			)




				def 					__lowercase  (     self  :Union[str, Any]			):
											self.assertEqual(str(unit_basis_vector(3						,    1			)			)						,    '''(0,1,0)'''			)




				def 					__lowercase  (     self  :List[Any]			):
											__lowerCamelCase      :      Any         =Vector([1, 2, 3]			)
											__lowerCamelCase      :      Optional[int]         =Vector([1, 0, 1]			)
											self.assertEqual(str(axpy(2						,    __lowercase						,    __lowercase			)			)						,    '''(3,4,7)'''			)




				def 					__lowercase  (     self  :Dict			):
											__lowerCamelCase      :      List[Any]         =Vector([1, 0, 0, 0, 0, 0]			)
											__lowerCamelCase      :      Optional[int]         =x.copy()
											self.assertEqual(str(__lowercase			)						,    str(__lowercase			)			)




				def 					__lowercase  (     self  :int			):
											__lowerCamelCase      :      str         =Vector([1, 0, 0]			)
											x.change_component(0						,    0			)
											x.change_component(1						,    1			)
											self.assertEqual(str(__lowercase			)						,    '''(0,1,0)'''			)




				def 					__lowercase  (     self  :int			):
											__lowerCamelCase      :      Any         =Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]						,    3						,    3			)
											self.assertEqual('''|1,2,3|\n|2,4,5|\n|6,7,8|\n'''						,    str(__lowercase			)			)




				def 					__lowercase  (     self  :int			):
											__lowerCamelCase      :      Tuple         =Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]						,    3						,    3			)
											__lowerCamelCase      :      List[Any]         =[[-3, -14, -10], [-5, -10, -5], [-2, -1, 0]]
											for x in range(a.height()			):
																		for y in range(a.width()			):
																									self.assertEqual(minors[x][y]						,    a.minor(__lowercase						,    __lowercase			)			)




				def 					__lowercase  (     self  :Optional[int]			):
											__lowerCamelCase      :      Optional[Any]         =Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]						,    3						,    3			)
											__lowerCamelCase      :      Tuple         =[[-3, 14, -10], [5, -10, 5], [-2, 1, 0]]
											for x in range(a.height()			):
																		for y in range(a.width()			):
																									self.assertEqual(cofactors[x][y]						,    a.cofactor(__lowercase						,    __lowercase			)			)




				def 					__lowercase  (     self  :Tuple			):
											__lowerCamelCase      :      Tuple         =Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]						,    3						,    3			)
											self.assertEqual(-5						,    a.determinant()			)




				def 					__lowercase  (     self  :int			):
											__lowerCamelCase      :      Union[str, Any]         =Matrix([[1, 2, 3], [4, 5, 6], [7, 8, 9]]						,    3						,    3			)
											__lowerCamelCase      :      Tuple         =Vector([1, 2, 3]			)
											self.assertEqual('''(14,32,50)'''						,    str(a * x			)			)
											self.assertEqual('''|2,4,6|\n|8,10,12|\n|14,16,18|\n'''						,    str(a * 2			)			)




				def 					__lowercase  (     self  :Optional[Any]			):
											__lowerCamelCase      :      Optional[int]         =Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]						,    3						,    3			)
											a.change_component(0						,    2						,    5			)
											self.assertEqual('''|1,2,5|\n|2,4,5|\n|6,7,8|\n'''						,    str(__lowercase			)			)




				def 					__lowercase  (     self  :str			):
											__lowerCamelCase      :      str         =Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]						,    3						,    3			)
											self.assertEqual(7						,    a.component(2						,    1			)						,    0.01			)




				def 					__lowercase  (     self  :Optional[int]			):
											__lowerCamelCase      :      List[str]         =Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]						,    3						,    3			)
											__lowerCamelCase      :      List[str]         =Matrix([[1, 2, 7], [2, 4, 5], [6, 7, 10]]						,    3						,    3			)
											self.assertEqual('''|2,4,10|\n|4,8,10|\n|12,14,18|\n'''						,    str(a + b			)			)




				def 					__lowercase  (     self  :Union[str, Any]			):
											__lowerCamelCase      :      int         =Matrix([[1, 2, 3], [2, 4, 5], [6, 7, 8]]						,    3						,    3			)
											__lowerCamelCase      :      Optional[int]         =Matrix([[1, 2, 7], [2, 4, 5], [6, 7, 10]]						,    3						,    3			)
											self.assertEqual('''|0,0,-4|\n|0,0,0|\n|0,0,-2|\n'''						,    str(a - b			)			)




				def 					__lowercase  (     self  :Any			):
											self.assertEqual(
											    '''|0,0,0,0,0|\n|0,0,0,0,0|\n|0,0,0,0,0|\n|0,0,0,0,0|\n|0,0,0,0,0|\n'''						,    str(square_zero_matrix(5			)			)						,    )


if __name__ == "__main__":
							unittest.main()




