



import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class       __lowerCAmelCase   (      A    ,						unittest.TestCase						):
			UpperCamelCase        =					CodeGenTokenizer
			UpperCamelCase        =					CodeGenTokenizerFast
			UpperCamelCase        =					True
			UpperCamelCase        =					{'''add_prefix_space''': True}
			UpperCamelCase        =					False



			def    _lowerCamelCase	(						self       :  Dict)       ->   Tuple:



						"""simple docstring"""






						super().setUp()

						# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
						_UpperCAmelCase			=       [
						    'l',
						    'o',
						    'w',
						    'e',
						    'r',
						    's',
						    't',
						    'i',
						    'd',
						    'n',
						    '\u0120',
						    '\u0120l',
						    '\u0120n',
						    '\u0120lo',
						    '\u0120low',
						    'er',
						    '\u0120lowest',
						    '\u0120newer',
						    '\u0120wider',
						    '<unk>',
						    '<|endoftext|>',
						]
						_UpperCAmelCase			=       dict(zip(A			,		range(len(A))))
						_UpperCAmelCase			=       ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
						_UpperCAmelCase			=       {'unk_token': '<unk>'}

						_UpperCAmelCase			=       os.path.join(self.tmpdirname			,		VOCAB_FILES_NAMES['vocab_file'])
						_UpperCAmelCase			=       os.path.join(self.tmpdirname			,		VOCAB_FILES_NAMES['merges_file'])
						with open(self.vocab_file			,		'w'			,		encoding='utf-8') as fp:
									fp.write(json.dumps(A) + '\n')
						with open(self.merges_file			,		'w'			,		encoding='utf-8') as fp:
									fp.write('\n'.join(A))



			def    _lowerCamelCase	(						self       :  int			,		**A       :  Union[str, Any])       ->   Optional[Any]:



						"""simple docstring"""






						kwargs.update(self.special_tokens_map)
						return CodeGenTokenizer.from_pretrained(self.tmpdirname			,		**A)



			def    _lowerCamelCase	(						self       :  Tuple			,		**A       :  Any)       ->   List[str]:



						"""simple docstring"""






						kwargs.update(self.special_tokens_map)
						return CodeGenTokenizerFast.from_pretrained(self.tmpdirname			,		**A)



			def    _lowerCamelCase	(						self       :  Union[str, Any]			,		A       :  List[Any])       ->   Union[str, Any]:



						"""simple docstring"""






						_UpperCAmelCase			=       'lower newer'
						_UpperCAmelCase			=       'lower newer'
						return input_text, output_text



			def    _lowerCamelCase	(						self       :  Dict)       ->   Optional[int]:



						"""simple docstring"""






						_UpperCAmelCase			=       CodeGenTokenizer(self.vocab_file			,		self.merges_file			,		**self.special_tokens_map)
						_UpperCAmelCase			=       'lower newer'
						_UpperCAmelCase			=       ['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
						_UpperCAmelCase			=       tokenizer.tokenize(A			,		add_prefix_space=A)
						self.assertListEqual(A			,		A)

						_UpperCAmelCase			=       tokens + [tokenizer.unk_token]
						_UpperCAmelCase			=       [14, 15, 10, 9, 3, 2, 15, 19]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(A)			,		A)



			def    _lowerCamelCase	(						self       :  int)       ->   List[Any]:



						"""simple docstring"""






						if not self.test_rust_tokenizer:
									return

						_UpperCAmelCase			=       self.get_tokenizer()
						_UpperCAmelCase			=       self.get_rust_tokenizer(add_prefix_space=A)

						_UpperCAmelCase			=       'lower newer'

						# Testing tokenization
						_UpperCAmelCase			=       tokenizer.tokenize(A			,		add_prefix_space=A)
						_UpperCAmelCase			=       rust_tokenizer.tokenize(A)
						self.assertListEqual(A			,		A)

						# Testing conversion to ids without special tokens
						_UpperCAmelCase			=       tokenizer.encode(A			,		add_special_tokens=A			,		add_prefix_space=A)
						_UpperCAmelCase			=       rust_tokenizer.encode(A			,		add_special_tokens=A)
						self.assertListEqual(A			,		A)

						# Testing conversion to ids with special tokens
						_UpperCAmelCase			=       self.get_rust_tokenizer(add_prefix_space=A)
						_UpperCAmelCase			=       tokenizer.encode(A			,		add_prefix_space=A)
						_UpperCAmelCase			=       rust_tokenizer.encode(A)
						self.assertListEqual(A			,		A)

						# Testing the unknown token
						_UpperCAmelCase			=       tokens + [rust_tokenizer.unk_token]
						_UpperCAmelCase			=       [14, 15, 10, 9, 3, 2, 15, 19]
						self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(A)			,		A)



			def    _lowerCamelCase	(						self       :  int			,		*A       :  int			,		**A       :  List[str])       ->   List[str]:



						"""simple docstring"""






						pass



			def    _lowerCamelCase	(						self       :  Dict			,		A       :  Optional[Any]=15)       ->   int:



						"""simple docstring"""






						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"):
												_UpperCAmelCase			=       self.rust_tokenizer_class.from_pretrained(A			,		**A)

												# Simple input
												_UpperCAmelCase			=       'This is a simple input'
												_UpperCAmelCase			=       ['This is a simple input 1', 'This is a simple input 2']
												_UpperCAmelCase			=       ('This is a simple input', 'This is a pair')
												_UpperCAmelCase			=       [
												    ('This is a simple input 1', 'This is a simple input 2'),
												    ('This is a simple pair 1', 'This is a simple pair 2'),
												]

												# Simple input tests
												self.assertRaises(A			,		tokenizer_r.encode			,		A			,		max_length=A			,		padding='max_length')

												# Simple input
												self.assertRaises(A			,		tokenizer_r.encode_plus			,		A			,		max_length=A			,		padding='max_length')

												# Simple input
												self.assertRaises(
												    A			,		tokenizer_r.batch_encode_plus			,		A			,		max_length=A			,		padding='max_length'			,		)

												# Pair input
												self.assertRaises(A			,		tokenizer_r.encode			,		A			,		max_length=A			,		padding='max_length')

												# Pair input
												self.assertRaises(A			,		tokenizer_r.encode_plus			,		A			,		max_length=A			,		padding='max_length')

												# Pair input
												self.assertRaises(
												    A			,		tokenizer_r.batch_encode_plus			,		A			,		max_length=A			,		padding='max_length'			,		)



			def    _lowerCamelCase	(						self       :  Dict)       ->   Tuple:



						"""simple docstring"""






						_UpperCAmelCase			=       CodeGenTokenizer.from_pretrained(self.tmpdirname			,		pad_token='<pad>')

						# Simple input
						_UpperCAmelCase			=       'This is a simple input'
						_UpperCAmelCase			=       ['This is a simple input looooooooong', 'This is a simple input']
						_UpperCAmelCase			=       ('This is a simple input', 'This is a pair')
						_UpperCAmelCase			=       [
						    ('This is a simple input loooooong', 'This is a simple input'),
						    ('This is a simple pair loooooong', 'This is a simple pair'),
						]

						_UpperCAmelCase			=       tokenizer.pad_token_id

						_UpperCAmelCase			=       tokenizer(A			,		padding='max_length'			,		max_length=30			,		return_tensors='np')
						_UpperCAmelCase			=       tokenizer(A			,		padding=A			,		truncate=A			,		return_tensors='np')
						_UpperCAmelCase			=       tokenizer(*A			,		padding='max_length'			,		max_length=60			,		return_tensors='np')
						_UpperCAmelCase			=       tokenizer(A			,		padding=A			,		truncate=A			,		return_tensors='np')

						# s
						# test single string max_length padding
						self.assertEqual(out_s['input_ids'].shape[-1]			,		30)
						self.assertTrue(pad_token_id in out_s['input_ids'])
						self.assertTrue(0 in out_s['attention_mask'])

						# s2
						# test automatic padding
						self.assertEqual(out_sa['input_ids'].shape[-1]			,		33)
						# long slice doesn't have padding
						self.assertFalse(pad_token_id in out_sa['input_ids'][0])
						self.assertFalse(0 in out_sa['attention_mask'][0])
						# short slice does have padding
						self.assertTrue(pad_token_id in out_sa['input_ids'][1])
						self.assertTrue(0 in out_sa['attention_mask'][1])

						# p
						# test single pair max_length padding
						self.assertEqual(out_p['input_ids'].shape[-1]			,		60)
						self.assertTrue(pad_token_id in out_p['input_ids'])
						self.assertTrue(0 in out_p['attention_mask'])

						# p2
						# test automatic padding pair
						self.assertEqual(out_pa['input_ids'].shape[-1]			,		52)
						# long slice pair doesn't have padding
						self.assertFalse(pad_token_id in out_pa['input_ids'][0])
						self.assertFalse(0 in out_pa['attention_mask'][0])
						# short slice pair does have padding
						self.assertTrue(pad_token_id in out_pa['input_ids'][1])
						self.assertTrue(0 in out_pa['attention_mask'][1])



			def    _lowerCamelCase	(						self       :  Tuple)       ->   Union[str, Any]:



						"""simple docstring"""






						_UpperCAmelCase			=       '$$$'
						_UpperCAmelCase			=       CodeGenTokenizer.from_pretrained(self.tmpdirname			,		bos_token=A			,		add_bos_token=A)

						_UpperCAmelCase			=       'This is a simple input'
						_UpperCAmelCase			=       ['This is a simple input 1', 'This is a simple input 2']

						_UpperCAmelCase			=       tokenizer.bos_token_id

						_UpperCAmelCase			=       tokenizer(A)
						_UpperCAmelCase			=       tokenizer(A)

						self.assertEqual(out_s.input_ids[0]			,		A)
						self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids))

						_UpperCAmelCase			=       tokenizer.decode(out_s.input_ids)
						_UpperCAmelCase			=       tokenizer.batch_decode(out_sa.input_ids)

						self.assertEqual(decode_s.split()[0]			,		A)
						self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa))



			@slow
			def    _lowerCamelCase	(						self       :  int)       ->   Optional[int]:



						"""simple docstring"""






						_UpperCAmelCase			=       CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono')

						_UpperCAmelCase			=       '\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
						_UpperCAmelCase			=       '\nif len_a > len_b:      result = a\nelse:      result = b'

						_UpperCAmelCase			=       tokenizer.encode(A)
						_UpperCAmelCase			=       ['^#', re.escape('<|endoftext|>'), '^\'\'\'', '^"""', '\n\n\n']
						_UpperCAmelCase			=       tokenizer.decode(A			,		truncate_before_pattern=A)
						self.assertEqual(A			,		A)



			def    _lowerCamelCase	(						self       :  Optional[int])       ->   List[Any]:



						"""simple docstring"""






						pass





from __future__ import annotations

from itertools import permutations
from random import randint
from timeit import repeat





def 							A     (					)    ->			tuple[list[int], int]:







			'''simple docstring'''
			_UpperCAmelCase			=       [randint(-1_000							,       1_000   ) for i in range(10   )]
			_UpperCAmelCase			=       randint(-5_000							,       5_000   )
			return (arr, r)


UpperCAmelCase__       =	make_dataset()





def 							A     (					_UpperCAmelCase    :       list[int]							,       _UpperCAmelCase    :       int   )    ->			tuple[int, ...]:







			'''simple docstring'''
			for triplet in permutations(_UpperCAmelCase							,       3   ):
						if sum(_UpperCAmelCase   ) == target:
									return tuple(sorted(_UpperCAmelCase   )   )
			return (0, 0, 0)





def 							A     (					_UpperCAmelCase    :       list[int]							,       _UpperCAmelCase    :       int   )    ->			tuple[int, int, int]:







			'''simple docstring'''
			arr.sort()
			_UpperCAmelCase			=       len(_UpperCAmelCase   )
			for i in range(n - 1   ):
						_UpperCAmelCase	,					_UpperCAmelCase			=       i + 1, n - 1
						while left < right:
									if arr[i] + arr[left] + arr[right] == target:
												return (arr[i], arr[left], arr[right])
									elif arr[i] + arr[left] + arr[right] < target:
												left += 1
									elif arr[i] + arr[left] + arr[right] > target:
												right -= 1
			return (0, 0, 0)





def 							A     (					)    ->			tuple[float, float]:







			'''simple docstring'''
			_UpperCAmelCase			=       '\nfrom __main__ import dataset, triplet_sum1, triplet_sum2\n'
			_UpperCAmelCase			=       '\ntriplet_sum1(*dataset)\n'
			_UpperCAmelCase			=       '\ntriplet_sum2(*dataset)\n'
			_UpperCAmelCase			=       repeat(setup=_UpperCAmelCase							,       stmt=_UpperCAmelCase							,       repeat=5							,       number=10_000   )
			_UpperCAmelCase			=       repeat(setup=_UpperCAmelCase							,       stmt=_UpperCAmelCase							,       repeat=5							,       number=10_000   )
			return (min(_UpperCAmelCase   ), min(_UpperCAmelCase   ))


if __name__ == "__main__":
							from doctest import testmod

							testmod()
							UpperCAmelCase__       =	solution_times()
							print(f"""The time for naive implementation is {times[0]}.""")
							print(f"""The time for optimized implementation is {times[1]}.""")
