




import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 						UpperCAmelCase  (							UpperCAmelCase__			,  unittest.TestCase     ):







						'''simple docstring'''





						lowerCAmelCase_							=  CodeGenTokenizer
						lowerCAmelCase_							=  CodeGenTokenizerFast
						lowerCAmelCase_							=  True
						lowerCAmelCase_							=  {'''add_prefix_space''': True}
						lowerCAmelCase_							=  False






						def 			snake_case__		(		self				:	Any		):






							"""simple docstring"""






							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							snake_case_						       =	[
							    "l",
							    "o",
							    "w",
							    "e",
							    "r",
							    "s",
							    "t",
							    "i",
							    "d",
							    "n",
							    "\u0120",
							    "\u0120l",
							    "\u0120n",
							    "\u0120lo",
							    "\u0120low",
							    "er",
							    "\u0120lowest",
							    "\u0120newer",
							    "\u0120wider",
							    "<unk>",
							    "<|endoftext|>",
							]
							snake_case_						       =	dict(zip(__lowercase			,      range(len(__lowercase		)		)		)		)
							snake_case_						       =	["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
							snake_case_						       =	{"unk_token": "<unk>"}

							snake_case_						       =	os.path.join(self.tmpdirname			,      VOCAB_FILES_NAMES["vocab_file"]		)
							snake_case_						       =	os.path.join(self.tmpdirname			,      VOCAB_FILES_NAMES["merges_file"]		)
							with open(self.vocab_file			,      "w"			,      encoding="utf-8"		) as fp:
								fp.write(json.dumps(__lowercase		) + "\n"		)
							with open(self.merges_file			,      "w"			,      encoding="utf-8"		) as fp:
								fp.write("\n".join(__lowercase		)		)






						def 			snake_case__		(		self				:	Tuple			,      **__lowercase				:	Any		):






							"""simple docstring"""






							kwargs.update(self.special_tokens_map		)
							return CodeGenTokenizer.from_pretrained(self.tmpdirname			,      **__lowercase		)






						def 			snake_case__		(		self				:	Tuple			,      **__lowercase				:	int		):






							"""simple docstring"""






							kwargs.update(self.special_tokens_map		)
							return CodeGenTokenizerFast.from_pretrained(self.tmpdirname			,      **__lowercase		)






						def 			snake_case__		(		self				:	Optional[int]			,      __lowercase				:	str		):






							"""simple docstring"""






							snake_case_						       =	"lower newer"
							snake_case_						       =	"lower newer"
							return input_text, output_text






						def 			snake_case__		(		self				:	Optional[Any]		):






							"""simple docstring"""






							snake_case_						       =	CodeGenTokenizer(self.vocab_file			,      self.merges_file			,      **self.special_tokens_map		)
							snake_case_						       =	"lower newer"
							snake_case_						       =	["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
							snake_case_						       =	tokenizer.tokenize(__lowercase			,      add_prefix_space=__lowercase		)
							self.assertListEqual(__lowercase			,      __lowercase		)

							snake_case_						       =	tokens + [tokenizer.unk_token]
							snake_case_						       =	[14, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase		)			,      __lowercase		)






						def 			snake_case__		(		self				:	Any		):






							"""simple docstring"""






							if not self.test_rust_tokenizer:
								return

							snake_case_						       =	self.get_tokenizer()
							snake_case_						       =	self.get_rust_tokenizer(add_prefix_space=__lowercase		)

							snake_case_						       =	"lower newer"

							# Testing tokenization
							snake_case_						       =	tokenizer.tokenize(__lowercase			,      add_prefix_space=__lowercase		)
							snake_case_						       =	rust_tokenizer.tokenize(__lowercase		)
							self.assertListEqual(__lowercase			,      __lowercase		)

							# Testing conversion to ids without special tokens
							snake_case_						       =	tokenizer.encode(__lowercase			,      add_special_tokens=__lowercase			,      add_prefix_space=__lowercase		)
							snake_case_						       =	rust_tokenizer.encode(__lowercase			,      add_special_tokens=__lowercase		)
							self.assertListEqual(__lowercase			,      __lowercase		)

							# Testing conversion to ids with special tokens
							snake_case_						       =	self.get_rust_tokenizer(add_prefix_space=__lowercase		)
							snake_case_						       =	tokenizer.encode(__lowercase			,      add_prefix_space=__lowercase		)
							snake_case_						       =	rust_tokenizer.encode(__lowercase		)
							self.assertListEqual(__lowercase			,      __lowercase		)

							# Testing the unknown token
							snake_case_						       =	tokens + [rust_tokenizer.unk_token]
							snake_case_						       =	[14, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__lowercase		)			,      __lowercase		)






						def 			snake_case__		(		self				:	Any			,      *__lowercase				:	List[str]			,      **__lowercase				:	Optional[Any]		):






							"""simple docstring"""






							pass






						def 			snake_case__		(		self				:	Any			,      __lowercase				:	List[str]=15		):






							"""simple docstring"""






							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
									snake_case_						       =	self.rust_tokenizer_class.from_pretrained(__lowercase			,      **__lowercase		)

									# Simple input
									snake_case_						       =	"This is a simple input"
									snake_case_						       =	["This is a simple input 1", "This is a simple input 2"]
									snake_case_						       =	("This is a simple input", "This is a pair")
									snake_case_						       =	[
									    ("This is a simple input 1", "This is a simple input 2"),
									    ("This is a simple pair 1", "This is a simple pair 2"),
									]

									# Simple input tests
									self.assertRaises(__lowercase			,      tokenizer_r.encode			,      __lowercase			,      max_length=__lowercase			,      padding="max_length"		)

									# Simple input
									self.assertRaises(__lowercase			,      tokenizer_r.encode_plus			,      __lowercase			,      max_length=__lowercase			,      padding="max_length"		)

									# Simple input
									self.assertRaises(
									    __lowercase			,      tokenizer_r.batch_encode_plus			,      __lowercase			,      max_length=__lowercase			,      padding="max_length"			,      )

									# Pair input
									self.assertRaises(__lowercase			,      tokenizer_r.encode			,      __lowercase			,      max_length=__lowercase			,      padding="max_length"		)

									# Pair input
									self.assertRaises(__lowercase			,      tokenizer_r.encode_plus			,      __lowercase			,      max_length=__lowercase			,      padding="max_length"		)

									# Pair input
									self.assertRaises(
									    __lowercase			,      tokenizer_r.batch_encode_plus			,      __lowercase			,      max_length=__lowercase			,      padding="max_length"			,      )






						def 			snake_case__		(		self				:	Any		):






							"""simple docstring"""






							snake_case_						       =	CodeGenTokenizer.from_pretrained(self.tmpdirname			,      pad_token="<pad>"		)

							# Simple input
							snake_case_						       =	"This is a simple input"
							snake_case_						       =	["This is a simple input looooooooong", "This is a simple input"]
							snake_case_						       =	("This is a simple input", "This is a pair")
							snake_case_						       =	[
							    ("This is a simple input loooooong", "This is a simple input"),
							    ("This is a simple pair loooooong", "This is a simple pair"),
							]

							snake_case_						       =	tokenizer.pad_token_id

							snake_case_						       =	tokenizer(__lowercase			,      padding="max_length"			,      max_length=30			,      return_tensors="np"		)
							snake_case_						       =	tokenizer(__lowercase			,      padding=__lowercase			,      truncate=__lowercase			,      return_tensors="np"		)
							snake_case_						       =	tokenizer(*__lowercase			,      padding="max_length"			,      max_length=60			,      return_tensors="np"		)
							snake_case_						       =	tokenizer(__lowercase			,      padding=__lowercase			,      truncate=__lowercase			,      return_tensors="np"		)

							# s
							# test single string max_length padding
							self.assertEqual(out_s["input_ids"].shape[-1]			,      30		)
							self.assertTrue(pad_token_id in out_s["input_ids"]		)
							self.assertTrue(0 in out_s["attention_mask"]		)

							# s2
							# test automatic padding
							self.assertEqual(out_sa["input_ids"].shape[-1]			,      33		)
							# long slice doesn't have padding
							self.assertFalse(pad_token_id in out_sa["input_ids"][0]		)
							self.assertFalse(0 in out_sa["attention_mask"][0]		)
							# short slice does have padding
							self.assertTrue(pad_token_id in out_sa["input_ids"][1]		)
							self.assertTrue(0 in out_sa["attention_mask"][1]		)

							# p
							# test single pair max_length padding
							self.assertEqual(out_p["input_ids"].shape[-1]			,      60		)
							self.assertTrue(pad_token_id in out_p["input_ids"]		)
							self.assertTrue(0 in out_p["attention_mask"]		)

							# p2
							# test automatic padding pair
							self.assertEqual(out_pa["input_ids"].shape[-1]			,      52		)
							# long slice pair doesn't have padding
							self.assertFalse(pad_token_id in out_pa["input_ids"][0]		)
							self.assertFalse(0 in out_pa["attention_mask"][0]		)
							# short slice pair does have padding
							self.assertTrue(pad_token_id in out_pa["input_ids"][1]		)
							self.assertTrue(0 in out_pa["attention_mask"][1]		)






						def 			snake_case__		(		self				:	List[str]		):






							"""simple docstring"""






							snake_case_						       =	"$$$"
							snake_case_						       =	CodeGenTokenizer.from_pretrained(self.tmpdirname			,      bos_token=__lowercase			,      add_bos_token=__lowercase		)

							snake_case_						       =	"This is a simple input"
							snake_case_						       =	["This is a simple input 1", "This is a simple input 2"]

							snake_case_						       =	tokenizer.bos_token_id

							snake_case_						       =	tokenizer(__lowercase		)
							snake_case_						       =	tokenizer(__lowercase		)

							self.assertEqual(out_s.input_ids[0]			,      __lowercase		)
							self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids		)		)

							snake_case_						       =	tokenizer.decode(out_s.input_ids		)
							snake_case_						       =	tokenizer.batch_decode(out_sa.input_ids		)

							self.assertEqual(decode_s.split()[0]			,      __lowercase		)
							self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa		)		)






						@slow
						def 			snake_case__		(		self				:	Union[str, Any]		):






							"""simple docstring"""






							snake_case_						       =	CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono"		)

							snake_case_						       =	"\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
							snake_case_						       =	"\nif len_a > len_b:      result = a\nelse:      result = b"

							snake_case_						       =	tokenizer.encode(__lowercase		)
							snake_case_						       =	["^#", re.escape("<|endoftext|>"		), "^'''", "^\"\"\"", "\n\n\n"]
							snake_case_						       =	tokenizer.decode(__lowercase			,      truncate_before_pattern=__lowercase		)
							self.assertEqual(__lowercase			,      __lowercase		)






						def 			snake_case__		(		self				:	List[Any]		):






							"""simple docstring"""






							pass






def 		lowerCamelCase__       (						):







	'''simple docstring'''





	for n in range(1	,      1000000				):
		yield n * (n + 1) // 2


def 		lowerCamelCase__       (						_A				):







	'''simple docstring'''





	snake_case_						       =	1
	snake_case_						       =	2
	while i * i <= n:
		snake_case_						       =	0
		while n % i == 0:
			n //= i
			multiplicity += 1
		divisors_count *= multiplicity + 1
		i += 1
	if n > 1:
		divisors_count *= 2
	return divisors_count


def 		lowerCamelCase__       (						):







	'''simple docstring'''





	return next(i for i in triangle_number_generator() if count_divisors(_A				) > 500				)


if __name__ == "__main__":
		print(solution())
