





import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class       lowercase					(				__A		,      unittest.TestCase  ):



						"""simple docstring"""

						_UpperCamelCase	:						str    		=						CodeGenTokenizer
						_UpperCamelCase	:						Union[str, Any]    		=						CodeGenTokenizerFast
						_UpperCamelCase	:						str    		=						True
						_UpperCamelCase	:						List[Any]    		=						{'add_prefix_space': True}
						_UpperCamelCase	:						List[Any]    		=						False







						def 		__UpperCAmelCase							(     self      :		Tuple       ):





										'''simple docstring'''







										super().setUp()

										# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
										_snake_case			:      Any	    = [
										    """l""",
										    """o""",
										    """w""",
										    """e""",
										    """r""",
										    """s""",
										    """t""",
										    """i""",
										    """d""",
										    """n""",
										    """\u0120""",
										    """\u0120l""",
										    """\u0120n""",
										    """\u0120lo""",
										    """\u0120low""",
										    """er""",
										    """\u0120lowest""",
										    """\u0120newer""",
										    """\u0120wider""",
										    """<unk>""",
										    """<|endoftext|>""",
										]
										_snake_case			:      Tuple	    = dict(zip(snake_case_			,					range(len(snake_case_       )       )       )       )
										_snake_case			:      Dict	    = ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
										_snake_case			:      Dict	    = {"""unk_token""": """<unk>"""}

										_snake_case			:      Optional[Any]	    = os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES['vocab_file']       )
										_snake_case			:      List[str]	    = os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES['merges_file']       )
										with open(self.vocab_file			,					'w'			,					encoding='utf-8'       ) as fp:
														fp.write(json.dumps(snake_case_       ) + '\n'       )
										with open(self.merges_file			,					'w'			,					encoding='utf-8'       ) as fp:
														fp.write('\n'.join(snake_case_       )       )







						def 		__UpperCAmelCase							(     self      :		Optional[int]			,					**lowerCamelCase_      :		str       ):





										'''simple docstring'''







										kwargs.update(self.special_tokens_map       )
										return CodeGenTokenizer.from_pretrained(self.tmpdirname			,					**snake_case_       )







						def 		__UpperCAmelCase							(     self      :		Tuple			,					**lowerCamelCase_      :		Optional[Any]       ):





										'''simple docstring'''







										kwargs.update(self.special_tokens_map       )
										return CodeGenTokenizerFast.from_pretrained(self.tmpdirname			,					**snake_case_       )







						def 		__UpperCAmelCase							(     self      :		List[str]			,					lowerCamelCase_      :		Any       ):





										'''simple docstring'''







										_snake_case			:      Optional[int]	    = """lower newer"""
										_snake_case			:      int	    = """lower newer"""
										return input_text, output_text







						def 		__UpperCAmelCase							(     self      :		List[Any]       ):





										'''simple docstring'''







										_snake_case			:      List[str]	    = CodeGenTokenizer(self.vocab_file			,					self.merges_file			,					**self.special_tokens_map       )
										_snake_case			:      Optional[Any]	    = """lower newer"""
										_snake_case			:      Dict	    = ["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
										_snake_case			:      List[str]	    = tokenizer.tokenize(snake_case_			,					add_prefix_space=snake_case_       )
										self.assertListEqual(snake_case_			,					snake_case_       )

										_snake_case			:      Optional[Any]	    = tokens + [tokenizer.unk_token]
										_snake_case			:      List[Any]	    = [14, 15, 10, 9, 3, 2, 15, 19]
										self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_       )			,					snake_case_       )







						def 		__UpperCAmelCase							(     self      :		Optional[int]       ):





										'''simple docstring'''







										if not self.test_rust_tokenizer:
														return

										_snake_case			:      Optional[int]	    = self.get_tokenizer()
										_snake_case			:      str	    = self.get_rust_tokenizer(add_prefix_space=snake_case_       )

										_snake_case			:      List[str]	    = """lower newer"""

										# Testing tokenization
										_snake_case			:      Dict	    = tokenizer.tokenize(snake_case_			,					add_prefix_space=snake_case_       )
										_snake_case			:      List[str]	    = rust_tokenizer.tokenize(snake_case_       )
										self.assertListEqual(snake_case_			,					snake_case_       )

										# Testing conversion to ids without special tokens
										_snake_case			:      Optional[int]	    = tokenizer.encode(snake_case_			,					add_special_tokens=snake_case_			,					add_prefix_space=snake_case_       )
										_snake_case			:      Tuple	    = rust_tokenizer.encode(snake_case_			,					add_special_tokens=snake_case_       )
										self.assertListEqual(snake_case_			,					snake_case_       )

										# Testing conversion to ids with special tokens
										_snake_case			:      Union[str, Any]	    = self.get_rust_tokenizer(add_prefix_space=snake_case_       )
										_snake_case			:      Optional[Any]	    = tokenizer.encode(snake_case_			,					add_prefix_space=snake_case_       )
										_snake_case			:      Optional[int]	    = rust_tokenizer.encode(snake_case_       )
										self.assertListEqual(snake_case_			,					snake_case_       )

										# Testing the unknown token
										_snake_case			:      List[Any]	    = tokens + [rust_tokenizer.unk_token]
										_snake_case			:      int	    = [14, 15, 10, 9, 3, 2, 15, 19]
										self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(snake_case_       )			,					snake_case_       )







						def 		__UpperCAmelCase							(     self      :		int			,					*lowerCamelCase_      :		List[Any]			,					**lowerCamelCase_      :		Optional[Any]       ):





										'''simple docstring'''







										pass







						def 		__UpperCAmelCase							(     self      :		str			,					lowerCamelCase_      :		Optional[Any]=15       ):





										'''simple docstring'''







										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
																		_snake_case			:      Union[str, Any]	    = self.rust_tokenizer_class.from_pretrained(snake_case_			,					**snake_case_       )

																		# Simple input
																		_snake_case			:      int	    = """This is a simple input"""
																		_snake_case			:      int	    = ["""This is a simple input 1""", """This is a simple input 2"""]
																		_snake_case			:      int	    = ("""This is a simple input""", """This is a pair""")
																		_snake_case			:      List[str]	    = [
																		    ("""This is a simple input 1""", """This is a simple input 2"""),
																		    ("""This is a simple pair 1""", """This is a simple pair 2"""),
																		]

																		# Simple input tests
																		self.assertRaises(snake_case_			,					tokenizer_r.encode			,					snake_case_			,					max_length=snake_case_			,					padding='max_length'       )

																		# Simple input
																		self.assertRaises(snake_case_			,					tokenizer_r.encode_plus			,					snake_case_			,					max_length=snake_case_			,					padding='max_length'       )

																		# Simple input
																		self.assertRaises(
																		    snake_case_			,					tokenizer_r.batch_encode_plus			,					snake_case_			,					max_length=snake_case_			,					padding='max_length'			,					)

																		# Pair input
																		self.assertRaises(snake_case_			,					tokenizer_r.encode			,					snake_case_			,					max_length=snake_case_			,					padding='max_length'       )

																		# Pair input
																		self.assertRaises(snake_case_			,					tokenizer_r.encode_plus			,					snake_case_			,					max_length=snake_case_			,					padding='max_length'       )

																		# Pair input
																		self.assertRaises(
																		    snake_case_			,					tokenizer_r.batch_encode_plus			,					snake_case_			,					max_length=snake_case_			,					padding='max_length'			,					)







						def 		__UpperCAmelCase							(     self      :		Any       ):





										'''simple docstring'''







										_snake_case			:      List[str]	    = CodeGenTokenizer.from_pretrained(self.tmpdirname			,					pad_token='<pad>'       )

										# Simple input
										_snake_case			:      Dict	    = """This is a simple input"""
										_snake_case			:      Optional[int]	    = ["""This is a simple input looooooooong""", """This is a simple input"""]
										_snake_case			:      Dict	    = ("""This is a simple input""", """This is a pair""")
										_snake_case			:      Tuple	    = [
										    ("""This is a simple input loooooong""", """This is a simple input"""),
										    ("""This is a simple pair loooooong""", """This is a simple pair"""),
										]

										_snake_case			:      List[str]	    = tokenizer.pad_token_id

										_snake_case			:      List[str]	    = tokenizer(snake_case_			,					padding='max_length'			,					max_length=30			,					return_tensors='np'       )
										_snake_case			:      str	    = tokenizer(snake_case_			,					padding=snake_case_			,					truncate=snake_case_			,					return_tensors='np'       )
										_snake_case			:      Optional[int]	    = tokenizer(*snake_case_			,					padding='max_length'			,					max_length=60			,					return_tensors='np'       )
										_snake_case			:      Optional[int]	    = tokenizer(snake_case_			,					padding=snake_case_			,					truncate=snake_case_			,					return_tensors='np'       )

										# s
										# test single string max_length padding
										self.assertEqual(out_s['input_ids'].shape[-1]			,					30       )
										self.assertTrue(pad_token_id in out_s['input_ids']       )
										self.assertTrue(0 in out_s['attention_mask']       )

										# s2
										# test automatic padding
										self.assertEqual(out_sa['input_ids'].shape[-1]			,					33       )
										# long slice doesn't have padding
										self.assertFalse(pad_token_id in out_sa['input_ids'][0]       )
										self.assertFalse(0 in out_sa['attention_mask'][0]       )
										# short slice does have padding
										self.assertTrue(pad_token_id in out_sa['input_ids'][1]       )
										self.assertTrue(0 in out_sa['attention_mask'][1]       )

										# p
										# test single pair max_length padding
										self.assertEqual(out_p['input_ids'].shape[-1]			,					60       )
										self.assertTrue(pad_token_id in out_p['input_ids']       )
										self.assertTrue(0 in out_p['attention_mask']       )

										# p2
										# test automatic padding pair
										self.assertEqual(out_pa['input_ids'].shape[-1]			,					52       )
										# long slice pair doesn't have padding
										self.assertFalse(pad_token_id in out_pa['input_ids'][0]       )
										self.assertFalse(0 in out_pa['attention_mask'][0]       )
										# short slice pair does have padding
										self.assertTrue(pad_token_id in out_pa['input_ids'][1]       )
										self.assertTrue(0 in out_pa['attention_mask'][1]       )







						def 		__UpperCAmelCase							(     self      :		Any       ):





										'''simple docstring'''







										_snake_case			:      Optional[int]	    = """$$$"""
										_snake_case			:      List[str]	    = CodeGenTokenizer.from_pretrained(self.tmpdirname			,					bos_token=snake_case_			,					add_bos_token=snake_case_       )

										_snake_case			:      Union[str, Any]	    = """This is a simple input"""
										_snake_case			:      Optional[int]	    = ["""This is a simple input 1""", """This is a simple input 2"""]

										_snake_case			:      List[Any]	    = tokenizer.bos_token_id

										_snake_case			:      int	    = tokenizer(snake_case_       )
										_snake_case			:      Union[str, Any]	    = tokenizer(snake_case_       )

										self.assertEqual(out_s.input_ids[0]			,					snake_case_       )
										self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids       )       )

										_snake_case			:      Dict	    = tokenizer.decode(out_s.input_ids       )
										_snake_case			:      Optional[Any]	    = tokenizer.batch_decode(out_sa.input_ids       )

										self.assertEqual(decode_s.split()[0]			,					snake_case_       )
										self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa       )       )







						@slow
						def 		__UpperCAmelCase							(     self      :		Union[str, Any]       ):





										'''simple docstring'''







										_snake_case			:      List[Any]	    = CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono'       )

										_snake_case			:      List[str]	    = """\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
										_snake_case			:      List[str]	    = """\nif len_a > len_b:      result = a\nelse:      result = b"""

										_snake_case			:      List[Any]	    = tokenizer.encode(snake_case_       )
										_snake_case			:      Tuple	    = ["""^#""", re.escape('<|endoftext|>'       ), """^'''""", """^\"\"\"""", """\n\n\n"""]
										_snake_case			:      Union[str, Any]	    = tokenizer.decode(snake_case_			,					truncate_before_pattern=snake_case_       )
										self.assertEqual(snake_case_			,					snake_case_       )







						def 		__UpperCAmelCase							(     self      :		Union[str, Any]       ):





										'''simple docstring'''







										pass







"""simple docstring"""



import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    IMAGE_PROCESSOR_MAPPING,
    AutoConfig,
    AutoImageProcessor,
    CLIPConfig,
    CLIPImageProcessor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_image_processing import CustomImageProcessor  # noqa E402
class 						__UpperCAmelCase    (unittest.TestCase     ):







			'''simple docstring'''





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     List[str]								=			0





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     List[str]								=			AutoImageProcessor.from_pretrained("""openai/clip-vit-base-patch32"""							)
										self.assertIsInstance(snake_case_   ,      snake_case_							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										with tempfile.TemporaryDirectory() as tmpdirname:
																	A__       :     Optional[Any]								=			Path(snake_case_							) / """preprocessor_config.json"""
																	A__       :     Any								=			Path(snake_case_							) / """config.json"""
																	json.dump(
																	    {"""image_processor_type""": """CLIPImageProcessor""", """processor_class""": """CLIPProcessor"""}   ,      open(snake_case_   ,      """w"""							)   ,      )
																	json.dump({"""model_type""": """clip"""}   ,      open(snake_case_   ,      """w"""							)							)

																	A__       :     Any								=			AutoImageProcessor.from_pretrained(snake_case_							)
																	self.assertIsInstance(snake_case_   ,      snake_case_							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										with tempfile.TemporaryDirectory() as tmpdirname:
																	A__       :     str								=			Path(snake_case_							) / """preprocessor_config.json"""
																	A__       :     Union[str, Any]								=			Path(snake_case_							) / """config.json"""
																	json.dump(
																	    {"""feature_extractor_type""": """CLIPFeatureExtractor""", """processor_class""": """CLIPProcessor"""}   ,      open(snake_case_   ,      """w"""							)   ,      )
																	json.dump({"""model_type""": """clip"""}   ,      open(snake_case_   ,      """w"""							)							)

																	A__       :     Optional[int]								=			AutoImageProcessor.from_pretrained(snake_case_							)
																	self.assertIsInstance(snake_case_   ,      snake_case_							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										with tempfile.TemporaryDirectory() as tmpdirname:
																	A__       :     Union[str, Any]								=			CLIPConfig()

																	# Create a dummy config file with image_proceesor_type
																	A__       :     Dict								=			Path(snake_case_							) / """preprocessor_config.json"""
																	A__       :     Optional[int]								=			Path(snake_case_							) / """config.json"""
																	json.dump(
																	    {"""image_processor_type""": """CLIPImageProcessor""", """processor_class""": """CLIPProcessor"""}   ,      open(snake_case_   ,      """w"""							)   ,      )
																	json.dump({"""model_type""": """clip"""}   ,      open(snake_case_   ,      """w"""							)							)

																	# remove image_processor_type to make sure config.json alone is enough to load image processor locally
																	A__       :     List[str]								=			AutoImageProcessor.from_pretrained(snake_case_							).to_dict()

																	config_dict.pop("""image_processor_type"""							)
																	A__       :     str								=			CLIPImageProcessor(**snake_case_							)

																	# save in new folder
																	model_config.save_pretrained(snake_case_							)
																	config.save_pretrained(snake_case_							)

																	A__       :     Optional[int]								=			AutoImageProcessor.from_pretrained(snake_case_							)

																	# make sure private variable is not incorrectly saved
																	A__       :     Dict								=			json.loads(config.to_json_string()							)
																	self.assertTrue("""_processor_class""" not in dict_as_saved							)

										self.assertIsInstance(snake_case_   ,      snake_case_							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										with tempfile.TemporaryDirectory() as tmpdirname:
																	A__       :     List[Any]								=			Path(snake_case_							) / """preprocessor_config.json"""
																	json.dump(
																	    {"""image_processor_type""": """CLIPImageProcessor""", """processor_class""": """CLIPProcessor"""}   ,      open(snake_case_   ,      """w"""							)   ,      )

																	A__       :     List[str]								=			AutoImageProcessor.from_pretrained(snake_case_							)
																	self.assertIsInstance(snake_case_   ,      snake_case_							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										with self.assertRaisesRegex(
										    snake_case_   ,      """clip-base is not a local folder and is not a valid model identifier"""							):
																	A__       :     Optional[Any]								=			AutoImageProcessor.from_pretrained("""clip-base"""							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										with self.assertRaisesRegex(
										    snake_case_   ,      r"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""							):
																	A__       :     Any								=			AutoImageProcessor.from_pretrained(snake_case_   ,      revision="""aaaaaa"""							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										with self.assertRaisesRegex(
										    snake_case_   ,      """hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json."""   ,      ):
																	A__       :     Optional[Any]								=			AutoImageProcessor.from_pretrained("""hf-internal-testing/config-no-model"""							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										with self.assertRaises(snake_case_							):
																	A__       :     str								=			AutoImageProcessor.from_pretrained("""hf-internal-testing/test_dynamic_image_processor"""							)
										# If remote code is disabled, we can't load this config.
										with self.assertRaises(snake_case_							):
																	A__       :     int								=			AutoImageProcessor.from_pretrained(
																	    """hf-internal-testing/test_dynamic_image_processor"""   ,      trust_remote_code=snake_case_							)

										A__       :     Optional[int]								=			AutoImageProcessor.from_pretrained(
										    """hf-internal-testing/test_dynamic_image_processor"""   ,      trust_remote_code=snake_case_							)
										self.assertEqual(image_processor.__class__.__name__   ,      """NewImageProcessor"""							)

										# Test image processor can be reloaded.
										with tempfile.TemporaryDirectory() as tmp_dir:
																	image_processor.save_pretrained(snake_case_							)
																	A__       :     Optional[Any]								=			AutoImageProcessor.from_pretrained(snake_case_   ,      trust_remote_code=snake_case_							)
										self.assertEqual(reloaded_image_processor.__class__.__name__   ,      """NewImageProcessor"""							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										try:
																	AutoConfig.register("""custom"""   ,      snake_case_							)
																	AutoImageProcessor.register(snake_case_   ,      snake_case_							)
																	# Trying to register something existing in the Transformers library will raise an error
																	with self.assertRaises(snake_case_							):
																								AutoImageProcessor.register(snake_case_   ,      snake_case_							)

																	with tempfile.TemporaryDirectory() as tmpdirname:
																								A__       :     Tuple								=			Path(snake_case_							) / """preprocessor_config.json"""
																								A__       :     List[str]								=			Path(snake_case_							) / """config.json"""
																								json.dump(
																								    {"""feature_extractor_type""": """CLIPFeatureExtractor""", """processor_class""": """CLIPProcessor"""}   ,      open(snake_case_   ,      """w"""							)   ,      )
																								json.dump({"""model_type""": """clip"""}   ,      open(snake_case_   ,      """w"""							)							)

																								A__       :     Optional[int]								=			CustomImageProcessor.from_pretrained(snake_case_							)

																	# Now that the config is registered, it can be used as any other config with the auto-API
																	with tempfile.TemporaryDirectory() as tmp_dir:
																								image_processor.save_pretrained(snake_case_							)
																								A__       :     int								=			AutoImageProcessor.from_pretrained(snake_case_							)
																								self.assertIsInstance(snake_case_   ,      snake_case_							)

										finally:
																	if "custom" in CONFIG_MAPPING._extra_content:
																								del CONFIG_MAPPING._extra_content["custom"]
																	if CustomConfig in IMAGE_PROCESSOR_MAPPING._extra_content:
																								del IMAGE_PROCESSOR_MAPPING._extra_content[CustomConfig]





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										class 						__UpperCAmelCase    (__A     ):







													'''simple docstring'''
													_UpperCamelCase  :						Union[str, Any]					     =				True

										try:
																	AutoConfig.register("""custom"""   ,      snake_case_							)
																	AutoImageProcessor.register(snake_case_   ,      snake_case_							)
																	# If remote code is not set, the default is to use local
																	A__       :     int								=			AutoImageProcessor.from_pretrained("""hf-internal-testing/test_dynamic_image_processor"""							)
																	self.assertEqual(image_processor.__class__.__name__   ,      """NewImageProcessor"""							)
																	self.assertTrue(image_processor.is_local							)

																	# If remote code is disabled, we load the local one.
																	A__       :     Any								=			AutoImageProcessor.from_pretrained(
																	    """hf-internal-testing/test_dynamic_image_processor"""   ,      trust_remote_code=snake_case_							)
																	self.assertEqual(image_processor.__class__.__name__   ,      """NewImageProcessor"""							)
																	self.assertTrue(image_processor.is_local							)

																	# If remote is enabled, we load from the Hub
																	A__       :     Optional[Any]								=			AutoImageProcessor.from_pretrained(
																	    """hf-internal-testing/test_dynamic_image_processor"""   ,      trust_remote_code=snake_case_							)
																	self.assertEqual(image_processor.__class__.__name__   ,      """NewImageProcessor"""							)
																	self.assertTrue(not hasattr(snake_case_   ,      """is_local"""							)							)

										finally:
																	if "custom" in CONFIG_MAPPING._extra_content:
																								del CONFIG_MAPPING._extra_content["custom"]
																	if CustomConfig in IMAGE_PROCESSOR_MAPPING._extra_content:
																								del IMAGE_PROCESSOR_MAPPING._extra_content[CustomConfig]





