



'''simple docstring'''





import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 				snake_case (UpperCamelCase		,      unittest.TestCase   ):
							lowerCAmelCase__  :Optional[int]						   =					CodeGenTokenizer
							lowerCAmelCase__  :List[Any]						   =					CodeGenTokenizerFast
							lowerCAmelCase__  :str						   =					True
							lowerCAmelCase__  :Tuple						   =					{"add_prefix_space": True}
							lowerCAmelCase__  :Dict						   =					False



							def 							_a      (				self      )  ->							Union[str, Any]:
											super().setUp()

											# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
											lowercase__		       =						[
											    "l",
											    "o",
											    "w",
											    "e",
											    "r",
											    "s",
											    "t",
											    "i",
											    "d",
											    "n",
											    "\u0120",
											    "\u0120l",
											    "\u0120n",
											    "\u0120lo",
											    "\u0120low",
											    "er",
											    "\u0120lowest",
											    "\u0120newer",
											    "\u0120wider",
											    "<unk>",
											    "<|endoftext|>",
											]
											lowercase__		       =						dict(zip(UpperCAmelCase_     ,range(len(UpperCAmelCase_      )      )      )      )
											lowercase__		       =						["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
											lowercase__		       =						{"unk_token": "<unk>"}

											lowercase__		       =						os.path.join(self.tmpdirname     ,VOCAB_FILES_NAMES["vocab_file"]      )
											lowercase__		       =						os.path.join(self.tmpdirname     ,VOCAB_FILES_NAMES["merges_file"]      )
											with open(self.vocab_file     ,"w"     ,encoding="utf-8"      ) as fp:
															fp.write(json.dumps(UpperCAmelCase_      ) + "\n"      )
											with open(self.merges_file     ,"w"     ,encoding="utf-8"      ) as fp:
															fp.write("\n".join(UpperCAmelCase_      )      )



							def 							_a      (				self     ,**UpperCAmelCase_      )  ->							int:
											kwargs.update(self.special_tokens_map      )
											return CodeGenTokenizer.from_pretrained(self.tmpdirname     ,**UpperCAmelCase_      )



							def 							_a      (				self     ,**UpperCAmelCase_      )  ->							Tuple:
											kwargs.update(self.special_tokens_map      )
											return CodeGenTokenizerFast.from_pretrained(self.tmpdirname     ,**UpperCAmelCase_      )



							def 							_a      (				self     ,UpperCAmelCase_      )  ->							List[Any]:
											lowercase__		       =						"lower newer"
											lowercase__		       =						"lower newer"
											return input_text, output_text



							def 							_a      (				self      )  ->							Optional[int]:
											lowercase__		       =						CodeGenTokenizer(self.vocab_file     ,self.merges_file     ,**self.special_tokens_map      )
											lowercase__		       =						"lower newer"
											lowercase__		       =						["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
											lowercase__		       =						tokenizer.tokenize(UpperCAmelCase_     ,add_prefix_space=UpperCAmelCase_      )
											self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

											lowercase__		       =						tokens + [tokenizer.unk_token]
											lowercase__		       =						[14, 15, 10, 9, 3, 2, 15, 19]
											self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )     ,UpperCAmelCase_      )



							def 							_a      (				self      )  ->							int:
											if not self.test_rust_tokenizer:
															return

											lowercase__		       =						self.get_tokenizer()
											lowercase__		       =						self.get_rust_tokenizer(add_prefix_space=UpperCAmelCase_      )

											lowercase__		       =						"lower newer"

											# Testing tokenization
											lowercase__		       =						tokenizer.tokenize(UpperCAmelCase_     ,add_prefix_space=UpperCAmelCase_      )
											lowercase__		       =						rust_tokenizer.tokenize(UpperCAmelCase_      )
											self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

											# Testing conversion to ids without special tokens
											lowercase__		       =						tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_     ,add_prefix_space=UpperCAmelCase_      )
											lowercase__		       =						rust_tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )
											self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

											# Testing conversion to ids with special tokens
											lowercase__		       =						self.get_rust_tokenizer(add_prefix_space=UpperCAmelCase_      )
											lowercase__		       =						tokenizer.encode(UpperCAmelCase_     ,add_prefix_space=UpperCAmelCase_      )
											lowercase__		       =						rust_tokenizer.encode(UpperCAmelCase_      )
											self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

											# Testing the unknown token
											lowercase__		       =						tokens + [rust_tokenizer.unk_token]
											lowercase__		       =						[14, 15, 10, 9, 3, 2, 15, 19]
											self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )     ,UpperCAmelCase_      )



							def 							_a      (				self     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )  ->							Any:
											# It's very difficult to mix/test pretokenization with byte-level
											# And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
											pass



							def 							_a      (				self     ,UpperCAmelCase_=15      )  ->							Union[str, Any]:
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''      ):
																			lowercase__		       =						self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_     ,**UpperCAmelCase_      )

																			# Simple input
																			lowercase__		       =						"This is a simple input"
																			lowercase__		       =						["This is a simple input 1", "This is a simple input 2"]
																			lowercase__		       =						("This is a simple input", "This is a pair")
																			lowercase__		       =						[
																			    ("This is a simple input 1", "This is a simple input 2"),
																			    ("This is a simple pair 1", "This is a simple pair 2"),
																			]

																			# Simple input tests
																			self.assertRaises(UpperCAmelCase_     ,tokenizer_r.encode     ,UpperCAmelCase_     ,max_length=UpperCAmelCase_     ,padding="max_length"      )

																			# Simple input
																			self.assertRaises(UpperCAmelCase_     ,tokenizer_r.encode_plus     ,UpperCAmelCase_     ,max_length=UpperCAmelCase_     ,padding="max_length"      )

																			# Simple input
																			self.assertRaises(
																			    UpperCAmelCase_     ,tokenizer_r.batch_encode_plus     ,UpperCAmelCase_     ,max_length=UpperCAmelCase_     ,padding="max_length"     ,)

																			# Pair input
																			self.assertRaises(UpperCAmelCase_     ,tokenizer_r.encode     ,UpperCAmelCase_     ,max_length=UpperCAmelCase_     ,padding="max_length"      )

																			# Pair input
																			self.assertRaises(UpperCAmelCase_     ,tokenizer_r.encode_plus     ,UpperCAmelCase_     ,max_length=UpperCAmelCase_     ,padding="max_length"      )

																			# Pair input
																			self.assertRaises(
																			    UpperCAmelCase_     ,tokenizer_r.batch_encode_plus     ,UpperCAmelCase_     ,max_length=UpperCAmelCase_     ,padding="max_length"     ,)



							def 							_a      (				self      )  ->							Optional[Any]:
											lowercase__		       =						CodeGenTokenizer.from_pretrained(self.tmpdirname     ,pad_token="<pad>"      )

											# Simple input
											lowercase__		       =						"This is a simple input"
											lowercase__		       =						["This is a simple input looooooooong", "This is a simple input"]
											lowercase__		       =						("This is a simple input", "This is a pair")
											lowercase__		       =						[
											    ("This is a simple input loooooong", "This is a simple input"),
											    ("This is a simple pair loooooong", "This is a simple pair"),
											]

											lowercase__		       =						tokenizer.pad_token_id

											lowercase__		       =						tokenizer(UpperCAmelCase_     ,padding="max_length"     ,max_length=30     ,return_tensors="np"      )
											lowercase__		       =						tokenizer(UpperCAmelCase_     ,padding=UpperCAmelCase_     ,truncate=UpperCAmelCase_     ,return_tensors="np"      )
											lowercase__		       =						tokenizer(*UpperCAmelCase_     ,padding="max_length"     ,max_length=60     ,return_tensors="np"      )
											lowercase__		       =						tokenizer(UpperCAmelCase_     ,padding=UpperCAmelCase_     ,truncate=UpperCAmelCase_     ,return_tensors="np"      )

											# s
											# test single string max_length padding
											self.assertEqual(out_s["input_ids"].shape[-1]     ,30      )
											self.assertTrue(pad_token_id in out_s["input_ids"]      )
											self.assertTrue(0 in out_s["attention_mask"]      )

											# s2
											# test automatic padding
											self.assertEqual(out_sa["input_ids"].shape[-1]     ,33      )
											# long slice doesn't have padding
											self.assertFalse(pad_token_id in out_sa["input_ids"][0]      )
											self.assertFalse(0 in out_sa["attention_mask"][0]      )
											# short slice does have padding
											self.assertTrue(pad_token_id in out_sa["input_ids"][1]      )
											self.assertTrue(0 in out_sa["attention_mask"][1]      )

											# p
											# test single pair max_length padding
											self.assertEqual(out_p["input_ids"].shape[-1]     ,60      )
											self.assertTrue(pad_token_id in out_p["input_ids"]      )
											self.assertTrue(0 in out_p["attention_mask"]      )

											# p2
											# test automatic padding pair
											self.assertEqual(out_pa["input_ids"].shape[-1]     ,52      )
											# long slice pair doesn't have padding
											self.assertFalse(pad_token_id in out_pa["input_ids"][0]      )
											self.assertFalse(0 in out_pa["attention_mask"][0]      )
											# short slice pair does have padding
											self.assertTrue(pad_token_id in out_pa["input_ids"][1]      )
											self.assertTrue(0 in out_pa["attention_mask"][1]      )



							def 							_a      (				self      )  ->							List[str]:
											lowercase__		       =						"$$$"
											lowercase__		       =						CodeGenTokenizer.from_pretrained(self.tmpdirname     ,bos_token=UpperCAmelCase_     ,add_bos_token=UpperCAmelCase_      )

											lowercase__		       =						"This is a simple input"
											lowercase__		       =						["This is a simple input 1", "This is a simple input 2"]

											lowercase__		       =						tokenizer.bos_token_id

											lowercase__		       =						tokenizer(UpperCAmelCase_      )
											lowercase__		       =						tokenizer(UpperCAmelCase_      )

											self.assertEqual(out_s.input_ids[0]     ,UpperCAmelCase_      )
											self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids      )      )

											lowercase__		       =						tokenizer.decode(out_s.input_ids      )
											lowercase__		       =						tokenizer.batch_decode(out_sa.input_ids      )

											self.assertEqual(decode_s.split()[0]     ,UpperCAmelCase_      )
											self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa      )      )



							@slow
							def 							_a      (				self      )  ->							List[Any]:
											lowercase__		       =						CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono"      )

											lowercase__		       =						"\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
											lowercase__		       =						"\nif len_a > len_b:      result = a\nelse:      result = b"

											lowercase__		       =						tokenizer.encode(UpperCAmelCase_      )
											lowercase__		       =						["^#", re.escape("<|endoftext|>"      ), "^'''", "^\"\"\"", "\n\n\n"]
											lowercase__		       =						tokenizer.decode(UpperCAmelCase_     ,truncate_before_pattern=UpperCAmelCase_      )
											self.assertEqual(UpperCAmelCase_     ,UpperCAmelCase_      )







							def 							_a      (				self      )  ->							Any:
											pass









'''simple docstring'''





import copy
import os
from typing import TYPE_CHECKING, List, Union


if TYPE_CHECKING:
 pass

from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE__      					=     logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__      					=     {
    "kakaobrain/align-base": "https://huggingface.co/kakaobrain/align-base/resolve/main/config.json",
}


class 				snake_case (UpperCamelCase   ):
       lowerCAmelCase__  :Optional[int]						   =					"align_text_model"



       def __init__(				self     ,UpperCAmelCase_=30_522     ,UpperCAmelCase_=768     ,UpperCAmelCase_=12     ,UpperCAmelCase_=12     ,UpperCAmelCase_=3_072     ,UpperCAmelCase_="gelu"     ,UpperCAmelCase_=0.1     ,UpperCAmelCase_=0.1     ,UpperCAmelCase_=512     ,UpperCAmelCase_=2     ,UpperCAmelCase_=0.02     ,UpperCAmelCase_=1E-1_2     ,UpperCAmelCase_=0     ,UpperCAmelCase_="absolute"     ,UpperCAmelCase_=True     ,**UpperCAmelCase_     ,)  ->							List[str]:
           super().__init__(**UpperCAmelCase_      )

           lowercase__		       =						vocab_size
           lowercase__		       =						hidden_size
           lowercase__		       =						num_hidden_layers
           lowercase__		       =						num_attention_heads
           lowercase__		       =						hidden_act
           lowercase__		       =						intermediate_size
           lowercase__		       =						hidden_dropout_prob
           lowercase__		       =						attention_probs_dropout_prob
           lowercase__		       =						max_position_embeddings
           lowercase__		       =						type_vocab_size
           lowercase__		       =						initializer_range
           lowercase__		       =						layer_norm_eps
           lowercase__		       =						position_embedding_type
           lowercase__		       =						use_cache
           lowercase__		       =						pad_token_id







       @classmethod
       def 							_a      (				cls     ,UpperCAmelCase_     ,**UpperCAmelCase_      )  ->							"PretrainedConfig":
           cls._set_token_in_kwargs(UpperCAmelCase_      )

           lowercase__       ,       lowercase__		       =						cls.get_config_dict(UpperCAmelCase_     ,**UpperCAmelCase_      )

           # get the text config dict if we are loading from AlignConfig
           if config_dict.get("model_type"      ) == "align":
               lowercase__		       =						config_dict["text_config"]

           if "model_type" in config_dict and hasattr(cls     ,"model_type"      ) and config_dict["model_type"] != cls.model_type:
               logger.warning(
                   F'''You are using a model of type {config_dict['model_type']} to instantiate a model of type '''
                   F'''{cls.model_type}. This is not supported for all configurations of models and can yield errors.'''      )

           return cls.from_dict(UpperCAmelCase_     ,**UpperCAmelCase_      )





class 				snake_case (UpperCamelCase   ):
       lowerCAmelCase__  :Tuple						   =					"align_vision_model"



       def __init__(				self     ,UpperCAmelCase_ = 3     ,UpperCAmelCase_ = 600     ,UpperCAmelCase_ = 2.0     ,UpperCAmelCase_ = 3.1     ,UpperCAmelCase_ = 8     ,UpperCAmelCase_ = [3, 3, 5, 3, 5, 5, 3]     ,UpperCAmelCase_ = [32, 16, 24, 40, 80, 112, 192]     ,UpperCAmelCase_ = [16, 24, 40, 80, 112, 192, 320]     ,UpperCAmelCase_ = []     ,UpperCAmelCase_ = [1, 2, 2, 2, 1, 2, 1]     ,UpperCAmelCase_ = [1, 2, 2, 3, 3, 4, 1]     ,UpperCAmelCase_ = [1, 6, 6, 6, 6, 6, 6]     ,UpperCAmelCase_ = 0.25     ,UpperCAmelCase_ = "swish"     ,UpperCAmelCase_ = 2_560     ,UpperCAmelCase_ = "mean"     ,UpperCAmelCase_ = 0.02     ,UpperCAmelCase_ = 0.0_01     ,UpperCAmelCase_ = 0.99     ,UpperCAmelCase_ = 0.2     ,**UpperCAmelCase_     ,)  ->							Union[str, Any]:
           super().__init__(**UpperCAmelCase_      )

           lowercase__		       =						num_channels
           lowercase__		       =						image_size
           lowercase__		       =						width_coefficient
           lowercase__		       =						depth_coefficient
           lowercase__		       =						depth_divisor
           lowercase__		       =						kernel_sizes
           lowercase__		       =						in_channels
           lowercase__		       =						out_channels
           lowercase__		       =						depthwise_padding
           lowercase__		       =						strides
           lowercase__		       =						num_block_repeats
           lowercase__		       =						expand_ratios
           lowercase__		       =						squeeze_expansion_ratio
           lowercase__		       =						hidden_act
           lowercase__		       =						hidden_dim
           lowercase__		       =						pooling_type
           lowercase__		       =						initializer_range
           lowercase__		       =						batch_norm_eps
           lowercase__		       =						batch_norm_momentum
           lowercase__		       =						drop_connect_rate
           lowercase__		       =						sum(UpperCAmelCase_      ) * 4







       @classmethod
       def 							_a      (				cls     ,UpperCAmelCase_     ,**UpperCAmelCase_      )  ->							"PretrainedConfig":
           cls._set_token_in_kwargs(UpperCAmelCase_      )

           lowercase__       ,       lowercase__		       =						cls.get_config_dict(UpperCAmelCase_     ,**UpperCAmelCase_      )

           # get the vision config dict if we are loading from AlignConfig
           if config_dict.get("model_type"      ) == "align":
               lowercase__		       =						config_dict["vision_config"]

           if "model_type" in config_dict and hasattr(cls     ,"model_type"      ) and config_dict["model_type"] != cls.model_type:
               logger.warning(
                   F'''You are using a model of type {config_dict['model_type']} to instantiate a model of type '''
                   F'''{cls.model_type}. This is not supported for all configurations of models and can yield errors.'''      )

           return cls.from_dict(UpperCAmelCase_     ,**UpperCAmelCase_      )





class 				snake_case (UpperCamelCase   ):
       lowerCAmelCase__  :Union[str, Any]						   =					"align"
       lowerCAmelCase__  :Tuple						   =					True



       def __init__(				self     ,UpperCAmelCase_=None     ,UpperCAmelCase_=None     ,UpperCAmelCase_=640     ,UpperCAmelCase_=1.0     ,UpperCAmelCase_=0.02     ,**UpperCAmelCase_     ,)  ->							Union[str, Any]:
           super().__init__(**UpperCAmelCase_      )

           if text_config is None:
               lowercase__		       =						{}
               logger.info("text_config is None. Initializing the AlignTextConfig with default values."      )

           if vision_config is None:
               lowercase__		       =						{}
               logger.info("vision_config is None. Initializing the AlignVisionConfig with default values."      )

           lowercase__		       =						AlignTextConfig(**UpperCAmelCase_      )
           lowercase__		       =						AlignVisionConfig(**UpperCAmelCase_      )

           lowercase__		       =						projection_dim
           lowercase__		       =						temperature_init_value
           lowercase__		       =						initializer_range



       @classmethod
       def 							_a      (				cls     ,UpperCAmelCase_     ,UpperCAmelCase_     ,**UpperCAmelCase_      )  ->							List[str]:
           return cls(text_config=text_config.to_dict()     ,vision_config=vision_config.to_dict()     ,**UpperCAmelCase_      )







       def 							_a      (				self      )  ->							List[Any]:
           lowercase__		       =						copy.deepcopy(self.__dict__      )
           lowercase__		       =						self.text_config.to_dict()
           lowercase__		       =						self.vision_config.to_dict()
           lowercase__		       =						self.__class__.model_type
           return output




