


import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class        lowerCAmelCase__ (					__lowerCamelCase,							unittest.TestCase  ):






						"""simple docstring"""
						__UpperCAmelCase       :						int    	=			CodeGenTokenizer
						__UpperCAmelCase       :						Optional[Any]    	=			CodeGenTokenizerFast
						__UpperCAmelCase       :						List[str]    	=			True
						__UpperCAmelCase       :						Optional[int]    	=			{'''add_prefix_space''': True}
						__UpperCAmelCase       :						Dict    	=			False




						def 	_UpperCamelCase					(       self      ):
							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							lowerCamelCase_					:				Dict              =  [
							    "l",
							    "o",
							    "w",
							    "e",
							    "r",
							    "s",
							    "t",
							    "i",
							    "d",
							    "n",
							    "\u0120",
							    "\u0120l",
							    "\u0120n",
							    "\u0120lo",
							    "\u0120low",
							    "er",
							    "\u0120lowest",
							    "\u0120newer",
							    "\u0120wider",
							    "<unk>",
							    "<|endoftext|>",
							]
							lowerCamelCase_					:				Any              =  dict(zip(a_ ,				range(len(a_      )      )      )      )
							lowerCamelCase_					:				List[str]              =  ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
							lowerCamelCase_					:				Union[str, Any]              =  {"unk_token": "<unk>"}

							lowerCamelCase_					:				List[Any]              =  os.path.join(self.tmpdirname ,				VOCAB_FILES_NAMES["vocab_file"]      )
							lowerCamelCase_					:				Any              =  os.path.join(self.tmpdirname ,				VOCAB_FILES_NAMES["merges_file"]      )
							with open(self.vocab_file ,				"w" ,				encoding="utf-8"      ) as fp:
								fp.write(json.dumps(a_      ) + "\n"      )
							with open(self.merges_file ,				"w" ,				encoding="utf-8"      ) as fp:
								fp.write("\n".join(a_      )      )




						def 	_UpperCamelCase					(       self ,				**a_      ):
							kwargs.update(self.special_tokens_map      )
							return CodeGenTokenizer.from_pretrained(self.tmpdirname ,				**a_      )




						def 	_UpperCamelCase					(       self ,				**a_      ):
							kwargs.update(self.special_tokens_map      )
							return CodeGenTokenizerFast.from_pretrained(self.tmpdirname ,				**a_      )




						def 	_UpperCamelCase					(       self ,				a_      ):
							lowerCamelCase_					:				Union[str, Any]              =  "lower newer"
							lowerCamelCase_					:				Dict              =  "lower newer"
							return input_text, output_text




						def 	_UpperCamelCase					(       self      ):
							lowerCamelCase_					:				Union[str, Any]              =  CodeGenTokenizer(self.vocab_file ,				self.merges_file ,				**self.special_tokens_map      )
							lowerCamelCase_					:				Union[str, Any]              =  "lower newer"
							lowerCamelCase_					:				List[Any]              =  ["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
							lowerCamelCase_					:				List[str]              =  tokenizer.tokenize(a_ ,				add_prefix_space=a_      )
							self.assertListEqual(a_ ,				a_      )

							lowerCamelCase_					:				Optional[int]              =  tokens + [tokenizer.unk_token]
							lowerCamelCase_					:				Dict              =  [14, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(a_      ) ,				a_      )




						def 	_UpperCamelCase					(       self      ):
							if not self.test_rust_tokenizer:
								return

							lowerCamelCase_					:				Optional[Any]              =  self.get_tokenizer()
							lowerCamelCase_					:				int              =  self.get_rust_tokenizer(add_prefix_space=a_      )

							lowerCamelCase_					:				Any              =  "lower newer"

							# Testing tokenization
							lowerCamelCase_					:				List[Any]              =  tokenizer.tokenize(a_ ,				add_prefix_space=a_      )
							lowerCamelCase_					:				Optional[Any]              =  rust_tokenizer.tokenize(a_      )
							self.assertListEqual(a_ ,				a_      )

							# Testing conversion to ids without special tokens
							lowerCamelCase_					:				List[str]              =  tokenizer.encode(a_ ,				add_special_tokens=a_ ,				add_prefix_space=a_      )
							lowerCamelCase_					:				str              =  rust_tokenizer.encode(a_ ,				add_special_tokens=a_      )
							self.assertListEqual(a_ ,				a_      )

							# Testing conversion to ids with special tokens
							lowerCamelCase_					:				Optional[Any]              =  self.get_rust_tokenizer(add_prefix_space=a_      )
							lowerCamelCase_					:				Optional[Any]              =  tokenizer.encode(a_ ,				add_prefix_space=a_      )
							lowerCamelCase_					:				int              =  rust_tokenizer.encode(a_      )
							self.assertListEqual(a_ ,				a_      )

							# Testing the unknown token
							lowerCamelCase_					:				Dict              =  tokens + [rust_tokenizer.unk_token]
							lowerCamelCase_					:				Any              =  [14, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(a_      ) ,				a_      )




						def 	_UpperCamelCase					(       self ,				*a_ ,				**a_      ):
							# It's very difficult to mix/test pretokenization with byte-level
							# And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
							pass




						def 	_UpperCamelCase					(       self ,				a_=15      ):
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
									lowerCamelCase_					:				Dict              =  self.rust_tokenizer_class.from_pretrained(a_ ,				**a_      )

									# Simple input
									lowerCamelCase_					:				Optional[int]              =  "This is a simple input"
									lowerCamelCase_					:				List[Any]              =  ["This is a simple input 1", "This is a simple input 2"]
									lowerCamelCase_					:				str              =  ("This is a simple input", "This is a pair")
									lowerCamelCase_					:				List[str]              =  [
									    ("This is a simple input 1", "This is a simple input 2"),
									    ("This is a simple pair 1", "This is a simple pair 2"),
									]

									# Simple input tests
									self.assertRaises(a_ ,				tokenizer_r.encode ,				a_ ,				max_length=a_ ,				padding="max_length"      )

									# Simple input
									self.assertRaises(a_ ,				tokenizer_r.encode_plus ,				a_ ,				max_length=a_ ,				padding="max_length"      )

									# Simple input
									self.assertRaises(
									    a_ ,				tokenizer_r.batch_encode_plus ,				a_ ,				max_length=a_ ,				padding="max_length" ,				)

									# Pair input
									self.assertRaises(a_ ,				tokenizer_r.encode ,				a_ ,				max_length=a_ ,				padding="max_length"      )

									# Pair input
									self.assertRaises(a_ ,				tokenizer_r.encode_plus ,				a_ ,				max_length=a_ ,				padding="max_length"      )

									# Pair input
									self.assertRaises(
									    a_ ,				tokenizer_r.batch_encode_plus ,				a_ ,				max_length=a_ ,				padding="max_length" ,				)




						def 	_UpperCamelCase					(       self      ):
							lowerCamelCase_					:				Dict              =  CodeGenTokenizer.from_pretrained(self.tmpdirname ,				pad_token="<pad>"      )

							# Simple input
							lowerCamelCase_					:				int              =  "This is a simple input"
							lowerCamelCase_					:				List[str]              =  ["This is a simple input looooooooong", "This is a simple input"]
							lowerCamelCase_					:				Optional[int]              =  ("This is a simple input", "This is a pair")
							lowerCamelCase_					:				Any              =  [
							    ("This is a simple input loooooong", "This is a simple input"),
							    ("This is a simple pair loooooong", "This is a simple pair"),
							]

							lowerCamelCase_					:				Any              =  tokenizer.pad_token_id

							lowerCamelCase_					:				str              =  tokenizer(a_ ,				padding="max_length" ,				max_length=30 ,				return_tensors="np"      )
							lowerCamelCase_					:				Any              =  tokenizer(a_ ,				padding=a_ ,				truncate=a_ ,				return_tensors="np"      )
							lowerCamelCase_					:				Dict              =  tokenizer(*a_ ,				padding="max_length" ,				max_length=60 ,				return_tensors="np"      )
							lowerCamelCase_					:				List[str]              =  tokenizer(a_ ,				padding=a_ ,				truncate=a_ ,				return_tensors="np"      )

							# s
							# test single string max_length padding
							self.assertEqual(out_s["input_ids"].shape[-1] ,				30      )
							self.assertTrue(pad_token_id in out_s["input_ids"]      )
							self.assertTrue(0 in out_s["attention_mask"]      )

							# s2
							# test automatic padding
							self.assertEqual(out_sa["input_ids"].shape[-1] ,				33      )
							# long slice doesn't have padding
							self.assertFalse(pad_token_id in out_sa["input_ids"][0]      )
							self.assertFalse(0 in out_sa["attention_mask"][0]      )
							# short slice does have padding
							self.assertTrue(pad_token_id in out_sa["input_ids"][1]      )
							self.assertTrue(0 in out_sa["attention_mask"][1]      )

							# p
							# test single pair max_length padding
							self.assertEqual(out_p["input_ids"].shape[-1] ,				60      )
							self.assertTrue(pad_token_id in out_p["input_ids"]      )
							self.assertTrue(0 in out_p["attention_mask"]      )

							# p2
							# test automatic padding pair
							self.assertEqual(out_pa["input_ids"].shape[-1] ,				52      )
							# long slice pair doesn't have padding
							self.assertFalse(pad_token_id in out_pa["input_ids"][0]      )
							self.assertFalse(0 in out_pa["attention_mask"][0]      )
							# short slice pair does have padding
							self.assertTrue(pad_token_id in out_pa["input_ids"][1]      )
							self.assertTrue(0 in out_pa["attention_mask"][1]      )




						def 	_UpperCamelCase					(       self      ):
							lowerCamelCase_					:				Dict              =  "$$$"
							lowerCamelCase_					:				Any              =  CodeGenTokenizer.from_pretrained(self.tmpdirname ,				bos_token=a_ ,				add_bos_token=a_      )

							lowerCamelCase_					:				Tuple              =  "This is a simple input"
							lowerCamelCase_					:				Union[str, Any]              =  ["This is a simple input 1", "This is a simple input 2"]

							lowerCamelCase_					:				Union[str, Any]              =  tokenizer.bos_token_id

							lowerCamelCase_					:				int              =  tokenizer(a_      )
							lowerCamelCase_					:				List[str]              =  tokenizer(a_      )

							self.assertEqual(out_s.input_ids[0] ,				a_      )
							self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids      )      )

							lowerCamelCase_					:				List[str]              =  tokenizer.decode(out_s.input_ids      )
							lowerCamelCase_					:				Dict              =  tokenizer.batch_decode(out_sa.input_ids      )

							self.assertEqual(decode_s.split()[0] ,				a_      )
							self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa      )      )




						@slow
						def 	_UpperCamelCase					(       self      ):
							lowerCamelCase_					:				List[str]              =  CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono"      )

							lowerCamelCase_					:				List[str]              =  "\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
							lowerCamelCase_					:				Tuple              =  "\nif len_a > len_b:      result = a\nelse:      result = b"

							lowerCamelCase_					:				int              =  tokenizer.encode(a_      )
							lowerCamelCase_					:				List[Any]              =  ["^#", re.escape("<|endoftext|>"      ), "^\'\'\'", "^\"\"\"", "\n\n\n"]
							lowerCamelCase_					:				Tuple              =  tokenizer.decode(a_ ,				truncate_before_pattern=a_      )
							self.assertEqual(a_ ,				a_      )




						def 	_UpperCamelCase					(       self      ):
							pass







'''simple docstring'''


import warnings
from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCAmelCase_					=   logging.get_logger(__name__)

UpperCAmelCase_					=   {
    "nvidia/segformer-b0-finetuned-ade-512-512": (
        "https://huggingface.co/nvidia/segformer-b0-finetuned-ade-512-512/resolve/main/config.json"
    ),
    # See all SegFormer models at https://huggingface.co/models?filter=segformer
}





class  __lowercase    (	__magic_name__			):
   _a    						=   """segformer"""







   def __init__(						self   ,				UpperCamelCase=3   ,				UpperCamelCase=4   ,				UpperCamelCase=[2, 2, 2, 2]   ,				UpperCamelCase=[8, 4, 2, 1]   ,				UpperCamelCase=[32, 64, 160, 256]   ,				UpperCamelCase=[7, 3, 3, 3]   ,				UpperCamelCase=[4, 2, 2, 2]   ,				UpperCamelCase=[1, 2, 5, 8]   ,				UpperCamelCase=[4, 4, 4, 4]   ,				UpperCamelCase="gelu"   ,				UpperCamelCase=0.0   ,				UpperCamelCase=0.0   ,				UpperCamelCase=0.1   ,				UpperCamelCase=0.02   ,				UpperCamelCase=0.1   ,				UpperCamelCase=1e-6   ,				UpperCamelCase=256   ,				UpperCamelCase=255   ,				**UpperCamelCase   ,				)					->     int:
     super().__init__(**UpperCamelCase   )

     if "reshape_last_stage" in kwargs and kwargs["reshape_last_stage"] is False:
       warnings.warn(
           'Reshape_last_stage is set to False in this config. This argument is deprecated and will soon be'
           ' removed, as the behaviour will default to that of reshape_last_stage = True.'   ,				UpperCamelCase   ,				)

     __a								=       num_channels
     __a								=       num_encoder_blocks
     __a								=       depths
     __a								=       sr_ratios
     __a								=       hidden_sizes
     __a								=       patch_sizes
     __a								=       strides
     __a								=       mlp_ratios
     __a								=       num_attention_heads
     __a								=       hidden_act
     __a								=       hidden_dropout_prob
     __a								=       attention_probs_dropout_prob
     __a								=       classifier_dropout_prob
     __a								=       initializer_range
     __a								=       drop_path_rate
     __a								=       layer_norm_eps
     __a								=       decoder_hidden_size
     __a								=       kwargs.get('reshape_last_stage'   ,				UpperCamelCase   )
     __a								=       semantic_loss_ignore_index





class  __lowercase    (	__magic_name__			):
   _a    						=   version.parse("""1.11"""			)







   @property
   def 		UpperCamelCase__      (						self   )					->     Mapping[str, Mapping[int, str]]:
     return OrderedDict(
         [
             ('pixel_values', {0: 'batch', 1: 'num_channels', 2: 'height', 3: 'width'}),
         ]   )







   @property
   def 		UpperCamelCase__      (						self   )					->     float:
     return 1e-4







   @property
   def 		UpperCamelCase__      (						self   )					->     int:
     return 12



