


import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class A    (  UpperCamelCase_       ,				unittest.TestCase	):
      UpperCamelCase__				:    Optional[int] =CodeGenTokenizer
      UpperCamelCase__				:    List[str] =CodeGenTokenizerFast
      UpperCamelCase__				:    Optional[Any] =True
      UpperCamelCase__				:    Dict ={'add_prefix_space': True}
      UpperCamelCase__				:    List[str] =False






      def 			lowerCamelCase			(   self		:	Union[str, Any]					)   ->		Union[str, Any]:
             """simple docstring"""




             super().setUp()

             # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
             _lowerCamelCase			:						Any            =[
                 'l',
                 'o',
                 'w',
                 'e',
                 'r',
                 's',
                 't',
                 'i',
                 'd',
                 'n',
                 '\u0120',
                 '\u0120l',
                 '\u0120n',
                 '\u0120lo',
                 '\u0120low',
                 'er',
                 '\u0120lowest',
                 '\u0120newer',
                 '\u0120wider',
                 '<unk>',
                 '<|endoftext|>',
             ]
             _lowerCamelCase			:						Dict            =dict(zip(lowercase_     ,  range(len(lowercase_					)					)					)					)
             _lowerCamelCase			:						Tuple            =['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
             _lowerCamelCase			:						Optional[int]            ={'unk_token': '<unk>'}

             _lowerCamelCase			:						List[Any]            =os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES['vocab_file']					)
             _lowerCamelCase			:						Union[str, Any]            =os.path.join(self.tmpdirname     ,  VOCAB_FILES_NAMES['merges_file']					)
             with open(self.vocab_file     ,  'w'     ,  encoding='utf-8'					) as fp:
                    fp.write(json.dumps(lowercase_					) + '\n'					)
             with open(self.merges_file     ,  'w'     ,  encoding='utf-8'					) as fp:
                    fp.write('\n'.join(lowercase_					)					)






      def 			lowerCamelCase			(   self		:	Union[str, Any]     ,  **lowercase_		:	Tuple					)   ->		Optional[Any]:
             """simple docstring"""




             kwargs.update(self.special_tokens_map					)
             return CodeGenTokenizer.from_pretrained(self.tmpdirname     ,  **lowercase_					)






      def 			lowerCamelCase			(   self		:	str     ,  **lowercase_		:	List[Any]					)   ->		Tuple:
             """simple docstring"""




             kwargs.update(self.special_tokens_map					)
             return CodeGenTokenizerFast.from_pretrained(self.tmpdirname     ,  **lowercase_					)






      def 			lowerCamelCase			(   self		:	int     ,  lowercase_		:	Any					)   ->		Optional[int]:
             """simple docstring"""




             _lowerCamelCase			:						Dict            ='lower newer'
             _lowerCamelCase			:						List[str]            ='lower newer'
             return input_text, output_text






      def 			lowerCamelCase			(   self		:	Dict					)   ->		int:
             """simple docstring"""




             _lowerCamelCase			:						List[Any]            =CodeGenTokenizer(self.vocab_file     ,  self.merges_file     ,  **self.special_tokens_map					)
             _lowerCamelCase			:						Tuple            ='lower newer'
             _lowerCamelCase			:						str            =['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
             _lowerCamelCase			:						Tuple            =tokenizer.tokenize(lowercase_     ,  add_prefix_space=lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						Optional[int]            =tokens + [tokenizer.unk_token]
             _lowerCamelCase			:						Union[str, Any]            =[14, 15, 10, 9, 3, 2, 15, 19]
             self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_					)     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	Optional[int]					)   ->		int:
             """simple docstring"""




             if not self.test_rust_tokenizer:
                    return

             _lowerCamelCase			:						Tuple            =self.get_tokenizer()
             _lowerCamelCase			:						Dict            =self.get_rust_tokenizer(add_prefix_space=lowercase_					)

             _lowerCamelCase			:						Union[str, Any]            ='lower newer'

             # Testing tokenization
             _lowerCamelCase			:						Optional[Any]            =tokenizer.tokenize(lowercase_     ,  add_prefix_space=lowercase_					)
             _lowerCamelCase			:						List[str]            =rust_tokenizer.tokenize(lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)

             # Testing conversion to ids without special tokens
             _lowerCamelCase			:						int            =tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_     ,  add_prefix_space=lowercase_					)
             _lowerCamelCase			:						List[str]            =rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)

             # Testing conversion to ids with special tokens
             _lowerCamelCase			:						Any            =self.get_rust_tokenizer(add_prefix_space=lowercase_					)
             _lowerCamelCase			:						Dict            =tokenizer.encode(lowercase_     ,  add_prefix_space=lowercase_					)
             _lowerCamelCase			:						List[str]            =rust_tokenizer.encode(lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)

             # Testing the unknown token
             _lowerCamelCase			:						List[str]            =tokens + [rust_tokenizer.unk_token]
             _lowerCamelCase			:						List[Any]            =[14, 15, 10, 9, 3, 2, 15, 19]
             self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowercase_					)     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	Any     ,  *lowercase_		:	Optional[Any]     ,  **lowercase_		:	Tuple					)   ->		str:
             """simple docstring"""




             pass






      def 			lowerCamelCase			(   self		:	int     ,  lowercase_		:	Any=15					)   ->		List[Any]:
             """simple docstring"""




             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                    with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
                           _lowerCamelCase			:						List[Any]            =self.rust_tokenizer_class.from_pretrained(lowercase_     ,  **lowercase_					)

                           # Simple input
                           _lowerCamelCase			:						Optional[Any]            ='This is a simple input'
                           _lowerCamelCase			:						Optional[Any]            =['This is a simple input 1', 'This is a simple input 2']
                           _lowerCamelCase			:						Tuple            =('This is a simple input', 'This is a pair')
                           _lowerCamelCase			:						Optional[Any]            =[
                               ('This is a simple input 1', 'This is a simple input 2'),
                               ('This is a simple pair 1', 'This is a simple pair 2'),
                           ]

                           # Simple input tests
                           self.assertRaises(lowercase_     ,  tokenizer_r.encode     ,  lowercase_     ,  max_length=lowercase_     ,  padding='max_length'					)

                           # Simple input
                           self.assertRaises(lowercase_     ,  tokenizer_r.encode_plus     ,  lowercase_     ,  max_length=lowercase_     ,  padding='max_length'					)

                           # Simple input
                           self.assertRaises(
                               lowercase_     ,  tokenizer_r.batch_encode_plus     ,  lowercase_     ,  max_length=lowercase_     ,  padding='max_length'     ,  )

                           # Pair input
                           self.assertRaises(lowercase_     ,  tokenizer_r.encode     ,  lowercase_     ,  max_length=lowercase_     ,  padding='max_length'					)

                           # Pair input
                           self.assertRaises(lowercase_     ,  tokenizer_r.encode_plus     ,  lowercase_     ,  max_length=lowercase_     ,  padding='max_length'					)

                           # Pair input
                           self.assertRaises(
                               lowercase_     ,  tokenizer_r.batch_encode_plus     ,  lowercase_     ,  max_length=lowercase_     ,  padding='max_length'     ,  )






      def 			lowerCamelCase			(   self		:	Optional[Any]					)   ->		Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase			:						List[Any]            =CodeGenTokenizer.from_pretrained(self.tmpdirname     ,  pad_token='<pad>'					)

             # Simple input
             _lowerCamelCase			:						Dict            ='This is a simple input'
             _lowerCamelCase			:						Optional[int]            =['This is a simple input looooooooong', 'This is a simple input']
             _lowerCamelCase			:						List[Any]            =('This is a simple input', 'This is a pair')
             _lowerCamelCase			:						Tuple            =[
                 ('This is a simple input loooooong', 'This is a simple input'),
                 ('This is a simple pair loooooong', 'This is a simple pair'),
             ]

             _lowerCamelCase			:						str            =tokenizer.pad_token_id

             _lowerCamelCase			:						List[Any]            =tokenizer(lowercase_     ,  padding='max_length'     ,  max_length=30     ,  return_tensors='np'					)
             _lowerCamelCase			:						Tuple            =tokenizer(lowercase_     ,  padding=lowercase_     ,  truncate=lowercase_     ,  return_tensors='np'					)
             _lowerCamelCase			:						Any            =tokenizer(*lowercase_     ,  padding='max_length'     ,  max_length=60     ,  return_tensors='np'					)
             _lowerCamelCase			:						Any            =tokenizer(lowercase_     ,  padding=lowercase_     ,  truncate=lowercase_     ,  return_tensors='np'					)

             # s
             # test single string max_length padding
             self.assertEqual(out_s['input_ids'].shape[-1]     ,  30					)
             self.assertTrue(pad_token_id in out_s['input_ids']					)
             self.assertTrue(0 in out_s['attention_mask']					)

             # s2
             # test automatic padding
             self.assertEqual(out_sa['input_ids'].shape[-1]     ,  33					)
             # long slice doesn't have padding
             self.assertFalse(pad_token_id in out_sa['input_ids'][0]					)
             self.assertFalse(0 in out_sa['attention_mask'][0]					)
             # short slice does have padding
             self.assertTrue(pad_token_id in out_sa['input_ids'][1]					)
             self.assertTrue(0 in out_sa['attention_mask'][1]					)

             # p
             # test single pair max_length padding
             self.assertEqual(out_p['input_ids'].shape[-1]     ,  60					)
             self.assertTrue(pad_token_id in out_p['input_ids']					)
             self.assertTrue(0 in out_p['attention_mask']					)

             # p2
             # test automatic padding pair
             self.assertEqual(out_pa['input_ids'].shape[-1]     ,  52					)
             # long slice pair doesn't have padding
             self.assertFalse(pad_token_id in out_pa['input_ids'][0]					)
             self.assertFalse(0 in out_pa['attention_mask'][0]					)
             # short slice pair does have padding
             self.assertTrue(pad_token_id in out_pa['input_ids'][1]					)
             self.assertTrue(0 in out_pa['attention_mask'][1]					)






      def 			lowerCamelCase			(   self		:	int					)   ->		Any:
             """simple docstring"""




             _lowerCamelCase			:						int            ='$$$'
             _lowerCamelCase			:						Union[str, Any]            =CodeGenTokenizer.from_pretrained(self.tmpdirname     ,  bos_token=lowercase_     ,  add_bos_token=lowercase_					)

             _lowerCamelCase			:						Optional[Any]            ='This is a simple input'
             _lowerCamelCase			:						Union[str, Any]            =['This is a simple input 1', 'This is a simple input 2']

             _lowerCamelCase			:						Union[str, Any]            =tokenizer.bos_token_id

             _lowerCamelCase			:						Optional[int]            =tokenizer(lowercase_					)
             _lowerCamelCase			:						List[str]            =tokenizer(lowercase_					)

             self.assertEqual(out_s.input_ids[0]     ,  lowercase_					)
             self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids					)					)

             _lowerCamelCase			:						List[Any]            =tokenizer.decode(out_s.input_ids					)
             _lowerCamelCase			:						Optional[Any]            =tokenizer.batch_decode(out_sa.input_ids					)

             self.assertEqual(decode_s.split()[0]     ,  lowercase_					)
             self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa					)					)






      @slow
      def 			lowerCamelCase			(   self		:	Optional[int]					)   ->		Optional[int]:
             """simple docstring"""




             _lowerCamelCase			:						List[Any]            =CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono'					)

             _lowerCamelCase			:						Union[str, Any]            ='\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
             _lowerCamelCase			:						Optional[int]            ='\nif len_a > len_b:      result = a\nelse:      result = b'

             _lowerCamelCase			:						List[str]            =tokenizer.encode(lowercase_					)
             _lowerCamelCase			:						str            =['^#', re.escape('<|endoftext|>'					), '^\'\'\'', '^"""', '\n\n\n']
             _lowerCamelCase			:						str            =tokenizer.decode(lowercase_     ,  truncate_before_pattern=lowercase_					)
             self.assertEqual(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	int					)   ->		Optional[int]:
             """simple docstring"""




             pass






from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ...utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices


lowerCamelCase		   =			logging.get_logger(__name__)

lowerCamelCase		   =			{
    'microsoft/focalnet-tiny': 'https://huggingface.co/microsoft/focalnet-tiny/resolve/main/config.json',
}




class A    (  UpperCamelCase_       ,				UpperCamelCase_	):
      UpperCamelCase__				:    str ='focalnet'






      def __init__(   self		:	Optional[int]     ,  lowercase_		:	List[Any]=224     ,  lowercase_		:	str=4     ,  lowercase_		:	Any=3     ,  lowercase_		:	Dict=96     ,  lowercase_		:	List[str]=False     ,  lowercase_		:	Any=[192, 384, 768, 768]     ,  lowercase_		:	Dict=[2, 2, 6, 2]     ,  lowercase_		:	Optional[int]=[2, 2, 2, 2]     ,  lowercase_		:	List[Any]=[3, 3, 3, 3]     ,  lowercase_		:	Any="gelu"     ,  lowercase_		:	str=4.0     ,  lowercase_		:	List[Any]=0.0     ,  lowercase_		:	Tuple=0.1     ,  lowercase_		:	Optional[Any]=False     ,  lowercase_		:	Optional[int]=1E-4     ,  lowercase_		:	Dict=False     ,  lowercase_		:	Optional[Any]=False     ,  lowercase_		:	Dict=False     ,  lowercase_		:	List[Any]=0.02     ,  lowercase_		:	Any=1E-5     ,  lowercase_		:	Any=32     ,  lowercase_		:	List[str]=None     ,  lowercase_		:	int=None     ,  **lowercase_		:	Optional[Any]     ,  )   ->		Optional[Any]:
             """simple docstring"""




             super().__init__(**lowercase_					)

             _lowerCamelCase			:						Union[str, Any]            =image_size
             _lowerCamelCase			:						str            =patch_size
             _lowerCamelCase			:						str            =num_channels
             _lowerCamelCase			:						Tuple            =embed_dim
             _lowerCamelCase			:						Dict            =use_conv_embed
             _lowerCamelCase			:						int            =hidden_sizes
             _lowerCamelCase			:						str            =depths
             _lowerCamelCase			:						int            =focal_levels
             _lowerCamelCase			:						List[Any]            =focal_windows
             _lowerCamelCase			:						List[Any]            =hidden_act
             _lowerCamelCase			:						int            =mlp_ratio
             _lowerCamelCase			:						Any            =hidden_dropout_prob
             _lowerCamelCase			:						Optional[Any]            =drop_path_rate
             _lowerCamelCase			:						int            =use_layerscale
             _lowerCamelCase			:						str            =layerscale_value
             _lowerCamelCase			:						Dict            =use_post_layernorm
             _lowerCamelCase			:						str            =use_post_layernorm_in_modulation
             _lowerCamelCase			:						List[Any]            =normalize_modulator
             _lowerCamelCase			:						Optional[Any]            =initializer_range
             _lowerCamelCase			:						List[str]            =layer_norm_eps
             _lowerCamelCase			:						Optional[int]            =encoder_stride
             _lowerCamelCase			:						List[str]            =['stem'] + [F'''stage{idx}''' for idx in range(1     ,  len(self.depths					) + 1					)]
             _lowerCamelCase      ,	_lowerCamelCase			:						Union[str, Any]            =get_aligned_output_features_output_indices(
                 out_features=lowercase_     ,  out_indices=lowercase_     ,  stage_names=self.stage_names					)


