






import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin
class 							A_    (						__lowercase ,   unittest.TestCase			):






 '''simple docstring'''




 _SCREAMING_SNAKE_CASE							:			Optional[int] =					DebertaTokenizer
 _SCREAMING_SNAKE_CASE							:			Optional[int] =					True
 _SCREAMING_SNAKE_CASE							:			List[Any] =					DebertaTokenizerFast


 def  snake_case__				(	self) ->		Optional[int]:

     """simple docstring"""



     super().setUp()

     # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
     _UpperCAmelCase				:      List[Any]			  =				[
         '''l''',
         '''o''',
         '''w''',
         '''e''',
         '''r''',
         '''s''',
         '''t''',
         '''i''',
         '''d''',
         '''n''',
         '''\u0120''',
         '''\u0120l''',
         '''\u0120n''',
         '''\u0120lo''',
         '''\u0120low''',
         '''er''',
         '''\u0120lowest''',
         '''\u0120newer''',
         '''\u0120wider''',
         '''[UNK]''',
     ]
     _UpperCAmelCase				:      Union[str, Any]			  =				dict(zip(_A							,  range(len(_A))))
     _UpperCAmelCase				:      Tuple			  =				['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
     _UpperCAmelCase				:      Tuple			  =				{'''unk_token''': '''[UNK]'''}

     _UpperCAmelCase				:      Any			  =				os.path.join(self.tmpdirname							,  VOCAB_FILES_NAMES['''vocab_file'''])
     _UpperCAmelCase				:      Any			  =				os.path.join(self.tmpdirname							,  VOCAB_FILES_NAMES['''merges_file'''])
     with open(self.vocab_file							,  '''w'''							,  encoding='''utf-8''') as fp:
         fp.write(json.dumps(_A) + '''\n''')
     with open(self.merges_file							,  '''w'''							,  encoding='''utf-8''') as fp:
         fp.write('''\n'''.join(_A))


 def  snake_case__				(	self							,  **_A) ->		List[Any]:

     """simple docstring"""



     kwargs.update(self.special_tokens_map)
     return self.tokenizer_class.from_pretrained(self.tmpdirname							,  **_A)


 def  snake_case__				(	self							,  _A) ->		str:

     """simple docstring"""



     _UpperCAmelCase				:      Union[str, Any]			  =				'''lower newer'''
     _UpperCAmelCase				:      Dict			  =				'''lower newer'''
     return input_text, output_text


 def  snake_case__				(	self) ->		Any:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[int]			  =				self.get_tokenizer()
     _UpperCAmelCase				:      Optional[int]			  =				'''lower newer'''
     _UpperCAmelCase				:      Any			  =				['''l''', '''o''', '''w''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
     _UpperCAmelCase				:      List[str]			  =				tokenizer.tokenize(_A)
     self.assertListEqual(_A							,  _A)

     _UpperCAmelCase				:      Tuple			  =				tokens + [tokenizer.unk_token]
     _UpperCAmelCase				:      Optional[Any]			  =				[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
     self.assertListEqual(tokenizer.convert_tokens_to_ids(_A)							,  _A)


 def  snake_case__				(	self) ->		Union[str, Any]:

     """simple docstring"""



     _UpperCAmelCase				:      int			  =				self.get_tokenizer()
     _UpperCAmelCase				:      Union[str, Any]			  =				tokenizer('''Hello'''							,  '''World''')
     _UpperCAmelCase				:      Union[str, Any]			  =				[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
     self.assertListEqual(tokd['''token_type_ids''']							,  _A)


 @slow
 def  snake_case__				(	self) ->		List[str]:

     """simple docstring"""



     _UpperCAmelCase				:      Union[str, Any]			  =				self.tokenizer_class.from_pretrained('''microsoft/deberta-base''')

     _UpperCAmelCase				:      int			  =				tokenizer.encode('''sequence builders'''							,  add_special_tokens=_A)
     _UpperCAmelCase				:      str			  =				tokenizer.encode('''multi-sequence build'''							,  add_special_tokens=_A)

     _UpperCAmelCase				:      Optional[Any]			  =				tokenizer.encode(
         '''sequence builders'''							,  add_special_tokens=_A							,  add_prefix_space=_A)
     _UpperCAmelCase				:      Union[str, Any]			  =				tokenizer.encode(
         '''sequence builders'''							,  '''multi-sequence build'''							,  add_special_tokens=_A							,  add_prefix_space=_A)

     _UpperCAmelCase				:      Optional[int]			  =				tokenizer.build_inputs_with_special_tokens(_A)
     _UpperCAmelCase				:      Dict			  =				tokenizer.build_inputs_with_special_tokens(_A							,  _A)

     assert encoded_sentence == encoded_text_from_decode
     assert encoded_pair == encoded_pair_from_decode


 @slow
 def  snake_case__				(	self) ->		int:

     """simple docstring"""



     _UpperCAmelCase				:      int			  =				[self.tokenizer_class]
     if self.test_rust_tokenizer:
         tokenizer_classes.append(self.rust_tokenizer_class)

     for tokenizer_class in tokenizer_classes:
         _UpperCAmelCase				:      str			  =				tokenizer_class.from_pretrained('''microsoft/deberta-base''')

         _UpperCAmelCase				:      int			  =				[
             '''ALBERT: A Lite BERT for Self-supervised Learning of Language Representations''',
             '''ALBERT incorporates two parameter reduction techniques''',
             '''The first one is a factorized embedding parameterization. By decomposing the large vocabulary'''
             ''' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'''
             ''' vocabulary embedding.''',
         ]

         _UpperCAmelCase				:      Any			  =				tokenizer(_A							,  padding=_A)
         _UpperCAmelCase				:      List[str]			  =				[tokenizer.decode(_A							,  skip_special_tokens=_A) for seq in encoding['''input_ids''']]

         # fmt: off
         _UpperCAmelCase				:      str			  =				{
             '''input_ids''': [
                 [1, 2118, 11126, 565, 35, 83, 25191, 163, 18854, 13, 12156, 12, 16101, 25376, 13807, 9, 22205, 27893, 1635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                 [1, 2118, 11126, 565, 24536, 80, 43797, 4878, 7373, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                 [1, 133, 78, 65, 16, 10, 3724, 1538, 33183, 11303, 43797, 1938, 4, 870, 24165, 29105, 5, 739, 32644, 33183, 11303, 36173, 88, 80, 650, 7821, 45940, 6, 52, 2559, 5, 1836, 9, 5, 7397, 13171, 31, 5, 1836, 9, 32644, 33183, 11303, 4, 2]
             ],
             '''token_type_ids''': [
                 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
             ],
             '''attention_mask''': [
                 [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                 [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                 [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
             ]
         }
         # fmt: on

         _UpperCAmelCase				:      Any			  =				[
             '''ALBERT: A Lite BERT for Self-supervised Learning of Language Representations''',
             '''ALBERT incorporates two parameter reduction techniques''',
             '''The first one is a factorized embedding parameterization. By decomposing the large vocabulary'''
             ''' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'''
             ''' vocabulary embedding.''',
         ]

         self.assertDictEqual(encoding.data							,  _A)

         for expected, decoded in zip(_A							,  _A):
             self.assertEqual(_A							,  _A)












import importlib
import json
import os
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
import transformers.models.auto
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.bert.configuration_bert import BertConfig
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402


SCREAMING_SNAKE_CASE 							=						get_tests_dir('fixtures/dummy-config.json')
class 							A_    (						unittest.TestCase			):






 '''simple docstring'''




 def  snake_case__				(	self) ->		Optional[int]:

     """simple docstring"""



     _UpperCAmelCase				:      Union[str, Any]			  =				0


 def  snake_case__				(	self) ->		Union[str, Any]:

     """simple docstring"""



     self.assertIsNotNone(transformers.models.auto.__spec__)
     self.assertIsNotNone(importlib.util.find_spec('''transformers.models.auto'''))


 def  snake_case__				(	self) ->		Optional[Any]:

     """simple docstring"""



     _UpperCAmelCase				:      List[str]			  =				AutoConfig.from_pretrained('''bert-base-uncased''')
     self.assertIsInstance(_A							,  _A)


 def  snake_case__				(	self) ->		Dict:

     """simple docstring"""



     _UpperCAmelCase				:      Tuple			  =				AutoConfig.from_pretrained(_A)
     self.assertIsInstance(_A							,  _A)


 def  snake_case__				(	self) ->		Optional[int]:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[Any]			  =				AutoConfig.from_pretrained(_A)
     self.assertIsInstance(_A							,  _A)


 def  snake_case__				(	self) ->		Any:

     """simple docstring"""



     _UpperCAmelCase				:      str			  =				AutoConfig.for_model('''roberta''')
     self.assertIsInstance(_A							,  _A)


 def  snake_case__				(	self) ->		int:

     """simple docstring"""



     with tempfile.TemporaryDirectory() as tmp_dir:
         # This model name contains bert and roberta, but roberta ends up being picked.
         _UpperCAmelCase				:      str			  =				os.path.join(_A							,  '''fake-roberta''')
         os.makedirs(_A							,  exist_ok=_A)
         with open(os.path.join(_A							,  '''config.json''')							,  '''w''') as f:
             f.write(json.dumps({}))
         _UpperCAmelCase				:      Any			  =				AutoConfig.from_pretrained(_A)
         self.assertEqual(type(_A)							,  _A)


 def  snake_case__				(	self) ->		List[Any]:

     """simple docstring"""



     try:
         AutoConfig.register('''custom'''							,  _A)
         # Wrong model type will raise an error
         with self.assertRaises(_A):
             AutoConfig.register('''model'''							,  _A)
         # Trying to register something existing in the Transformers library will raise an error
         with self.assertRaises(_A):
             AutoConfig.register('''bert'''							,  _A)

         # Now that the config is registered, it can be used as any other config with the auto-API
         _UpperCAmelCase				:      Tuple			  =				CustomConfig()
         with tempfile.TemporaryDirectory() as tmp_dir:
             config.save_pretrained(_A)
             _UpperCAmelCase				:      Tuple			  =				AutoConfig.from_pretrained(_A)
             self.assertIsInstance(_A							,  _A)

     finally:
         if "custom" in CONFIG_MAPPING._extra_content:
             del CONFIG_MAPPING._extra_content["custom"]


 def  snake_case__				(	self) ->		Union[str, Any]:

     """simple docstring"""



     with self.assertRaisesRegex(
         _A							,  '''bert-base is not a local folder and is not a valid model identifier'''):
         _UpperCAmelCase				:      Tuple			  =				AutoConfig.from_pretrained('''bert-base''')


 def  snake_case__				(	self) ->		int:

     """simple docstring"""



     with self.assertRaisesRegex(
         _A							,  R'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''):
         _UpperCAmelCase				:      List[Any]			  =				AutoConfig.from_pretrained(_A							,  revision='''aaaaaa''')


 def  snake_case__				(	self) ->		Optional[Any]:

     """simple docstring"""



     with self.assertRaisesRegex(
         _A							,  '''hf-internal-testing/no-config-test-repo does not appear to have a file named config.json.'''							,  ):
         _UpperCAmelCase				:      List[str]			  =				AutoConfig.from_pretrained('''hf-internal-testing/no-config-test-repo''')


 def  snake_case__				(	self) ->		List[str]:

     """simple docstring"""



     with self.assertRaises(_A):
         _UpperCAmelCase				:      Dict			  =				AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model''')
     # If remote code is disabled, we can't load this config.
     with self.assertRaises(_A):
         _UpperCAmelCase				:      Optional[int]			  =				AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model'''							,  trust_remote_code=_A)

     _UpperCAmelCase				:      int			  =				AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model'''							,  trust_remote_code=_A)
     self.assertEqual(config.__class__.__name__							,  '''NewModelConfig''')

     # Test config can be reloaded.
     with tempfile.TemporaryDirectory() as tmp_dir:
         config.save_pretrained(_A)
         _UpperCAmelCase				:      Any			  =				AutoConfig.from_pretrained(_A							,  trust_remote_code=_A)
     self.assertEqual(reloaded_config.__class__.__name__							,  '''NewModelConfig''')


 def  snake_case__				(	self) ->		Tuple:

     """simple docstring"""



     class 							A_    (						__lowercase			):






      '''simple docstring'''




      _SCREAMING_SNAKE_CASE							:			Union[str, Any] =					"new-model"

     try:
         AutoConfig.register('''new-model'''							,  _A)
         # If remote code is not set, the default is to use local
         _UpperCAmelCase				:      List[str]			  =				AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model''')
         self.assertEqual(config.__class__.__name__							,  '''NewModelConfigLocal''')

         # If remote code is disabled, we load the local one.
         _UpperCAmelCase				:      List[Any]			  =				AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model'''							,  trust_remote_code=_A)
         self.assertEqual(config.__class__.__name__							,  '''NewModelConfigLocal''')

         # If remote is enabled, we load from the Hub
         _UpperCAmelCase				:      Dict			  =				AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model'''							,  trust_remote_code=_A)
         self.assertEqual(config.__class__.__name__							,  '''NewModelConfig''')

     finally:
         if "new-model" in CONFIG_MAPPING._extra_content:
             del CONFIG_MAPPING._extra_content["new-model"]




