

import importlib
import json
import os
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
import transformers.models.auto
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.bert.configuration_bert import BertConfig
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402


UpperCAmelCase_							:					Optional[int]		 =   get_tests_dir('''fixtures/dummy-config.json''')






class      lowerCamelCase_      (	unittest.TestCase	):




	def        lowerCAmelCase_       (     self						:     Tuple							):
		__A :			List[str]		    =			0




	def        lowerCAmelCase_       (     self						:     str							):
		self.assertIsNotNone(transformers.models.auto.__spec__							)
		self.assertIsNotNone(importlib.util.find_spec("""transformers.models.auto"""							)							)




	def        lowerCAmelCase_       (     self						:     str							):
		__A :			Union[str, Any]		    =			AutoConfig.from_pretrained("""bert-base-uncased"""							)
		self.assertIsInstance(a__					,      a__							)




	def        lowerCAmelCase_       (     self						:     List[str]							):
		__A :			int		    =			AutoConfig.from_pretrained(a__							)
		self.assertIsInstance(a__					,      a__							)




	def        lowerCAmelCase_       (     self						:     Optional[Any]							):
		__A :			Optional[int]		    =			AutoConfig.from_pretrained(a__							)
		self.assertIsInstance(a__					,      a__							)




	def        lowerCAmelCase_       (     self						:     str							):
		__A :			Dict		    =			AutoConfig.for_model("""roberta"""							)
		self.assertIsInstance(a__					,      a__							)




	def        lowerCAmelCase_       (     self						:     str							):





		with tempfile.TemporaryDirectory() as tmp_dir:
			# This model name contains bert and roberta, but roberta ends up being picked.
			__A :			List[str]		    =			os.path.join(a__					,      """fake-roberta"""							)
			os.makedirs(a__					,      exist_ok=a__							)
			with open(os.path.join(a__					,      """config.json"""							)					,      """w"""							) as f:
				f.write(json.dumps({}							)							)
			__A :			Any		    =			AutoConfig.from_pretrained(a__							)
			self.assertEqual(type(a__							)					,      a__							)




	def        lowerCAmelCase_       (     self						:     Dict							):





		try:
			AutoConfig.register("""custom"""					,      a__							)
			# Wrong model type will raise an error
			with self.assertRaises(a__							):
				AutoConfig.register("""model"""					,      a__							)
			# Trying to register something existing in the Transformers library will raise an error
			with self.assertRaises(a__							):
				AutoConfig.register("""bert"""					,      a__							)

			# Now that the config is registered, it can be used as any other config with the auto-API
			__A :			int		    =			CustomConfig()
			with tempfile.TemporaryDirectory() as tmp_dir:
				config.save_pretrained(a__							)
				__A :			str		    =			AutoConfig.from_pretrained(a__							)
				self.assertIsInstance(a__					,      a__							)

		finally:
			if "custom" in CONFIG_MAPPING._extra_content:
				del CONFIG_MAPPING._extra_content["custom"]




	def        lowerCAmelCase_       (     self						:     int							):





		with self.assertRaisesRegex(
		    a__					,      """bert-base is not a local folder and is not a valid model identifier"""							):
			__A :			List[str]		    =			AutoConfig.from_pretrained("""bert-base"""							)




	def        lowerCAmelCase_       (     self						:     Optional[int]							):





		with self.assertRaisesRegex(
		    a__					,      R"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""							):
			__A :			Dict		    =			AutoConfig.from_pretrained(a__					,      revision="""aaaaaa"""							)




	def        lowerCAmelCase_       (     self						:     Union[str, Any]							):





		with self.assertRaisesRegex(
		    a__					,      """hf-internal-testing/no-config-test-repo does not appear to have a file named config.json."""					,      ):
			__A :			Any		    =			AutoConfig.from_pretrained("""hf-internal-testing/no-config-test-repo"""							)




	def        lowerCAmelCase_       (     self						:     Tuple							):





		with self.assertRaises(a__							):
			__A :			Dict		    =			AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""							)
		# If remote code is disabled, we can't load this config.
		with self.assertRaises(a__							):
			__A :			Tuple		    =			AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""					,      trust_remote_code=a__							)

		__A :			Dict		    =			AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""					,      trust_remote_code=a__							)
		self.assertEqual(config.__class__.__name__					,      """NewModelConfig"""							)

		# Test config can be reloaded.
		with tempfile.TemporaryDirectory() as tmp_dir:
			config.save_pretrained(a__							)
			__A :			Tuple		    =			AutoConfig.from_pretrained(a__					,      trust_remote_code=a__							)
		self.assertEqual(reloaded_config.__class__.__name__					,      """NewModelConfig"""							)




	def        lowerCAmelCase_       (     self						:     Optional[int]							):






		class      lowerCamelCase_      (	lowerCamelCase__	):
			_lowercase		:   Optional[Any]     					=				'''new-model'''

		try:
			AutoConfig.register("""new-model"""					,      a__							)
			# If remote code is not set, the default is to use local
			__A :			str		    =			AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""							)
			self.assertEqual(config.__class__.__name__					,      """NewModelConfigLocal"""							)

			# If remote code is disabled, we load the local one.
			__A :			Optional[Any]		    =			AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""					,      trust_remote_code=a__							)
			self.assertEqual(config.__class__.__name__					,      """NewModelConfigLocal"""							)

			# If remote is enabled, we load from the Hub
			__A :			Tuple		    =			AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""					,      trust_remote_code=a__							)
			self.assertEqual(config.__class__.__name__					,      """NewModelConfig"""							)

		finally:
			if "new-model" in CONFIG_MAPPING._extra_content:
				del CONFIG_MAPPING._extra_content["new-model"]












import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class       __A							(   lowerCamelCase__			,unittest.TestCase	):




      """simple docstring"""







      UpperCAmelCase__							=   OpenAIGPTTokenizer
      UpperCAmelCase__							=   OpenAIGPTTokenizerFast
      UpperCAmelCase__							=   True
      UpperCAmelCase__							=   False


      def 					__snake_case      (		self):



             """simple docstring"""





             super().setUp()

             # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
             _lowerCamelCase				:  Tuple						=		[
                 '''l''',
                 '''o''',
                 '''w''',
                 '''e''',
                 '''r''',
                 '''s''',
                 '''t''',
                 '''i''',
                 '''d''',
                 '''n''',
                 '''w</w>''',
                 '''r</w>''',
                 '''t</w>''',
                 '''lo''',
                 '''low''',
                 '''er</w>''',
                 '''low</w>''',
                 '''lowest</w>''',
                 '''newer</w>''',
                 '''wider</w>''',
                 '''<unk>''',
             ]
             _lowerCamelCase				:  Any						=		dict(zip(a__   ,				range(len(a__))))
             _lowerCamelCase				:  List[str]						=		['''#version: 0.2''', '''l o''', '''lo w''', '''e r</w>''', '''''']

             _lowerCamelCase				:  str						=		os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''vocab_file'''])
             _lowerCamelCase				:  int						=		os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''merges_file'''])
             with open(self.vocab_file   ,				'''w''') as fp:
                    fp.write(json.dumps(a__))
             with open(self.merges_file   ,				'''w''') as fp:
                    fp.write('''\n'''.join(a__))


      def 					__snake_case      (		self   ,				a__):



             """simple docstring"""





             return "lower newer", "lower newer"


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Optional[Any]						=		OpenAIGPTTokenizer(self.vocab_file   ,				self.merges_file)

             _lowerCamelCase				:  Dict						=		'''lower'''
             _lowerCamelCase				:  Optional[Any]						=		['''low''', '''er</w>''']
             _lowerCamelCase				:  List[Any]						=		tokenizer.tokenize(a__)
             self.assertListEqual(a__   ,				a__)

             _lowerCamelCase				:  Any						=		tokens + ['''<unk>''']
             _lowerCamelCase				:  Optional[int]						=		[14, 15, 20]
             self.assertListEqual(tokenizer.convert_tokens_to_ids(a__)   ,				a__)


      def 					__snake_case      (		self   ,				a__=15):



             """simple docstring"""





             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                    with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                           _lowerCamelCase				:  Dict						=		self.rust_tokenizer_class.from_pretrained(a__   ,				**a__)

                           # Simple input
                           _lowerCamelCase				:  Optional[int]						=		'''This is a simple input'''
                           _lowerCamelCase				:  Optional[Any]						=		['''This is a simple input 1''', '''This is a simple input 2''']
                           _lowerCamelCase				:  Optional[int]						=		('''This is a simple input''', '''This is a pair''')
                           _lowerCamelCase				:  List[Any]						=		[
                               ('''This is a simple input 1''', '''This is a simple input 2'''),
                               ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                           ]

                           # Simple input tests
                           self.assertRaises(a__   ,				tokenizer_r.encode   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Simple input
                           self.assertRaises(a__   ,				tokenizer_r.encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Simple input
                           self.assertRaises(
                               a__   ,				tokenizer_r.batch_encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length'''   ,				)

                           # Pair input
                           self.assertRaises(a__   ,				tokenizer_r.encode   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Pair input
                           self.assertRaises(a__   ,				tokenizer_r.encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Pair input
                           self.assertRaises(
                               a__   ,				tokenizer_r.batch_encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length'''   ,				)



      def 					__snake_case      (		self):



             """simple docstring"""





             pass







@require_ftfy
@require_spacy
@require_tokenizers
class       __A							(   lowerCamelCase__	):




      """simple docstring"""







      pass

