
'''simple docstring'''


from __future__ import annotations

import copy
import tempfile
import unittest

from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPTaConfig, TaConfig, TapasConfig, is_tf_available
from transformers.testing_utils import (
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tensorflow_probability,
    require_tf,
    slow,
)

from ..bert.test_modeling_bert import BertModelTester


if is_tf_available():
					from transformers import (
					    TFAutoModel,
					    TFAutoModelForCausalLM,
					    TFAutoModelForMaskedLM,
					    TFAutoModelForPreTraining,
					    TFAutoModelForQuestionAnswering,
					    TFAutoModelForSeqaSeqLM,
					    TFAutoModelForSequenceClassification,
					    TFAutoModelForTableQuestionAnswering,
					    TFAutoModelForTokenClassification,
					    TFAutoModelWithLMHead,
					    TFBertForMaskedLM,
					    TFBertForPreTraining,
					    TFBertForQuestionAnswering,
					    TFBertForSequenceClassification,
					    TFBertModel,
					    TFFunnelBaseModel,
					    TFFunnelModel,
					    TFGPTaLMHeadModel,
					    TFRobertaForMaskedLM,
					    TFTaForConditionalGeneration,
					    TFTapasForQuestionAnswering,
					)
					from transformers.models.auto.modeling_tf_auto import (
					    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
					    TF_MODEL_FOR_MASKED_LM_MAPPING,
					    TF_MODEL_FOR_PRETRAINING_MAPPING,
					    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
					    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
					    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
					    TF_MODEL_MAPPING,
					)
					from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
					from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
					from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
					from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST

class 	__lowercase				(  _lowercase    ):
			lowerCamelCase			:		Optional[Any]	      =			"new-model"


if is_tf_available():

					class 	__lowercase				(  _lowercase    ):
								lowerCamelCase			:		Any	      =			NewModelConfig

@require_tf
class 	__lowercase				(  unittest.TestCase    ):

			@slow
			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Any     =		'''bert-base-cased'''
								lowerCamelCase_					:      List[Any]     =		AutoConfig.from_pretrained(__A      )
								self.assertIsNotNone(__A      )
								self.assertIsInstance(__A			,   __A      )

								lowerCamelCase_					:      int     =		TFAutoModel.from_pretrained(__A      )
								self.assertIsNotNone(__A      )
								self.assertIsInstance(__A			,   __A      )

			@slow
			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Tuple     =		'''bert-base-cased'''
								lowerCamelCase_					:      List[str]     =		AutoConfig.from_pretrained(__A      )
								self.assertIsNotNone(__A      )
								self.assertIsInstance(__A			,   __A      )

								lowerCamelCase_					:      Dict     =		TFAutoModelForPreTraining.from_pretrained(__A      )
								self.assertIsNotNone(__A      )
								self.assertIsInstance(__A			,   __A      )

			@slow
			def        UpperCAmelCase__ (self      ):
								for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
													lowerCamelCase_					:      Optional[int]     =		AutoConfig.from_pretrained(__A      )
													self.assertIsNotNone(__A      )
													self.assertIsInstance(__A			,   __A      )

													lowerCamelCase_					:      List[Any]     =		TFAutoModelForCausalLM.from_pretrained(__A      )
													lowerCamelCase_,      lowerCamelCase_					:      List[Any]     =		TFAutoModelForCausalLM.from_pretrained(__A			,   output_loading_info=__A      )
													self.assertIsNotNone(__A      )
													self.assertIsInstance(__A			,   __A      )

			@slow
			def        UpperCAmelCase__ (self      ):
								for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
													lowerCamelCase_					:      Union[str, Any]     =		AutoConfig.from_pretrained(__A      )
													self.assertIsNotNone(__A      )
													self.assertIsInstance(__A			,   __A      )

													lowerCamelCase_					:      Any     =		TFAutoModelWithLMHead.from_pretrained(__A      )
													self.assertIsNotNone(__A      )
													self.assertIsInstance(__A			,   __A      )

			@slow
			def        UpperCAmelCase__ (self      ):
								for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
													lowerCamelCase_					:      Optional[int]     =		AutoConfig.from_pretrained(__A      )
													self.assertIsNotNone(__A      )
													self.assertIsInstance(__A			,   __A      )

													lowerCamelCase_					:      List[Any]     =		TFAutoModelForMaskedLM.from_pretrained(__A      )
													lowerCamelCase_,      lowerCamelCase_					:      List[str]     =		TFAutoModelForMaskedLM.from_pretrained(__A			,   output_loading_info=__A      )
													self.assertIsNotNone(__A      )
													self.assertIsInstance(__A			,   __A      )

			@slow
			def        UpperCAmelCase__ (self      ):
								for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
													lowerCamelCase_					:      Any     =		AutoConfig.from_pretrained(__A      )
													self.assertIsNotNone(__A      )
													self.assertIsInstance(__A			,   __A      )

													lowerCamelCase_					:      Tuple     =		TFAutoModelForSeqaSeqLM.from_pretrained(__A      )
													lowerCamelCase_,      lowerCamelCase_					:      Tuple     =		TFAutoModelForSeqaSeqLM.from_pretrained(__A			,   output_loading_info=__A      )
													self.assertIsNotNone(__A      )
													self.assertIsInstance(__A			,   __A      )

			@slow
			def        UpperCAmelCase__ (self      ):
								# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								for model_name in ["bert-base-uncased"]:
													lowerCamelCase_					:      List[str]     =		AutoConfig.from_pretrained(__A      )
													self.assertIsNotNone(__A      )
													self.assertIsInstance(__A			,   __A      )

													lowerCamelCase_					:      str     =		TFAutoModelForSequenceClassification.from_pretrained(__A      )
													self.assertIsNotNone(__A      )
													self.assertIsInstance(__A			,   __A      )

			@slow
			def        UpperCAmelCase__ (self      ):
								# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								for model_name in ["bert-base-uncased"]:
													lowerCamelCase_					:      Dict     =		AutoConfig.from_pretrained(__A      )
													self.assertIsNotNone(__A      )
													self.assertIsInstance(__A			,   __A      )

													lowerCamelCase_					:      Tuple     =		TFAutoModelForQuestionAnswering.from_pretrained(__A      )
													self.assertIsNotNone(__A      )
													self.assertIsInstance(__A			,   __A      )

			@slow
			@require_tensorflow_probability
			def        UpperCAmelCase__ (self      ):
								for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
													lowerCamelCase_					:      Tuple     =		AutoConfig.from_pretrained(__A      )
													self.assertIsNotNone(__A      )
													self.assertIsInstance(__A			,   __A      )

													lowerCamelCase_					:      Dict     =		TFAutoModelForTableQuestionAnswering.from_pretrained(__A      )
													lowerCamelCase_,      lowerCamelCase_					:      Tuple     =		TFAutoModelForTableQuestionAnswering.from_pretrained(
													    __A			,   output_loading_info=__A      )
													self.assertIsNotNone(__A      )
													self.assertIsInstance(__A			,   __A      )

			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Optional[Any]     =		TFAutoModelWithLMHead.from_pretrained(__A      )
								self.assertIsInstance(__A			,   __A      )
								self.assertEqual(model.num_parameters()			,   1_4_4_1_0      )
								self.assertEqual(model.num_parameters(only_trainable=__A      )			,   1_4_4_1_0      )

			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Any     =		TFAutoModelWithLMHead.from_pretrained(__A      )
								self.assertIsInstance(__A			,   __A      )
								self.assertEqual(model.num_parameters()			,   1_4_4_1_0      )
								self.assertEqual(model.num_parameters(only_trainable=__A      )			,   1_4_4_1_0      )

			def        UpperCAmelCase__ (self      ):
								# For the auto model mapping, FunnelConfig has two models: FunnelModel and FunnelBaseModel
								lowerCamelCase_					:      Any     =		TFAutoModel.from_pretrained('''sgugger/funnel-random-tiny'''      )
								self.assertIsInstance(__A			,   __A      )

								lowerCamelCase_					:      Union[str, Any]     =		copy.deepcopy(model.config      )
								lowerCamelCase_					:      str     =		['''FunnelBaseModel''']
								lowerCamelCase_					:      int     =		TFAutoModel.from_config(__A      )
								self.assertIsInstance(__A			,   __A      )

								with tempfile.TemporaryDirectory() as tmp_dir:
													model.save_pretrained(__A      )
													lowerCamelCase_					:      Optional[Any]     =		TFAutoModel.from_pretrained(__A      )
													self.assertIsInstance(__A			,   __A      )

			def        UpperCAmelCase__ (self      ):
								try:
													AutoConfig.register('''new-model'''			,   __A      )

													lowerCamelCase_					:      Any     =		[
													    TFAutoModel,
													    TFAutoModelForCausalLM,
													    TFAutoModelForMaskedLM,
													    TFAutoModelForPreTraining,
													    TFAutoModelForQuestionAnswering,
													    TFAutoModelForSequenceClassification,
													    TFAutoModelForTokenClassification,
													]

													for auto_class in auto_classes:
																		with self.subTest(auto_class.__name__      ):
																							# Wrong config class will raise an error
																							with self.assertRaises(__A      ):
																												auto_class.register(__A			,   __A      )
																							auto_class.register(__A			,   __A      )
																							# Trying to register something existing in the Transformers library will raise an error
																							with self.assertRaises(__A      ):
																												auto_class.register(__A			,   __A      )

																							# Now that the config is registered, it can be used as any other config with the auto-API
																							lowerCamelCase_					:      Optional[int]     =		BertModelTester(self      ).get_config()
																							lowerCamelCase_					:      int     =		NewModelConfig(**tiny_config.to_dict()      )
																							lowerCamelCase_					:      Union[str, Any]     =		auto_class.from_config(__A      )
																							self.assertIsInstance(__A			,   __A      )

																							with tempfile.TemporaryDirectory() as tmp_dir:
																												model.save_pretrained(__A      )
																												lowerCamelCase_					:      int     =		auto_class.from_pretrained(__A      )
																												self.assertIsInstance(__A			,   __A      )

								finally:
													if "new-model" in CONFIG_MAPPING._extra_content:
																		del CONFIG_MAPPING._extra_content["new-model"]
													for mapping in (
													    TF_MODEL_MAPPING,
													    TF_MODEL_FOR_PRETRAINING_MAPPING,
													    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
													    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
													    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
													    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
													    TF_MODEL_FOR_MASKED_LM_MAPPING,
													):
																		if NewModelConfig in mapping._extra_content:
																							del mapping._extra_content[NewModelConfig]

			def        UpperCAmelCase__ (self      ):
								with self.assertRaisesRegex(
								    __A			,   '''bert-base is not a local folder and is not a valid model identifier'''      ):
													lowerCamelCase_					:      List[str]     =		TFAutoModel.from_pretrained('''bert-base'''      )

			def        UpperCAmelCase__ (self      ):
								with self.assertRaisesRegex(
								    __A			,   R'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''      ):
													lowerCamelCase_					:      str     =		TFAutoModel.from_pretrained(__A			,   revision='''aaaaaa'''      )

			def        UpperCAmelCase__ (self      ):
								with self.assertRaisesRegex(
								    __A			,   '''hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin'''			,   ):
													lowerCamelCase_					:      str     =		TFAutoModel.from_pretrained('''hf-internal-testing/config-no-model'''      )

			def        UpperCAmelCase__ (self      ):
								with self.assertRaisesRegex(__A			,   '''Use `from_pt=True` to load this model'''      ):
													lowerCamelCase_					:      Optional[int]     =		TFAutoModel.from_pretrained('''hf-internal-testing/tiny-bert-pt-only'''      )






			def        UpperCAmelCase__ (self      ):
								# Make sure we have cached the model.
								lowerCamelCase_					:      List[Any]     =		TFAutoModel.from_pretrained('''hf-internal-testing/tiny-random-bert'''      )
								with RequestCounter() as counter:
													lowerCamelCase_					:      Tuple     =		TFAutoModel.from_pretrained('''hf-internal-testing/tiny-random-bert'''      )
													self.assertEqual(counter.get_request_count			,   0      )
													self.assertEqual(counter.head_request_count			,   1      )
													self.assertEqual(counter.other_request_count			,   0      )

								# With a sharded checkpoint
								lowerCamelCase_					:      List[str]     =		TFAutoModel.from_pretrained('''ArthurZ/tiny-random-bert-sharded'''      )
								with RequestCounter() as counter:
													lowerCamelCase_					:      List[str]     =		TFAutoModel.from_pretrained('''ArthurZ/tiny-random-bert-sharded'''      )
													self.assertEqual(counter.get_request_count			,   0      )
													self.assertEqual(counter.head_request_count			,   1      )
													self.assertEqual(counter.other_request_count			,   0      )





'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available


__lowercase							:   List[Any]					=     {}


try:
					if not is_sentencepiece_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__lowercase							:   List[Any]					=     ['''MLukeTokenizer''']

if TYPE_CHECKING:
					try:
										if not is_sentencepiece_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tokenization_mluke import MLukeTokenizer


else:
					import sys

					__lowercase							:   List[str]					=     _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)


