



from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import tensorflow as tf

				from transformers import (
				    TFConvBertForMaskedLM,
				    TFConvBertForMultipleChoice,
				    TFConvBertForQuestionAnswering,
				    TFConvBertForSequenceClassification,
				    TFConvBertForTokenClassification,
				    TFConvBertModel,
				)






class     __a			:



	def __init__(    self				:     Any					,						snake_case_				:     Optional[int]					,						snake_case_				:     Any=13					,						snake_case_				:     Dict=7					,						snake_case_				:     str=True					,						snake_case_				:     Tuple=True					,						snake_case_				:     List[str]=True					,						snake_case_				:     Union[str, Any]=True					,						snake_case_				:     str=99					,						snake_case_				:     str=32					,						snake_case_				:     List[str]=2					,						snake_case_				:     List[Any]=4					,						snake_case_				:     Any=37					,						snake_case_				:     int="gelu"					,						snake_case_				:     Optional[int]=0.1					,						snake_case_				:     Optional[int]=0.1					,						snake_case_				:     Tuple=5_12					,						snake_case_				:     int=16					,						snake_case_				:     Union[str, Any]=2					,						snake_case_				:     int=0.0_2					,						snake_case_				:     Union[str, Any]=3					,						snake_case_				:     int=4					,						snake_case_				:     List[Any]=None					,						)->				List[Any]:
						__lowerCAmelCase						 =parent
						__lowerCAmelCase						 =13
						__lowerCAmelCase						 =7
						__lowerCAmelCase						 =True
						__lowerCAmelCase						 =True
						__lowerCAmelCase						 =True
						__lowerCAmelCase						 =True
						__lowerCAmelCase						 =99
						__lowerCAmelCase						 =3_84
						__lowerCAmelCase						 =2
						__lowerCAmelCase						 =4
						__lowerCAmelCase						 =37
						__lowerCAmelCase						 ="""gelu"""
						__lowerCAmelCase						 =0.1
						__lowerCAmelCase						 =0.1
						__lowerCAmelCase						 =5_12
						__lowerCAmelCase						 =16
						__lowerCAmelCase						 =2
						__lowerCAmelCase						 =0.0_2
						__lowerCAmelCase						 =3
						__lowerCAmelCase						 =4
						__lowerCAmelCase						 =1_28
						__lowerCAmelCase						 =2
						__lowerCAmelCase						 =9
						__lowerCAmelCase						 =1
						__lowerCAmelCase						 =None



	def 		UpperCamelCase				(    self				:     Optional[Any])->				Dict:
						__lowerCAmelCase						 =ids_tensor([self.batch_size, self.seq_length]					,						self.vocab_size)

						__lowerCAmelCase						 =None
						if self.use_input_mask:
											__lowerCAmelCase						 =random_attention_mask([self.batch_size, self.seq_length])

						__lowerCAmelCase						 =None
						if self.use_token_type_ids:
											__lowerCAmelCase						 =ids_tensor([self.batch_size, self.seq_length]					,						self.type_vocab_size)

						__lowerCAmelCase						 =None
						__lowerCAmelCase						 =None
						__lowerCAmelCase						 =None
						if self.use_labels:
											__lowerCAmelCase						 =ids_tensor([self.batch_size]					,						self.type_sequence_label_size)
											__lowerCAmelCase						 =ids_tensor([self.batch_size, self.seq_length]					,						self.num_labels)
											__lowerCAmelCase						 =ids_tensor([self.batch_size]					,						self.num_choices)

						__lowerCAmelCase						 =ConvBertConfig(
						    vocab_size=self.vocab_size					,						hidden_size=self.hidden_size					,						num_hidden_layers=self.num_hidden_layers					,						num_attention_heads=self.num_attention_heads					,						intermediate_size=self.intermediate_size					,						hidden_act=self.hidden_act					,						hidden_dropout_prob=self.hidden_dropout_prob					,						attention_probs_dropout_prob=self.attention_probs_dropout_prob					,						max_position_embeddings=self.max_position_embeddings					,						type_vocab_size=self.type_vocab_size					,						initializer_range=self.initializer_range					,						return_dict=snake_case_					,						)

						return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



	def 		UpperCamelCase				(    self				:     int					,						snake_case_				:     Dict					,						snake_case_				:     Any					,						snake_case_				:     Union[str, Any]					,						snake_case_				:     Optional[int]					,						snake_case_				:     Any					,						snake_case_				:     int					,						snake_case_				:     Union[str, Any])->				Optional[Any]:
						__lowerCAmelCase						 =TFConvBertModel(config=snake_case_)
						__lowerCAmelCase						 ={"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}

						__lowerCAmelCase						 =[input_ids, input_mask]
						__lowerCAmelCase						 =model(snake_case_)

						__lowerCAmelCase						 =model(snake_case_)

						self.parent.assertEqual(result.last_hidden_state.shape					,						(self.batch_size, self.seq_length, self.hidden_size))



	def 		UpperCamelCase				(    self				:     List[str]					,						snake_case_				:     Optional[int]					,						snake_case_				:     Optional[Any]					,						snake_case_				:     Tuple					,						snake_case_				:     Dict					,						snake_case_				:     List[str]					,						snake_case_				:     List[Any]					,						snake_case_				:     List[Any])->				str:
						__lowerCAmelCase						 =TFConvBertForMaskedLM(config=snake_case_)
						__lowerCAmelCase						 ={
						    """input_ids""": input_ids,
						    """attention_mask""": input_mask,
						    """token_type_ids""": token_type_ids,
						}
						__lowerCAmelCase						 =model(snake_case_)
						self.parent.assertEqual(result.logits.shape					,						(self.batch_size, self.seq_length, self.vocab_size))



	def 		UpperCamelCase				(    self				:     str					,						snake_case_				:     Any					,						snake_case_				:     Union[str, Any]					,						snake_case_				:     int					,						snake_case_				:     List[str]					,						snake_case_				:     int					,						snake_case_				:     Tuple					,						snake_case_				:     Any)->				Dict:
						__lowerCAmelCase						 =self.num_labels
						__lowerCAmelCase						 =TFConvBertForSequenceClassification(config=snake_case_)
						__lowerCAmelCase						 ={
						    """input_ids""": input_ids,
						    """attention_mask""": input_mask,
						    """token_type_ids""": token_type_ids,
						}

						__lowerCAmelCase						 =model(snake_case_)
						self.parent.assertEqual(result.logits.shape					,						(self.batch_size, self.num_labels))



	def 		UpperCamelCase				(    self				:     List[str]					,						snake_case_				:     List[Any]					,						snake_case_				:     Optional[Any]					,						snake_case_				:     List[Any]					,						snake_case_				:     int					,						snake_case_				:     Union[str, Any]					,						snake_case_				:     Any					,						snake_case_				:     int)->				int:
						__lowerCAmelCase						 =self.num_choices
						__lowerCAmelCase						 =TFConvBertForMultipleChoice(config=snake_case_)
						__lowerCAmelCase						 =tf.tile(tf.expand_dims(snake_case_					,						1)					,						(1, self.num_choices, 1))
						__lowerCAmelCase						 =tf.tile(tf.expand_dims(snake_case_					,						1)					,						(1, self.num_choices, 1))
						__lowerCAmelCase						 =tf.tile(tf.expand_dims(snake_case_					,						1)					,						(1, self.num_choices, 1))
						__lowerCAmelCase						 ={
						    """input_ids""": multiple_choice_inputs_ids,
						    """attention_mask""": multiple_choice_input_mask,
						    """token_type_ids""": multiple_choice_token_type_ids,
						}
						__lowerCAmelCase						 =model(snake_case_)
						self.parent.assertEqual(result.logits.shape					,						(self.batch_size, self.num_choices))



	def 		UpperCamelCase				(    self				:     Dict					,						snake_case_				:     Optional[int]					,						snake_case_				:     Union[str, Any]					,						snake_case_				:     Optional[Any]					,						snake_case_				:     int					,						snake_case_				:     Optional[Any]					,						snake_case_				:     Union[str, Any]					,						snake_case_				:     Dict)->				Tuple:
						__lowerCAmelCase						 =self.num_labels
						__lowerCAmelCase						 =TFConvBertForTokenClassification(config=snake_case_)
						__lowerCAmelCase						 ={
						    """input_ids""": input_ids,
						    """attention_mask""": input_mask,
						    """token_type_ids""": token_type_ids,
						}
						__lowerCAmelCase						 =model(snake_case_)
						self.parent.assertEqual(result.logits.shape					,						(self.batch_size, self.seq_length, self.num_labels))



	def 		UpperCamelCase				(    self				:     List[Any]					,						snake_case_				:     Optional[Any]					,						snake_case_				:     Tuple					,						snake_case_				:     Optional[int]					,						snake_case_				:     List[str]					,						snake_case_				:     Any					,						snake_case_				:     Tuple					,						snake_case_				:     List[Any])->				Dict:
						__lowerCAmelCase						 =TFConvBertForQuestionAnswering(config=snake_case_)
						__lowerCAmelCase						 ={
						    """input_ids""": input_ids,
						    """attention_mask""": input_mask,
						    """token_type_ids""": token_type_ids,
						}

						__lowerCAmelCase						 =model(snake_case_)
						self.parent.assertEqual(result.start_logits.shape					,						(self.batch_size, self.seq_length))
						self.parent.assertEqual(result.end_logits.shape					,						(self.batch_size, self.seq_length))



	def 		UpperCamelCase				(    self				:     Optional[Any])->				List[str]:
						__lowerCAmelCase						 =self.prepare_config_and_inputs()
						(
						    (
						    __lowerCAmelCase
						) ,						(
						    __lowerCAmelCase
						) ,						(
						    __lowerCAmelCase
						) ,						(
						    __lowerCAmelCase
						) ,						(
						    __lowerCAmelCase
						) ,						(
						    __lowerCAmelCase
						) ,						(
						    __lowerCAmelCase
						) ,						
						)						 =config_and_inputs
						__lowerCAmelCase						 ={"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
						return config, inputs_dict






@require_tf
class     __a			(  SCREAMING_SNAKE_CASE						,     SCREAMING_SNAKE_CASE						,     unittest.TestCase					):
	SCREAMING_SNAKE_CASE  	=							(
	    (
	        TFConvBertModel,
	        TFConvBertForMaskedLM,
	        TFConvBertForQuestionAnswering,
	        TFConvBertForSequenceClassification,
	        TFConvBertForTokenClassification,
	        TFConvBertForMultipleChoice,
	    )
	    if is_tf_available()
	    else ()
	)
	SCREAMING_SNAKE_CASE  	=							(
	    {
	        "feature-extraction": TFConvBertModel,
	        "fill-mask": TFConvBertForMaskedLM,
	        "question-answering": TFConvBertForQuestionAnswering,
	        "text-classification": TFConvBertForSequenceClassification,
	        "token-classification": TFConvBertForTokenClassification,
	        "zero-shot": TFConvBertForSequenceClassification,
	    }
	    if is_tf_available()
	    else {}
	)
	SCREAMING_SNAKE_CASE  	=							False
	SCREAMING_SNAKE_CASE  	=							False
	SCREAMING_SNAKE_CASE  	=							False



	def 		UpperCamelCase				(    self				:     Tuple)->				List[Any]:
						__lowerCAmelCase						 =TFConvBertModelTester(self)
						__lowerCAmelCase						 =ConfigTester(self					,						config_class=snake_case_					,						hidden_size=37)



	def 		UpperCamelCase				(    self				:     Dict)->				int:
						self.config_tester.run_common_tests()



	def 		UpperCamelCase				(    self				:     Union[str, Any])->				Union[str, Any]:
						__lowerCAmelCase						 =self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*snake_case_)



	def 		UpperCamelCase				(    self				:     Optional[int])->				Union[str, Any]:
						__lowerCAmelCase						 =self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_masked_lm(*snake_case_)



	def 		UpperCamelCase				(    self				:     Optional[Any])->				Tuple:
						__lowerCAmelCase						 =self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_multiple_choice(*snake_case_)



	def 		UpperCamelCase				(    self				:     Dict)->				str:
						__lowerCAmelCase						 =self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_question_answering(*snake_case_)



	def 		UpperCamelCase				(    self				:     List[Any])->				Any:
						__lowerCAmelCase						 =self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_sequence_classification(*snake_case_)



	def 		UpperCamelCase				(    self				:     Optional[Any])->				Tuple:
						__lowerCAmelCase						 =self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_token_classification(*snake_case_)



	@slow
	def 		UpperCamelCase				(    self				:     Any)->				Optional[Any]:
						__lowerCAmelCase ,						__lowerCAmelCase						 =self.model_tester.prepare_config_and_inputs_for_common()
						__lowerCAmelCase						 =True
						__lowerCAmelCase						 =True

						if hasattr(snake_case_					,						"""use_cache"""):
											__lowerCAmelCase						 =True

						__lowerCAmelCase						 =getattr(self.model_tester					,						"""encoder_seq_length"""					,						self.model_tester.seq_length)
						__lowerCAmelCase						 =getattr(self.model_tester					,						"""key_length"""					,						snake_case_)

						for model_class in self.all_model_classes:
											__lowerCAmelCase						 =self._prepare_for_class(snake_case_					,						snake_case_)
											__lowerCAmelCase						 =model_class(snake_case_)
											__lowerCAmelCase						 =len(model(snake_case_))

											with tempfile.TemporaryDirectory() as tmpdirname:
																model.save_pretrained(snake_case_					,						saved_model=snake_case_)
																__lowerCAmelCase						 =os.path.join(snake_case_					,						"""saved_model"""					,						"""1""")
																__lowerCAmelCase						 =tf.keras.models.load_model(snake_case_)
																__lowerCAmelCase						 =model(snake_case_)

																if self.is_encoder_decoder:
																					__lowerCAmelCase						 =outputs["""encoder_hidden_states"""]
																					__lowerCAmelCase						 =outputs["""encoder_attentions"""]
																else:
																					__lowerCAmelCase						 =outputs["""hidden_states"""]
																					__lowerCAmelCase						 =outputs["""attentions"""]

																self.assertEqual(len(snake_case_)					,						snake_case_)

																__lowerCAmelCase						 =getattr(
																    self.model_tester					,						"""expected_num_hidden_layers"""					,						self.model_tester.num_hidden_layers + 1)

																self.assertEqual(len(snake_case_)					,						snake_case_)
																self.assertListEqual(
																    list(output_hidden_states[0].shape[-2:])					,						[self.model_tester.seq_length, self.model_tester.hidden_size]					,						)

																self.assertEqual(len(snake_case_)					,						self.model_tester.num_hidden_layers)
																self.assertListEqual(
																    list(output_attentions[0].shape[-3:])					,						[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]					,						)



	@slow
	def 		UpperCamelCase				(    self				:     Dict)->				Tuple:
						__lowerCAmelCase						 =TFConvBertModel.from_pretrained("""YituTech/conv-bert-base""")
						self.assertIsNotNone(snake_case_)



	def 		UpperCamelCase				(    self				:     str)->				int:
						__lowerCAmelCase ,						__lowerCAmelCase						 =self.model_tester.prepare_config_and_inputs_for_common()
						__lowerCAmelCase						 =True
						__lowerCAmelCase						 =getattr(self.model_tester					,						"""decoder_seq_length"""					,						self.model_tester.seq_length)
						__lowerCAmelCase						 =getattr(self.model_tester					,						"""encoder_seq_length"""					,						self.model_tester.seq_length)
						__lowerCAmelCase						 =getattr(self.model_tester					,						"""key_length"""					,						snake_case_)
						__lowerCAmelCase						 =getattr(self.model_tester					,						"""key_length"""					,						snake_case_)

						def check_decoder_attentions_output(snake_case_				:     Union[str, Any]):
											__lowerCAmelCase						 =len(snake_case_)
											self.assertEqual(out_len % 2					,						0)
											__lowerCAmelCase						 =outputs.decoder_attentions
											self.assertEqual(len(snake_case_)					,						self.model_tester.num_hidden_layers)
											self.assertListEqual(
											    list(decoder_attentions[0].shape[-3:])					,						[self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length]					,						)

						def check_encoder_attentions_output(snake_case_				:     List[Any]):
											__lowerCAmelCase						 =[
											    t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
											]
											self.assertEqual(len(snake_case_)					,						self.model_tester.num_hidden_layers)
											self.assertListEqual(
											    list(attentions[0].shape[-3:])					,						[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]					,						)

						for model_class in self.all_model_classes:
											__lowerCAmelCase						 =True
											__lowerCAmelCase						 =False
											__lowerCAmelCase						 =model_class(snake_case_)
											__lowerCAmelCase						 =model(self._prepare_for_class(snake_case_					,						snake_case_))
											__lowerCAmelCase						 =len(snake_case_)
											self.assertEqual(config.output_hidden_states					,						snake_case_)
											check_encoder_attentions_output(snake_case_)

											if self.is_encoder_decoder:
																__lowerCAmelCase						 =model_class(snake_case_)
																__lowerCAmelCase						 =model(self._prepare_for_class(snake_case_					,						snake_case_))
																self.assertEqual(config.output_hidden_states					,						snake_case_)
																check_decoder_attentions_output(snake_case_)

											# Check that output attentions can also be changed via the config
											del inputs_dict["output_attentions"]
											__lowerCAmelCase						 =True
											__lowerCAmelCase						 =model_class(snake_case_)
											__lowerCAmelCase						 =model(self._prepare_for_class(snake_case_					,						snake_case_))
											self.assertEqual(config.output_hidden_states					,						snake_case_)
											check_encoder_attentions_output(snake_case_)

											# Check attention is always last and order is fine
											__lowerCAmelCase						 =True
											__lowerCAmelCase						 =True
											__lowerCAmelCase						 =model_class(snake_case_)
											__lowerCAmelCase						 =model(self._prepare_for_class(snake_case_					,						snake_case_))

											self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)					,						len(snake_case_))
											self.assertEqual(model.config.output_hidden_states					,						snake_case_)
											check_encoder_attentions_output(snake_case_)






@require_tf
class     __a			(  unittest.TestCase					):



	@slow
	def 		UpperCamelCase				(    self				:     str)->				Dict:
						__lowerCAmelCase						 =TFConvBertModel.from_pretrained("""YituTech/conv-bert-base""")
						__lowerCAmelCase						 =tf.constant([[0, 1, 2, 3, 4, 5]])
						__lowerCAmelCase						 =model(snake_case_)[0]

						__lowerCAmelCase						 =[1, 6, 7_68]
						self.assertEqual(output.shape					,						snake_case_)

						__lowerCAmelCase						 =tf.constant(
						    [
						        [
						            [-0.0_3_4_7_5_4_9_3, -0.4_6_8_6_0_3_4, -0.3_0_6_3_8_8_3_2],
						            [0.2_2_6_3_7_2_4_8, -0.2_6_9_8_8_6_4_6, -0.7_4_2_3_4_2_4],
						            [0.1_0_3_2_4_8_6_8, -0.4_5_0_1_3_5_0_8, -0.5_8_2_8_0_7_8_4],
						        ]
						    ])
						tf.debugging.assert_near(output[:, :3, :3]					,						snake_case_					,						atol=1e-4)









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowercase_													=      {
    '''configuration_albert''': ['''ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''AlbertConfig''', '''AlbertOnnxConfig'''],
}

try:
				if not is_sentencepiece_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_													=      ['''AlbertTokenizer''']

try:
				if not is_tokenizers_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_													=      ['''AlbertTokenizerFast''']

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_													=      [
				    '''ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
				    '''AlbertForMaskedLM''',
				    '''AlbertForMultipleChoice''',
				    '''AlbertForPreTraining''',
				    '''AlbertForQuestionAnswering''',
				    '''AlbertForSequenceClassification''',
				    '''AlbertForTokenClassification''',
				    '''AlbertModel''',
				    '''AlbertPreTrainedModel''',
				    '''load_tf_weights_in_albert''',
				]

try:
				if not is_tf_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_													=      [
				    '''TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
				    '''TFAlbertForMaskedLM''',
				    '''TFAlbertForMultipleChoice''',
				    '''TFAlbertForPreTraining''',
				    '''TFAlbertForQuestionAnswering''',
				    '''TFAlbertForSequenceClassification''',
				    '''TFAlbertForTokenClassification''',
				    '''TFAlbertMainLayer''',
				    '''TFAlbertModel''',
				    '''TFAlbertPreTrainedModel''',
				]

try:
				if not is_flax_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_													=      [
				    '''FlaxAlbertForMaskedLM''',
				    '''FlaxAlbertForMultipleChoice''',
				    '''FlaxAlbertForPreTraining''',
				    '''FlaxAlbertForQuestionAnswering''',
				    '''FlaxAlbertForSequenceClassification''',
				    '''FlaxAlbertForTokenClassification''',
				    '''FlaxAlbertModel''',
				    '''FlaxAlbertPreTrainedModel''',
				]

if TYPE_CHECKING:
				from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig, AlbertOnnxConfig

				try:
								if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_albert import AlbertTokenizer

				try:
								if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_albert_fast import AlbertTokenizerFast

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_albert import (
								    ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    AlbertForMaskedLM,
								    AlbertForMultipleChoice,
								    AlbertForPreTraining,
								    AlbertForQuestionAnswering,
								    AlbertForSequenceClassification,
								    AlbertForTokenClassification,
								    AlbertModel,
								    AlbertPreTrainedModel,
								    load_tf_weights_in_albert,
								)

				try:
								if not is_tf_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_tf_albert import (
								    TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    TFAlbertForMaskedLM,
								    TFAlbertForMultipleChoice,
								    TFAlbertForPreTraining,
								    TFAlbertForQuestionAnswering,
								    TFAlbertForSequenceClassification,
								    TFAlbertForTokenClassification,
								    TFAlbertMainLayer,
								    TFAlbertModel,
								    TFAlbertPreTrainedModel,
								)

				try:
								if not is_flax_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_flax_albert import (
								    FlaxAlbertForMaskedLM,
								    FlaxAlbertForMultipleChoice,
								    FlaxAlbertForPreTraining,
								    FlaxAlbertForQuestionAnswering,
								    FlaxAlbertForSequenceClassification,
								    FlaxAlbertForTokenClassification,
								    FlaxAlbertModel,
								    FlaxAlbertPreTrainedModel,
								)
else:
				import sys

				lowercase_													=      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




