

import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    MODEL_FOR_PRETRAINING_MAPPING,
			    AlbertForMaskedLM,
			    AlbertForMultipleChoice,
			    AlbertForPreTraining,
			    AlbertForQuestionAnswering,
			    AlbertForSequenceClassification,
			    AlbertForTokenClassification,
			    AlbertModel,
			)
			from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST






class      lowerCamelCase_      :




	def __init__(     self						:     str					,      __A						:     Tuple					,      __A						:     Tuple=13					,      __A						:     List[str]=7					,      __A						:     List[str]=True					,      __A						:     Any=True					,      __A						:     List[str]=True					,      __A						:     Optional[int]=True					,      __A						:     Union[str, Any]=99					,      __A						:     str=16					,      __A						:     Optional[Any]=36					,      __A						:     Union[str, Any]=6					,      __A						:     Optional[int]=6					,      __A						:     Optional[int]=6					,      __A						:     List[str]=37					,      __A						:     Optional[int]="gelu"					,      __A						:     List[str]=0.1					,      __A						:     List[str]=0.1					,      __A						:     Tuple=512					,      __A						:     List[str]=16					,      __A						:     int=2					,      __A						:     Tuple=0.0_2					,      __A						:     Optional[int]=3					,      __A						:     List[Any]=4					,      __A						:     int=None					,      ):
		__A :			Optional[int]		    =			parent
		__A :			str		    =			batch_size
		__A :			List[Any]		    =			seq_length
		__A :			Union[str, Any]		    =			is_training
		__A :			Dict		    =			use_input_mask
		__A :			str		    =			use_token_type_ids
		__A :			List[Any]		    =			use_labels
		__A :			Optional[int]		    =			vocab_size
		__A :			Dict		    =			embedding_size
		__A :			Optional[int]		    =			hidden_size
		__A :			Optional[Any]		    =			num_hidden_layers
		__A :			Optional[Any]		    =			num_hidden_groups
		__A :			List[Any]		    =			num_attention_heads
		__A :			Optional[Any]		    =			intermediate_size
		__A :			List[Any]		    =			hidden_act
		__A :			Tuple		    =			hidden_dropout_prob
		__A :			str		    =			attention_probs_dropout_prob
		__A :			Dict		    =			max_position_embeddings
		__A :			Optional[int]		    =			type_vocab_size
		__A :			Dict		    =			type_sequence_label_size
		__A :			List[str]		    =			initializer_range
		__A :			Optional[int]		    =			num_labels
		__A :			Tuple		    =			num_choices
		__A :			Union[str, Any]		    =			scope




	def        lowerCAmelCase_       (     self						:     Union[str, Any]							):
		__A :			List[Any]		    =			ids_tensor([self.batch_size, self.seq_length]					,      self.vocab_size							)

		__A :			Any		    =			None
		if self.use_input_mask:
			__A :			Optional[Any]		    =			random_attention_mask([self.batch_size, self.seq_length]							)

		__A :			Dict		    =			None
		if self.use_token_type_ids:
			__A :			List[Any]		    =			ids_tensor([self.batch_size, self.seq_length]					,      self.type_vocab_size							)

		__A :			Optional[Any]		    =			None
		__A :			Dict		    =			None
		__A :			List[str]		    =			None
		if self.use_labels:
			__A :			Union[str, Any]		    =			ids_tensor([self.batch_size]					,      self.type_sequence_label_size							)
			__A :			Tuple		    =			ids_tensor([self.batch_size, self.seq_length]					,      self.num_labels							)
			__A :			Optional[Any]		    =			ids_tensor([self.batch_size]					,      self.num_choices							)

		__A :			Union[str, Any]		    =			self.get_config()

		return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




	def        lowerCAmelCase_       (     self						:     Dict							):
		return AlbertConfig(
		    vocab_size=self.vocab_size					,      hidden_size=self.hidden_size					,      num_hidden_layers=self.num_hidden_layers					,      num_attention_heads=self.num_attention_heads					,      intermediate_size=self.intermediate_size					,      hidden_act=self.hidden_act					,      hidden_dropout_prob=self.hidden_dropout_prob					,      attention_probs_dropout_prob=self.attention_probs_dropout_prob					,      max_position_embeddings=self.max_position_embeddings					,      type_vocab_size=self.type_vocab_size					,      initializer_range=self.initializer_range					,      num_hidden_groups=self.num_hidden_groups					,      )




	def        lowerCAmelCase_       (     self						:     int					,      __A						:     Dict					,      __A						:     Dict					,      __A						:     str					,      __A						:     Optional[int]					,      __A						:     Optional[Any]					,      __A						:     str					,      __A						:     int							):
		__A :			List[Any]		    =			AlbertModel(config=__A							)
		model.to(__A							)
		model.eval()
		__A :			int		    =			model(__A					,      attention_mask=__A					,      token_type_ids=__A							)
		__A :			Dict		    =			model(__A					,      token_type_ids=__A							)
		__A :			int		    =			model(__A							)
		self.parent.assertEqual(result.last_hidden_state.shape					,      (self.batch_size, self.seq_length, self.hidden_size)							)
		self.parent.assertEqual(result.pooler_output.shape					,      (self.batch_size, self.hidden_size)							)




	def        lowerCAmelCase_       (     self						:     Union[str, Any]					,      __A						:     str					,      __A						:     Tuple					,      __A						:     Union[str, Any]					,      __A						:     Any					,      __A						:     Union[str, Any]					,      __A						:     Tuple					,      __A						:     List[Any]							):
		__A :			int		    =			AlbertForPreTraining(config=__A							)
		model.to(__A							)
		model.eval()
		__A :			Any		    =			model(
		    __A					,      attention_mask=__A					,      token_type_ids=__A					,      labels=__A					,      sentence_order_label=__A					,      )
		self.parent.assertEqual(result.prediction_logits.shape					,      (self.batch_size, self.seq_length, self.vocab_size)							)
		self.parent.assertEqual(result.sop_logits.shape					,      (self.batch_size, config.num_labels)							)




	def        lowerCAmelCase_       (     self						:     List[Any]					,      __A						:     Dict					,      __A						:     int					,      __A						:     Optional[int]					,      __A						:     Optional[int]					,      __A						:     Any					,      __A						:     Dict					,      __A						:     Optional[int]							):
		__A :			List[str]		    =			AlbertForMaskedLM(config=__A							)
		model.to(__A							)
		model.eval()
		__A :			Tuple		    =			model(__A					,      attention_mask=__A					,      token_type_ids=__A					,      labels=__A							)
		self.parent.assertEqual(result.logits.shape					,      (self.batch_size, self.seq_length, self.vocab_size)							)




	def        lowerCAmelCase_       (     self						:     List[str]					,      __A						:     Optional[int]					,      __A						:     Any					,      __A						:     Union[str, Any]					,      __A						:     int					,      __A						:     Any					,      __A						:     Optional[Any]					,      __A						:     Union[str, Any]							):
		__A :			List[str]		    =			AlbertForQuestionAnswering(config=__A							)
		model.to(__A							)
		model.eval()
		__A :			str		    =			model(
		    __A					,      attention_mask=__A					,      token_type_ids=__A					,      start_positions=__A					,      end_positions=__A					,      )
		self.parent.assertEqual(result.start_logits.shape					,      (self.batch_size, self.seq_length)							)
		self.parent.assertEqual(result.end_logits.shape					,      (self.batch_size, self.seq_length)							)




	def        lowerCAmelCase_       (     self						:     Optional[int]					,      __A						:     Tuple					,      __A						:     str					,      __A						:     Tuple					,      __A						:     List[str]					,      __A						:     int					,      __A						:     List[Any]					,      __A						:     Optional[Any]							):
		__A :			Dict		    =			self.num_labels
		__A :			Optional[Any]		    =			AlbertForSequenceClassification(__A							)
		model.to(__A							)
		model.eval()
		__A :			str		    =			model(__A					,      attention_mask=__A					,      token_type_ids=__A					,      labels=__A							)
		self.parent.assertEqual(result.logits.shape					,      (self.batch_size, self.num_labels)							)




	def        lowerCAmelCase_       (     self						:     Any					,      __A						:     List[Any]					,      __A						:     Tuple					,      __A						:     Optional[Any]					,      __A						:     Optional[int]					,      __A						:     str					,      __A						:     Tuple					,      __A						:     str							):
		__A :			Any		    =			self.num_labels
		__A :			Optional[int]		    =			AlbertForTokenClassification(config=__A							)
		model.to(__A							)
		model.eval()
		__A :			Optional[int]		    =			model(__A					,      attention_mask=__A					,      token_type_ids=__A					,      labels=__A							)
		self.parent.assertEqual(result.logits.shape					,      (self.batch_size, self.seq_length, self.num_labels)							)




	def        lowerCAmelCase_       (     self						:     Optional[Any]					,      __A						:     List[Any]					,      __A						:     Any					,      __A						:     Optional[int]					,      __A						:     Optional[int]					,      __A						:     Union[str, Any]					,      __A						:     Dict					,      __A						:     int							):
		__A :			Dict		    =			self.num_choices
		__A :			List[str]		    =			AlbertForMultipleChoice(config=__A							)
		model.to(__A							)
		model.eval()
		__A :			List[str]		    =			input_ids.unsqueeze(1							).expand(-1					,      self.num_choices					,      -1							).contiguous()
		__A :			int		    =			token_type_ids.unsqueeze(1							).expand(-1					,      self.num_choices					,      -1							).contiguous()
		__A :			Optional[Any]		    =			input_mask.unsqueeze(1							).expand(-1					,      self.num_choices					,      -1							).contiguous()
		__A :			Union[str, Any]		    =			model(
		    __A					,      attention_mask=__A					,      token_type_ids=__A					,      labels=__A					,      )
		self.parent.assertEqual(result.logits.shape					,      (self.batch_size, self.num_choices)							)




	def        lowerCAmelCase_       (     self						:     Optional[Any]							):
		__A :			List[Any]		    =			self.prepare_config_and_inputs()
		(
		    (
		    __A
		)    ,				(
		    __A
		)    ,				(
		    __A
		)    ,				(
		    __A
		)    ,				(
		    __A
		)    ,				(
		    __A
		)    ,				(
		    __A
		)    ,				
		) :			List[str]		    =			config_and_inputs
		__A :			Any		    =			{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
		return config, inputs_dict






@require_torch
class      lowerCamelCase_      (	_lowercase						,      _lowercase						,      unittest.TestCase	):
	_lowercase		:   Optional[Any]     					=				(
	    (
	        AlbertModel,
	        AlbertForPreTraining,
	        AlbertForMaskedLM,
	        AlbertForMultipleChoice,
	        AlbertForSequenceClassification,
	        AlbertForTokenClassification,
	        AlbertForQuestionAnswering,
	    )
	    if is_torch_available()
	    else ()
	)
	_lowercase		:   List[str]     					=				(
	    {
	        '''feature-extraction''': AlbertModel,
	        '''fill-mask''': AlbertForMaskedLM,
	        '''question-answering''': AlbertForQuestionAnswering,
	        '''text-classification''': AlbertForSequenceClassification,
	        '''token-classification''': AlbertForTokenClassification,
	        '''zero-shot''': AlbertForSequenceClassification,
	    }
	    if is_torch_available()
	    else {}
	)
	_lowercase		:   str     					=				True




	def        lowerCAmelCase_       (     self						:     Any					,      __A						:     Optional[Any]					,      __A						:     Optional[Any]					,      __A						:     Tuple=False							):
		__A :			List[str]		    =			super()._prepare_for_class(__A					,      __A					,      return_labels=__A							)

		if return_labels:
			if model_class in get_values(__A							):
				__A :			Union[str, Any]		    =			torch.zeros(
				    (self.model_tester.batch_size, self.model_tester.seq_length)					,      dtype=torch.long					,      device=__A							)
				__A :			Any		    =			torch.zeros(
				    self.model_tester.batch_size					,      dtype=torch.long					,      device=__A							)
		return inputs_dict




	def        lowerCAmelCase_       (     self						:     Dict							):
		__A :			Optional[Any]		    =			AlbertModelTester(self							)
		__A :			Optional[int]		    =			ConfigTester(self					,      config_class=__A					,      hidden_size=37							)




	def        lowerCAmelCase_       (     self						:     List[str]							):
		self.config_tester.run_common_tests()




	def        lowerCAmelCase_       (     self						:     List[str]							):
		__A :			Optional[Any]		    =			self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_model(*__A							)




	def        lowerCAmelCase_       (     self						:     Optional[Any]							):
		__A :			Dict		    =			self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_for_pretraining(*__A							)




	def        lowerCAmelCase_       (     self						:     int							):
		__A :			Any		    =			self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_for_masked_lm(*__A							)




	def        lowerCAmelCase_       (     self						:     int							):
		__A :			Optional[int]		    =			self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_for_multiple_choice(*__A							)




	def        lowerCAmelCase_       (     self						:     Optional[int]							):
		__A :			Any		    =			self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_for_question_answering(*__A							)




	def        lowerCAmelCase_       (     self						:     Union[str, Any]							):
		__A :			List[Any]		    =			self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_for_sequence_classification(*__A							)




	def        lowerCAmelCase_       (     self						:     Tuple							):
		__A :			Any		    =			self.model_tester.prepare_config_and_inputs()
		for type in ["absolute", "relative_key", "relative_key_query"]:
			__A :			Optional[int]		    =			type
			self.model_tester.create_and_check_model(*__A							)




	@slow
	def        lowerCAmelCase_       (     self						:     int							):
		for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
			__A :			int		    =			AlbertModel.from_pretrained(__A							)
			self.assertIsNotNone(__A							)






@require_torch
class      lowerCamelCase_      (	unittest.TestCase	):




	@slow
	def        lowerCAmelCase_       (     self						:     List[str]							):
		__A :			Dict		    =			AlbertModel.from_pretrained("""albert-base-v2"""							)
		__A :			Union[str, Any]		    =			torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]							)
		__A :			List[Any]		    =			torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]							)
		with torch.no_grad():
			__A :			Any		    =			model(__A					,      attention_mask=__A							)[0]
		__A :			str		    =			torch.Size((1, 11, 768)							)
		self.assertEqual(output.shape					,      __A							)
		__A :			Any		    =			torch.tensor(
		    [[[-0.6_5_1_3, 1.5_0_3_5, -0.2_7_6_6], [-0.6_5_1_5, 1.5_0_4_6, -0.2_7_8_0], [-0.6_5_1_2, 1.5_0_4_9, -0.2_7_8_4]]]							)

		self.assertTrue(torch.allclose(output[:, 1:4, 1:4]					,      __A					,      atol=1e-4							)							)








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


UpperCAmelCase_							:					Optional[Any]		 =   {
    '''configuration_wav2vec2''': ['''WAV_2_VEC_2_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''Wav2Vec2Config'''],
    '''feature_extraction_wav2vec2''': ['''Wav2Vec2FeatureExtractor'''],
    '''processing_wav2vec2''': ['''Wav2Vec2Processor'''],
    '''tokenization_wav2vec2''': ['''Wav2Vec2CTCTokenizer''', '''Wav2Vec2Tokenizer'''],
}


try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			UpperCAmelCase_							:					Optional[Any]		 =   [
			    '''WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST''',
			    '''Wav2Vec2ForAudioFrameClassification''',
			    '''Wav2Vec2ForCTC''',
			    '''Wav2Vec2ForMaskedLM''',
			    '''Wav2Vec2ForPreTraining''',
			    '''Wav2Vec2ForSequenceClassification''',
			    '''Wav2Vec2ForXVector''',
			    '''Wav2Vec2Model''',
			    '''Wav2Vec2PreTrainedModel''',
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			UpperCAmelCase_							:					List[Any]		 =   [
			    '''TF_WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST''',
			    '''TFWav2Vec2ForCTC''',
			    '''TFWav2Vec2Model''',
			    '''TFWav2Vec2PreTrainedModel''',
			    '''TFWav2Vec2ForSequenceClassification''',
			]

try:
			if not is_flax_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			UpperCAmelCase_							:					Dict		 =   [
			    '''FlaxWav2Vec2ForCTC''',
			    '''FlaxWav2Vec2ForPreTraining''',
			    '''FlaxWav2Vec2Model''',
			    '''FlaxWav2Vec2PreTrainedModel''',
			]


if TYPE_CHECKING:
			from .configuration_wavaveca import WAV_2_VEC_2_PRETRAINED_CONFIG_ARCHIVE_MAP, WavaVecaConfig
			from .feature_extraction_wavaveca import WavaVecaFeatureExtractor
			from .processing_wavaveca import WavaVecaProcessor
			from .tokenization_wavaveca import WavaVecaCTCTokenizer, WavaVecaTokenizer

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_wavaveca import (
						    WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST,
						    WavaVecaForAudioFrameClassification,
						    WavaVecaForCTC,
						    WavaVecaForMaskedLM,
						    WavaVecaForPreTraining,
						    WavaVecaForSequenceClassification,
						    WavaVecaForXVector,
						    WavaVecaModel,
						    WavaVecaPreTrainedModel,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_wavaveca import (
						    TF_WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFWavaVecaForCTC,
						    TFWavaVecaForSequenceClassification,
						    TFWavaVecaModel,
						    TFWavaVecaPreTrainedModel,
						)

			try:
						if not is_flax_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_wavaveca import (
						    FlaxWavaVecaForCTC,
						    FlaxWavaVecaForPreTraining,
						    FlaxWavaVecaModel,
						    FlaxWavaVecaPreTrainedModel,
						)


else:
			import sys

			UpperCAmelCase_							:					Optional[Any]		 =   _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





