
from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy





@add_start_docstrings(
    '''The RoBERTa Model transformer with early exiting (DeeRoBERTa). '''      ,   __a      ,   )
class 			_lowerCAmelCase      (			__a ):
				_lowercase    	=RobertaConfig
				_lowercase    	='''roberta'''




				def __init__(      self       ,       _UpperCamelCase  )       ->   List[Any]:
							super().__init__(_UpperCamelCase  )

							lowerCAmelCase_      			=			RobertaEmbeddings(_UpperCamelCase  )
							self.init_weights()





@add_start_docstrings(
    '''RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. '''      ,   __a      ,   )
class 			_lowerCAmelCase      (			__a ):
				_lowercase    	=RobertaConfig
				_lowercase    	='''roberta'''




				def __init__(      self       ,       _UpperCamelCase  )       ->   Any:
							super().__init__(_UpperCamelCase  )
							lowerCAmelCase_      			=			config.num_labels
							lowerCAmelCase_      			=			config.num_hidden_layers

							lowerCAmelCase_      			=			DeeRobertaModel(_UpperCamelCase  )
							lowerCAmelCase_      			=			nn.Dropout(config.hidden_dropout_prob  )
							lowerCAmelCase_      			=			nn.Linear(config.hidden_size       ,       self.config.num_labels  )




				@add_start_docstrings_to_model_forward(_UpperCamelCase  )
				def   __a		(      self       ,       _UpperCamelCase=None       ,       _UpperCamelCase=None       ,       _UpperCamelCase=None       ,       _UpperCamelCase=None       ,       _UpperCamelCase=None       ,       _UpperCamelCase=None       ,       _UpperCamelCase=None       ,       _UpperCamelCase=-1       ,       _UpperCamelCase=False       ,       )       ->   str:
							lowerCAmelCase_      			=			self.num_layers
							try:
										lowerCAmelCase_      			=			self.roberta(
										    _UpperCamelCase       ,       attention_mask=_UpperCamelCase       ,       token_type_ids=_UpperCamelCase       ,       position_ids=_UpperCamelCase       ,       head_mask=_UpperCamelCase       ,       inputs_embeds=_UpperCamelCase       ,       )

										lowerCAmelCase_      			=			outputs[1]

										lowerCAmelCase_      			=			self.dropout(_UpperCamelCase  )
										lowerCAmelCase_      			=			self.classifier(_UpperCamelCase  )
										lowerCAmelCase_      			=			(logits,) + outputs[2:]  # add hidden states and attention if they are here
							except HighwayException as e:
										lowerCAmelCase_      			=			e.message
										lowerCAmelCase_      			=			e.exit_layer
										lowerCAmelCase_      			=			outputs[0]

							if not self.training:
										lowerCAmelCase_      			=			entropy(_UpperCamelCase  )
										lowerCAmelCase_      			=			[]
										lowerCAmelCase_      			=			[]
							if labels is not None:
										if self.num_labels == 1:
													#  We are doing regression
													lowerCAmelCase_      			=			MSELoss()
													lowerCAmelCase_      			=			loss_fct(logits.view(-1  )       ,       labels.view(-1  )  )
										else:
													lowerCAmelCase_      			=			CrossEntropyLoss()
													lowerCAmelCase_      			=			loss_fct(logits.view(-1       ,       self.num_labels  )       ,       labels.view(-1  )  )

										# work with highway exits
										lowerCAmelCase_      			=			[]
										for highway_exit in outputs[-1]:
													lowerCAmelCase_      			=			highway_exit[0]
													if not self.training:
																highway_logits_all.append(_UpperCamelCase  )
																highway_entropy.append(highway_exit[2]  )
													if self.num_labels == 1:
																#  We are doing regression
																lowerCAmelCase_      			=			MSELoss()
																lowerCAmelCase_      			=			loss_fct(highway_logits.view(-1  )       ,       labels.view(-1  )  )
													else:
																lowerCAmelCase_      			=			CrossEntropyLoss()
																lowerCAmelCase_      			=			loss_fct(highway_logits.view(-1       ,       self.num_labels  )       ,       labels.view(-1  )  )
													highway_losses.append(_UpperCamelCase  )

										if train_highway:
													lowerCAmelCase_      			=			(sum(highway_losses[:-1]  ),) + outputs
													# exclude the final highway, of course
										else:
													lowerCAmelCase_      			=			(loss,) + outputs
							if not self.training:
										lowerCAmelCase_      			=			outputs + ((original_entropy, highway_entropy), exit_layer)
										if output_layer >= 0:
													lowerCAmelCase_      			=			(
													    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
													)  # use the highway of the last layer

							return outputs  # (loss), logits, (hidden_states), (attentions), entropy





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


_A     =  {
    "configuration_wav2vec2": ["WAV_2_VEC_2_PRETRAINED_CONFIG_ARCHIVE_MAP", "Wav2Vec2Config"],
    "feature_extraction_wav2vec2": ["Wav2Vec2FeatureExtractor"],
    "processing_wav2vec2": ["Wav2Vec2Processor"],
    "tokenization_wav2vec2": ["Wav2Vec2CTCTokenizer", "Wav2Vec2Tokenizer"],
}


try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A     =  [
			    "WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "Wav2Vec2ForAudioFrameClassification",
			    "Wav2Vec2ForCTC",
			    "Wav2Vec2ForMaskedLM",
			    "Wav2Vec2ForPreTraining",
			    "Wav2Vec2ForSequenceClassification",
			    "Wav2Vec2ForXVector",
			    "Wav2Vec2Model",
			    "Wav2Vec2PreTrainedModel",
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A     =  [
			    "TF_WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "TFWav2Vec2ForCTC",
			    "TFWav2Vec2Model",
			    "TFWav2Vec2PreTrainedModel",
			    "TFWav2Vec2ForSequenceClassification",
			]

try:
			if not is_flax_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A     =  [
			    "FlaxWav2Vec2ForCTC",
			    "FlaxWav2Vec2ForPreTraining",
			    "FlaxWav2Vec2Model",
			    "FlaxWav2Vec2PreTrainedModel",
			]


if TYPE_CHECKING:
			from .configuration_wavaveca import WAV_2_VEC_2_PRETRAINED_CONFIG_ARCHIVE_MAP, WavaVecaConfig
			from .feature_extraction_wavaveca import WavaVecaFeatureExtractor
			from .processing_wavaveca import WavaVecaProcessor
			from .tokenization_wavaveca import WavaVecaCTCTokenizer, WavaVecaTokenizer

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_wavaveca import (
						    WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST,
						    WavaVecaForAudioFrameClassification,
						    WavaVecaForCTC,
						    WavaVecaForMaskedLM,
						    WavaVecaForPreTraining,
						    WavaVecaForSequenceClassification,
						    WavaVecaForXVector,
						    WavaVecaModel,
						    WavaVecaPreTrainedModel,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_wavaveca import (
						    TF_WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFWavaVecaForCTC,
						    TFWavaVecaForSequenceClassification,
						    TFWavaVecaModel,
						    TFWavaVecaPreTrainedModel,
						)

			try:
						if not is_flax_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_wavaveca import (
						    FlaxWavaVecaForCTC,
						    FlaxWavaVecaForPreTraining,
						    FlaxWavaVecaModel,
						    FlaxWavaVecaPreTrainedModel,
						)


else:
			import sys

			_A     =  _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



