


'''simple docstring'''
from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy
@add_start_docstrings(
    'The RoBERTa Model transformer with early exiting (DeeRoBERTa). '      ,						A__      ,						)
class   lowercase      (				A__   ):

		"""simple docstring"""

		_a       					=							RobertaConfig
		_a       					=							'''roberta'''




		def __init__(						self       ,  UpperCamelCase_				):

							'''simple docstring'''


							super().__init__(A__				)

							UpperCamelCase__  :Dict				   =					RobertaEmbeddings(A__				)
							self.init_weights()



@add_start_docstrings(
    'RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. '      ,						A__      ,						)
class   lowercase      (				A__   ):

		"""simple docstring"""

		_a       					=							RobertaConfig
		_a       					=							'''roberta'''




		def __init__(						self       ,  UpperCamelCase_				):

							'''simple docstring'''


							super().__init__(A__				)
							UpperCamelCase__  :Dict				   =					config.num_labels
							UpperCamelCase__  :Optional[Any]				   =					config.num_hidden_layers

							UpperCamelCase__  :Tuple				   =					DeeRobertaModel(A__				)
							UpperCamelCase__  :str				   =					nn.Dropout(config.hidden_dropout_prob				)
							UpperCamelCase__  :int				   =					nn.Linear(config.hidden_size       ,  self.config.num_labels				)




		@add_start_docstrings_to_model_forward(A__				)
		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_=None       ,  UpperCamelCase_=None       ,  UpperCamelCase_=None       ,  UpperCamelCase_=None       ,  UpperCamelCase_=None       ,  UpperCamelCase_=None       ,  UpperCamelCase_=None       ,  UpperCamelCase_=-1       ,  UpperCamelCase_=False       ,  ):

							'''simple docstring'''


							UpperCamelCase__  :Optional[Any]				   =					self.num_layers
							try:
												UpperCamelCase__  :Optional[Any]				   =					self.roberta(
												    A__       ,  attention_mask=A__       ,  token_type_ids=A__       ,  position_ids=A__       ,  head_mask=A__       ,  inputs_embeds=A__       ,  )

												UpperCamelCase__  :Union[str, Any]				   =					outputs[1]

												UpperCamelCase__  :Union[str, Any]				   =					self.dropout(A__				)
												UpperCamelCase__  :Optional[int]				   =					self.classifier(A__				)
												UpperCamelCase__  :Union[str, Any]				   =					(logits,) + outputs[2:]  # add hidden states and attention if they are here
							except HighwayException as e:
												UpperCamelCase__  :Any				   =					e.message
												UpperCamelCase__  :str				   =					e.exit_layer
												UpperCamelCase__  :Dict				   =					outputs[0]

							if not self.training:
												UpperCamelCase__  :List[str]				   =					entropy(A__				)
												UpperCamelCase__  :Union[str, Any]				   =					[]
												UpperCamelCase__  :Any				   =					[]
							if labels is not None:
												if self.num_labels == 1:
																	#  We are doing regression
																	UpperCamelCase__  :Any				   =					MSELoss()
																	UpperCamelCase__  :Any				   =					loss_fct(logits.view(-1				)       ,  labels.view(-1				)				)
												else:
																	UpperCamelCase__  :Union[str, Any]				   =					CrossEntropyLoss()
																	UpperCamelCase__  :Dict				   =					loss_fct(logits.view(-1       ,  self.num_labels				)       ,  labels.view(-1				)				)

												# work with highway exits
												UpperCamelCase__  :Dict				   =					[]
												for highway_exit in outputs[-1]:
																	UpperCamelCase__  :Optional[int]				   =					highway_exit[0]
																	if not self.training:
																						highway_logits_all.append(A__				)
																						highway_entropy.append(highway_exit[2]				)
																	if self.num_labels == 1:
																						#  We are doing regression
																						UpperCamelCase__  :Union[str, Any]				   =					MSELoss()
																						UpperCamelCase__  :Any				   =					loss_fct(highway_logits.view(-1				)       ,  labels.view(-1				)				)
																	else:
																						UpperCamelCase__  :Any				   =					CrossEntropyLoss()
																						UpperCamelCase__  :List[Any]				   =					loss_fct(highway_logits.view(-1       ,  self.num_labels				)       ,  labels.view(-1				)				)
																	highway_losses.append(A__				)

												if train_highway:
																	UpperCamelCase__  :int				   =					(sum(highway_losses[:-1]				),) + outputs
																	# exclude the final highway, of course
												else:
																	UpperCamelCase__  :str				   =					(loss,) + outputs
							if not self.training:
												UpperCamelCase__  :Union[str, Any]				   =					outputs + ((original_entropy, highway_entropy), exit_layer)
												if output_layer >= 0:
																	UpperCamelCase__  :List[str]				   =					(
																	    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
																	)  # use the highway of the last layer

							return outputs  # (loss), logits, (hidden_states), (attentions), entropy


import tensorflow as tf

from ...tf_utils import shape_list



class        _a		(tf.keras.layers.Layer							):


					'''simple docstring'''






					def __init__(					self       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__=1       ,			A__=False       ,			**A__     ):
						super().__init__(**A__     )

						A__				:       Tuple	  =						vocab_size
						A__				:       List[str]	  =						d_embed
						A__				:       int	  =						d_proj

						A__				:       Tuple	  =						cutoffs + [vocab_size]
						A__				:       Any	  =						[0] + self.cutoffs
						A__				:       str	  =						div_val

						A__				:       int	  =						self.cutoffs[0]
						A__				:       List[str]	  =						len(self.cutoffs     ) - 1
						A__				:       Optional[int]	  =						self.shortlist_size + self.n_clusters
						A__				:       List[Any]	  =						keep_order

						A__				:       Optional[int]	  =						[]
						A__				:       Optional[Any]	  =						[]






					def     __A     (					self       ,			A__     ):
						if self.n_clusters > 0:
							A__				:       List[str]	  =						self.add_weight(
							    shape=(self.n_clusters, self.d_embed)       ,			initializer="""zeros"""       ,			trainable=A__       ,			name="""cluster_weight"""     )
							A__				:       Union[str, Any]	  =						self.add_weight(
							    shape=(self.n_clusters,)       ,			initializer="""zeros"""       ,			trainable=A__       ,			name="""cluster_bias"""     )

						if self.div_val == 1:
							for i in range(len(self.cutoffs     )     ):
								if self.d_proj != self.d_embed:
									A__				:       List[str]	  =						self.add_weight(
									    shape=(self.d_embed, self.d_proj)       ,			initializer="""zeros"""       ,			trainable=A__       ,			name=F"""out_projs_._{i}"""       ,			)
									self.out_projs.append(A__     )
								else:
									self.out_projs.append(A__     )
								A__				:       int	  =						self.add_weight(
								    shape=(self.vocab_size, self.d_embed)       ,			initializer="""zeros"""       ,			trainable=A__       ,			name=F"""out_layers_._{i}_._weight"""       ,			)
								A__				:       str	  =						self.add_weight(
								    shape=(self.vocab_size,)       ,			initializer="""zeros"""       ,			trainable=A__       ,			name=F"""out_layers_._{i}_._bias"""       ,			)
								self.out_layers.append((weight, bias)     )
						else:
							for i in range(len(self.cutoffs     )     ):
								A__  ,  A__				:       Optional[int]	  =						self.cutoff_ends[i], self.cutoff_ends[i + 1]
								A__				:       Optional[int]	  =						self.d_embed // (self.div_val**i)

								A__				:       Optional[Any]	  =						self.add_weight(
								    shape=(d_emb_i, self.d_proj)       ,			initializer="""zeros"""       ,			trainable=A__       ,			name=F"""out_projs_._{i}"""     )
								self.out_projs.append(A__     )
								A__				:       List[str]	  =						self.add_weight(
								    shape=(r_idx - l_idx, d_emb_i)       ,			initializer="""zeros"""       ,			trainable=A__       ,			name=F"""out_layers_._{i}_._weight"""       ,			)
								A__				:       Optional[int]	  =						self.add_weight(
								    shape=(r_idx - l_idx,)       ,			initializer="""zeros"""       ,			trainable=A__       ,			name=F"""out_layers_._{i}_._bias"""       ,			)
								self.out_layers.append((weight, bias)     )
						super().build(A__     )






					@staticmethod
					def     __A     (					A__       ,			A__       ,			A__       ,			A__=None     ):
						A__				:       Tuple	  =						x
						if proj is not None:
							A__				:       Tuple	  =						tf.einsum("""ibd,ed->ibe"""       ,			A__       ,			A__     )
						return tf.einsum("""ibd,nd->ibn"""       ,			A__       ,			A__     ) + b






					@staticmethod
					def     __A     (					A__       ,			A__     ):
						A__				:       Optional[Any]	  =						shape_list(A__     )
						A__				:       Optional[Any]	  =						tf.range(lp_size[0]       ,			dtype=target.dtype     )
						A__				:       List[str]	  =						tf.stack([r, target]       ,			1     )
						return tf.gather_nd(A__       ,			A__     )






					def     __A     (					self       ,			A__       ,			A__       ,			A__=True       ,			A__=False     ):
						A__				:       str	  =						0
						if self.n_clusters == 0:
							A__				:       List[str]	  =						self._logit(A__       ,			self.out_layers[0][0]       ,			self.out_layers[0][1]       ,			self.out_projs[0]     )
							if target is not None:
								A__				:       Union[str, Any]	  =						tf.nn.sparse_softmax_cross_entropy_with_logits(labels=A__       ,			logits=A__     )
							A__				:       int	  =						tf.nn.log_softmax(A__       ,			axis=-1     )
						else:
							A__				:       Any	  =						shape_list(A__     )
							A__				:       int	  =						[]
							A__				:       Union[str, Any]	  =						tf.zeros(hidden_sizes[:2]     )
							for i in range(len(self.cutoffs     )     ):
								A__  ,  A__				:       Union[str, Any]	  =						self.cutoff_ends[i], self.cutoff_ends[i + 1]
								if target is not None:
									A__				:       Any	  =						(target >= l_idx) & (target < r_idx)
									A__				:       Optional[Any]	  =						tf.where(A__     )
									A__				:       int	  =						tf.boolean_mask(A__       ,			A__     ) - l_idx

								if self.div_val == 1:
									A__				:       Any	  =						self.out_layers[0][0][l_idx:r_idx]
									A__				:       List[Any]	  =						self.out_layers[0][1][l_idx:r_idx]
								else:
									A__				:       Optional[int]	  =						self.out_layers[i][0]
									A__				:       Tuple	  =						self.out_layers[i][1]

								if i == 0:
									A__				:       Union[str, Any]	  =						tf.concat([cur_W, self.cluster_weight]       ,			0     )
									A__				:       Optional[int]	  =						tf.concat([cur_b, self.cluster_bias]       ,			0     )

									A__				:       Any	  =						self._logit(A__       ,			A__       ,			A__       ,			self.out_projs[0]     )
									A__				:       Optional[Any]	  =						tf.nn.log_softmax(A__     )
									out.append(head_logprob[..., : self.cutoffs[0]]     )
									if target is not None:
										A__				:       Optional[Any]	  =						tf.boolean_mask(A__       ,			A__     )
										A__				:       List[str]	  =						self._gather_logprob(A__       ,			A__     )
								else:
									A__				:       str	  =						self._logit(A__       ,			A__       ,			A__       ,			self.out_projs[i]     )
									A__				:       Dict	  =						tf.nn.log_softmax(A__     )
									A__				:       List[str]	  =						self.cutoffs[0] + i - 1  # No probability for the head cluster
									A__				:       Optional[Any]	  =						head_logprob[..., cluster_prob_idx, None] + tail_logprob
									out.append(A__     )
									if target is not None:
										A__				:       Any	  =						tf.boolean_mask(A__       ,			A__     )
										A__				:       List[Any]	  =						tf.boolean_mask(A__       ,			A__     )
										A__				:       List[Any]	  =						self._gather_logprob(A__       ,			A__     )
										cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
								if target is not None:
									loss += tf.scatter_nd(A__       ,			-cur_logprob       ,			shape_list(A__     )     )
							A__				:       List[str]	  =						tf.concat(A__       ,			axis=-1     )

						if target is not None:
							if return_mean:
								A__				:       List[Any]	  =						tf.reduce_mean(A__     )
							# Add the training-time loss value to the layer using `self.add_loss()`.
							self.add_loss(A__     )

							# Log the loss as a metric (we could log arbitrary metrics,
							# including different metrics for training and inference.
							self.add_metric(A__       ,			name=self.name       ,			aggregation="""mean""" if return_mean else """"""     )

						return out



