"""simple docstring"""
from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy



@add_start_docstrings(
    """The RoBERTa Model transformer with early exiting (DeeRoBERTa). """     ,    __A     ,    )
class   lowerCAmelCase_    (			__A					):



				"""simple docstring"""
				_lowerCAmelCase     :    str			  =						RobertaConfig
				_lowerCAmelCase     :    Tuple			  =						"""roberta"""

				def __init__(				self		,    lowerCAmelCase    ):






										"""simple docstring"""

										super().__init__(lowerCAmelCase    )

										snake_case								=			RobertaEmbeddings(lowerCAmelCase    )
										self.init_weights()





@add_start_docstrings(
    """RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. """     ,    __A     ,    )
class   lowerCAmelCase_    (			__A					):



				"""simple docstring"""
				_lowerCAmelCase     :    Dict			  =						RobertaConfig
				_lowerCAmelCase     :    Dict			  =						"""roberta"""

				def __init__(				self		,    lowerCAmelCase    ):






										"""simple docstring"""

										super().__init__(lowerCAmelCase    )
										snake_case								=			config.num_labels
										snake_case								=			config.num_hidden_layers

										snake_case								=			DeeRobertaModel(lowerCAmelCase    )
										snake_case								=			nn.Dropout(config.hidden_dropout_prob    )
										snake_case								=			nn.Linear(config.hidden_size		,    self.config.num_labels    )

				@add_start_docstrings_to_model_forward(lowerCAmelCase    )
				def 		snake_case (				self		,    lowerCAmelCase=None		,    lowerCAmelCase=None		,    lowerCAmelCase=None		,    lowerCAmelCase=None		,    lowerCAmelCase=None		,    lowerCAmelCase=None		,    lowerCAmelCase=None		,    lowerCAmelCase=-1		,    lowerCAmelCase=False		,    ):






										"""simple docstring"""

										snake_case								=			self.num_layers
										try:
																snake_case								=			self.roberta(
																    lowerCAmelCase		,    attention_mask=lowerCAmelCase		,    token_type_ids=lowerCAmelCase		,    position_ids=lowerCAmelCase		,    head_mask=lowerCAmelCase		,    inputs_embeds=lowerCAmelCase		,    )

																snake_case								=			outputs[1]

																snake_case								=			self.dropout(lowerCAmelCase    )
																snake_case								=			self.classifier(lowerCAmelCase    )
																snake_case								=			(logits,) + outputs[2:]  # add hidden states and attention if they are here
										except HighwayException as e:
																snake_case								=			e.message
																snake_case								=			e.exit_layer
																snake_case								=			outputs[0]

										if not self.training:
																snake_case								=			entropy(lowerCAmelCase    )
																snake_case								=			[]
																snake_case								=			[]
										if labels is not None:
																if self.num_labels == 1:
																						#  We are doing regression
																						snake_case								=			MSELoss()
																						snake_case								=			loss_fct(logits.view(-1    )		,    labels.view(-1    )    )
																else:
																						snake_case								=			CrossEntropyLoss()
																						snake_case								=			loss_fct(logits.view(-1		,    self.num_labels    )		,    labels.view(-1    )    )

																# work with highway exits
																snake_case								=			[]
																for highway_exit in outputs[-1]:
																						snake_case								=			highway_exit[0]
																						if not self.training:
																												highway_logits_all.append(lowerCAmelCase    )
																												highway_entropy.append(highway_exit[2]    )
																						if self.num_labels == 1:
																												#  We are doing regression
																												snake_case								=			MSELoss()
																												snake_case								=			loss_fct(highway_logits.view(-1    )		,    labels.view(-1    )    )
																						else:
																												snake_case								=			CrossEntropyLoss()
																												snake_case								=			loss_fct(highway_logits.view(-1		,    self.num_labels    )		,    labels.view(-1    )    )
																						highway_losses.append(lowerCAmelCase    )

																if train_highway:
																						snake_case								=			(sum(highway_losses[:-1]    ),) + outputs
																						# exclude the final highway, of course
																else:
																						snake_case								=			(loss,) + outputs
										if not self.training:
																snake_case								=			outputs + ((original_entropy, highway_entropy), exit_layer)
																if output_layer >= 0:
																						snake_case								=			(
																						    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
																						)  # use the highway of the last layer

										return outputs  # (loss), logits, (hidden_states), (attentions), entropy





"""simple docstring"""
def    lowerCAmelCase__    (			_UpperCamelCase       :				int ,			_UpperCamelCase       :				int					)					->				int:





						"""simple docstring"""






						return int((input_a, input_a).count(0					) == 0					)





def    lowerCAmelCase__    (			)					->				None:





						"""simple docstring"""






						assert and_gate(0 ,			0					) == 0
						assert and_gate(0 ,			1					) == 0
						assert and_gate(1 ,			0					) == 0
						assert and_gate(1 ,			1					) == 1


if __name__ == "__main__":
					test_and_gate()
					print(and_gate(1, 0))
					print(and_gate(0, 0))
					print(and_gate(0, 1))
					print(and_gate(1, 1))


