




from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy







@add_start_docstrings(
    '''The RoBERTa Model transformer with early exiting (DeeRoBERTa). '''				, _UpperCamelCase				, )
class     A(		_UpperCamelCase  ):



 '''simple docstring'''


 UpperCamelCase		  =		RobertaConfig
 UpperCamelCase		  =		'roberta'

 def __init__(							self   :    Optional[int]		,				A_   :    List[Any]    )  ->      Dict:

       """simple docstring"""






       super().__init__(A_    )

       lowerCamelCase_							=       RobertaEmbeddings(A_    )
       self.init_weights()







@add_start_docstrings(
    '''RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. '''				, _UpperCamelCase				, )
class     A(		_UpperCamelCase  ):



 '''simple docstring'''


 UpperCamelCase		  =		RobertaConfig
 UpperCamelCase		  =		'roberta'

 def __init__(							self   :    Any		,				A_   :    Tuple    )  ->      Dict:

       """simple docstring"""






       super().__init__(A_    )
       lowerCamelCase_							=       config.num_labels
       lowerCamelCase_							=       config.num_hidden_layers

       lowerCamelCase_							=       DeeRobertaModel(A_    )
       lowerCamelCase_							=       nn.Dropout(config.hidden_dropout_prob    )
       lowerCamelCase_							=       nn.Linear(config.hidden_size		,				self.config.num_labels    )





 @add_start_docstrings_to_model_forward(A_    )
 def 							a__					(							self   :    Tuple		,				A_   :    Optional[int]=None		,				A_   :    str=None		,				A_   :    Optional[Any]=None		,				A_   :    Tuple=None		,				A_   :    int=None		,				A_   :    Any=None		,				A_   :    Dict=None		,				A_   :    Union[str, Any]=-1		,				A_   :    str=False		,				)  ->      List[str]:

       """simple docstring"""






       lowerCamelCase_							=       self.num_layers
       try:
             lowerCamelCase_							=       self.roberta(
                 A_		,				attention_mask=A_		,				token_type_ids=A_		,				position_ids=A_		,				head_mask=A_		,				inputs_embeds=A_		,				)

             lowerCamelCase_							=       outputs[1]

             lowerCamelCase_							=       self.dropout(A_    )
             lowerCamelCase_							=       self.classifier(A_    )
             lowerCamelCase_							=       (logits,) + outputs[2:]  # add hidden states and attention if they are here
       except HighwayException as e:
             lowerCamelCase_							=       e.message
             lowerCamelCase_							=       e.exit_layer
             lowerCamelCase_							=       outputs[0]

       if not self.training:
             lowerCamelCase_							=       entropy(A_    )
             lowerCamelCase_							=       []
             lowerCamelCase_							=       []
       if labels is not None:
             if self.num_labels == 1:
                   #  We are doing regression
                   lowerCamelCase_							=       MSELoss()
                   lowerCamelCase_							=       loss_fct(logits.view(-1    )		,				labels.view(-1    )    )
             else:
                   lowerCamelCase_							=       CrossEntropyLoss()
                   lowerCamelCase_							=       loss_fct(logits.view(-1		,				self.num_labels    )		,				labels.view(-1    )    )

             # work with highway exits
             lowerCamelCase_							=       []
             for highway_exit in outputs[-1]:
                   lowerCamelCase_							=       highway_exit[0]
                   if not self.training:
                         highway_logits_all.append(A_    )
                         highway_entropy.append(highway_exit[2]    )
                   if self.num_labels == 1:
                         #  We are doing regression
                         lowerCamelCase_							=       MSELoss()
                         lowerCamelCase_							=       loss_fct(highway_logits.view(-1    )		,				labels.view(-1    )    )
                   else:
                         lowerCamelCase_							=       CrossEntropyLoss()
                         lowerCamelCase_							=       loss_fct(highway_logits.view(-1		,				self.num_labels    )		,				labels.view(-1    )    )
                   highway_losses.append(A_    )

             if train_highway:
                   lowerCamelCase_							=       (sum(highway_losses[:-1]    ),) + outputs
                   # exclude the final highway, of course
             else:
                   lowerCamelCase_							=       (loss,) + outputs
       if not self.training:
             lowerCamelCase_							=       outputs + ((original_entropy, highway_entropy), exit_layer)
             if output_layer >= 0:
                   lowerCamelCase_							=       (
                       (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
                   )  # use the highway of the last layer

       return outputs  # (loss), logits, (hidden_states), (attentions), entropy









from tempfile import TemporaryDirectory
from unittest import TestCase
from unittest.mock import MagicMock, patch

from transformers import AutoModel, TFAutoModel
from transformers.onnx import FeaturesManager
from transformers.testing_utils import SMALL_MODEL_IDENTIFIER, require_tf, require_torch


@require_torch
@require_tf
class    SCREAMING_SNAKE_CASE_				(       _UpperCamelCase   ):



	"""simple docstring"""


	def 					lowerCamelCase__     (					self		:						Dict	)	->       List[Any]:







						"""simple docstring"""







						__UpperCamelCase					:	Optional[Any]             =      SMALL_MODEL_IDENTIFIER
						__UpperCamelCase					:	List[str]             =      """pt"""
						__UpperCamelCase					:	List[Any]             =      """tf"""






	def 					lowerCamelCase__     (					self		:						Any					,     lowerCAmelCase		:						List[Any]	)	->       List[Any]:







						"""simple docstring"""







						__UpperCamelCase					:	str             =      AutoModel.from_pretrained(self.test_model	)
						model_pt.save_pretrained(lowerCAmelCase	)






	def 					lowerCamelCase__     (					self		:						List[Any]					,     lowerCAmelCase		:						Any	)	->       Optional[Any]:







						"""simple docstring"""







						__UpperCamelCase					:	List[str]             =      TFAutoModel.from_pretrained(self.test_model					,     from_pt=lowerCAmelCase	)
						model_tf.save_pretrained(lowerCAmelCase	)






	def 					lowerCamelCase__     (					self		:						str	)	->       Tuple:







						"""simple docstring"""







						__UpperCamelCase					:	Any             =      """mock_framework"""

						# Framework provided - return whatever the user provides
						__UpperCamelCase					:	List[Any]             =      FeaturesManager.determine_framework(self.test_model					,     lowerCAmelCase	)
						self.assertEqual(lowerCAmelCase					,     lowerCAmelCase	)

						# Local checkpoint and framework provided - return provided framework
						# PyTorch checkpoint
						with TemporaryDirectory() as local_pt_ckpt:
											self._setup_pt_ckpt(lowerCAmelCase	)
											__UpperCamelCase					:	Union[str, Any]             =      FeaturesManager.determine_framework(lowerCAmelCase					,     lowerCAmelCase	)
											self.assertEqual(lowerCAmelCase					,     lowerCAmelCase	)

						# TensorFlow checkpoint
						with TemporaryDirectory() as local_tf_ckpt:
											self._setup_tf_ckpt(lowerCAmelCase	)
											__UpperCamelCase					:	Any             =      FeaturesManager.determine_framework(lowerCAmelCase					,     lowerCAmelCase	)
											self.assertEqual(lowerCAmelCase					,     lowerCAmelCase	)






	def 					lowerCamelCase__     (					self		:						Optional[Any]	)	->       str:







						"""simple docstring"""







						with TemporaryDirectory() as local_pt_ckpt:
											self._setup_pt_ckpt(lowerCAmelCase	)
											__UpperCamelCase					:	Optional[Any]             =      FeaturesManager.determine_framework(lowerCAmelCase	)
											self.assertEqual(lowerCAmelCase					,     self.framework_pt	)

						# TensorFlow checkpoint
						with TemporaryDirectory() as local_tf_ckpt:
											self._setup_tf_ckpt(lowerCAmelCase	)
											__UpperCamelCase					:	Union[str, Any]             =      FeaturesManager.determine_framework(lowerCAmelCase	)
											self.assertEqual(lowerCAmelCase					,     self.framework_tf	)

						# Invalid local checkpoint
						with TemporaryDirectory() as local_invalid_ckpt:
											with self.assertRaises(lowerCAmelCase	):
																__UpperCamelCase					:	int             =      FeaturesManager.determine_framework(lowerCAmelCase	)






	def 					lowerCamelCase__     (					self		:						Dict	)	->       Tuple:







						"""simple docstring"""







						__UpperCamelCase					:	str             =      MagicMock(return_value=lowerCAmelCase	)
						with patch("""transformers.onnx.features.is_tf_available"""					,     lowerCAmelCase	):
											__UpperCamelCase					:	Any             =      FeaturesManager.determine_framework(self.test_model	)
											self.assertEqual(lowerCAmelCase					,     self.framework_pt	)

						# PyTorch not in environment -> use TensorFlow
						__UpperCamelCase					:	Optional[int]             =      MagicMock(return_value=lowerCAmelCase	)
						with patch("""transformers.onnx.features.is_torch_available"""					,     lowerCAmelCase	):
											__UpperCamelCase					:	Any             =      FeaturesManager.determine_framework(self.test_model	)
											self.assertEqual(lowerCAmelCase					,     self.framework_tf	)

						# Both in environment -> use PyTorch
						__UpperCamelCase					:	Dict             =      MagicMock(return_value=lowerCAmelCase	)
						__UpperCamelCase					:	Any             =      MagicMock(return_value=lowerCAmelCase	)
						with patch("""transformers.onnx.features.is_tf_available"""					,     lowerCAmelCase	), patch(
						    """transformers.onnx.features.is_torch_available"""					,     lowerCAmelCase	):
											__UpperCamelCase					:	int             =      FeaturesManager.determine_framework(self.test_model	)
											self.assertEqual(lowerCAmelCase					,     self.framework_pt	)

						# Both not in environment -> raise error
						__UpperCamelCase					:	Union[str, Any]             =      MagicMock(return_value=lowerCAmelCase	)
						__UpperCamelCase					:	Tuple             =      MagicMock(return_value=lowerCAmelCase	)
						with patch("""transformers.onnx.features.is_tf_available"""					,     lowerCAmelCase	), patch(
						    """transformers.onnx.features.is_torch_available"""					,     lowerCAmelCase	):
											with self.assertRaises(lowerCAmelCase	):
																__UpperCamelCase					:	str             =      FeaturesManager.determine_framework(self.test_model	)






