'''simple docstring'''






from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy





@add_start_docstrings(
    '''The RoBERTa Model transformer with early exiting (DeeRoBERTa). '''   ,						UpperCamelCase   ,						)
class        A__				(			UpperCamelCase  ):





   """simple docstring"""





   UpperCamelCase_			:		int       							=	RobertaConfig
   UpperCamelCase_			:		str       							=	'''roberta'''




   def __init__(					self	:						Tuple		,	lowerCAmelCase__	:						Union[str, Any]     )			->       Optional[int]:







      """simple docstring"""




      super().__init__(lowerCamelCase__     )

      _UpperCAmelCase				:      Optional[Any]				=  RobertaEmbeddings(lowerCamelCase__     )
      self.init_weights()





@add_start_docstrings(
    '''RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. '''   ,						UpperCamelCase   ,						)
class        A__				(			UpperCamelCase  ):





   """simple docstring"""





   UpperCamelCase_			:		str       							=	RobertaConfig
   UpperCamelCase_			:		Optional[int]       							=	'''roberta'''




   def __init__(					self	:						int		,	lowerCAmelCase__	:						Optional[Any]     )			->       Tuple:







      """simple docstring"""




      super().__init__(lowerCamelCase__     )
      _UpperCAmelCase				:      Union[str, Any]				=  config.num_labels
      _UpperCAmelCase				:      Tuple				=  config.num_hidden_layers

      _UpperCAmelCase				:      int				=  DeeRobertaModel(lowerCamelCase__     )
      _UpperCAmelCase				:      Dict				=  nn.Dropout(config.hidden_dropout_prob     )
      _UpperCAmelCase				:      Optional[Any]				=  nn.Linear(config.hidden_size		,	self.config.num_labels     )




   @add_start_docstrings_to_model_forward(lowerCamelCase__     )
   def      _lowerCAmelCase						(					self	:						Tuple		,	lowerCAmelCase__	:						List[Any]=None		,	lowerCAmelCase__	:						Dict=None		,	lowerCAmelCase__	:						Tuple=None		,	lowerCAmelCase__	:						Union[str, Any]=None		,	lowerCAmelCase__	:						Tuple=None		,	lowerCAmelCase__	:						Any=None		,	lowerCAmelCase__	:						Any=None		,	lowerCAmelCase__	:						List[str]=-1		,	lowerCAmelCase__	:						Dict=False		,	)			->       str:







      """simple docstring"""




      _UpperCAmelCase				:      Dict				=  self.num_layers
      try:
         _UpperCAmelCase				:      List[str]				=  self.roberta(
             lowerCamelCase__		,	attention_mask=lowerCamelCase__		,	token_type_ids=lowerCamelCase__		,	position_ids=lowerCamelCase__		,	head_mask=lowerCamelCase__		,	inputs_embeds=lowerCamelCase__		,	)

         _UpperCAmelCase				:      str				=  outputs[1]

         _UpperCAmelCase				:      str				=  self.dropout(lowerCamelCase__     )
         _UpperCAmelCase				:      List[str]				=  self.classifier(lowerCamelCase__     )
         _UpperCAmelCase				:      List[str]				=  (logits,) + outputs[2:]  # add hidden states and attention if they are here
      except HighwayException as e:
         _UpperCAmelCase				:      Optional[Any]				=  e.message
         _UpperCAmelCase				:      Tuple				=  e.exit_layer
         _UpperCAmelCase				:      List[Any]				=  outputs[0]

      if not self.training:
         _UpperCAmelCase				:      Tuple				=  entropy(lowerCamelCase__     )
         _UpperCAmelCase				:      int				=  []
         _UpperCAmelCase				:      Dict				=  []
      if labels is not None:
         if self.num_labels == 1:
            #  We are doing regression
            _UpperCAmelCase				:      int				=  MSELoss()
            _UpperCAmelCase				:      str				=  loss_fct(logits.view(-1     )		,	labels.view(-1     )     )
         else:
            _UpperCAmelCase				:      Union[str, Any]				=  CrossEntropyLoss()
            _UpperCAmelCase				:      Tuple				=  loss_fct(logits.view(-1		,	self.num_labels     )		,	labels.view(-1     )     )

         # work with highway exits
         _UpperCAmelCase				:      Dict				=  []
         for highway_exit in outputs[-1]:
            _UpperCAmelCase				:      int				=  highway_exit[0]
            if not self.training:
               highway_logits_all.append(lowerCamelCase__     )
               highway_entropy.append(highway_exit[2]     )
            if self.num_labels == 1:
               #  We are doing regression
               _UpperCAmelCase				:      Tuple				=  MSELoss()
               _UpperCAmelCase				:      Optional[int]				=  loss_fct(highway_logits.view(-1     )		,	labels.view(-1     )     )
            else:
               _UpperCAmelCase				:      int				=  CrossEntropyLoss()
               _UpperCAmelCase				:      Tuple				=  loss_fct(highway_logits.view(-1		,	self.num_labels     )		,	labels.view(-1     )     )
            highway_losses.append(lowerCamelCase__     )

         if train_highway:
            _UpperCAmelCase				:      Optional[int]				=  (sum(highway_losses[:-1]     ),) + outputs
            # exclude the final highway, of course
         else:
            _UpperCAmelCase				:      Union[str, Any]				=  (loss,) + outputs
      if not self.training:
         _UpperCAmelCase				:      Tuple				=  outputs + ((original_entropy, highway_entropy), exit_layer)
         if output_layer >= 0:
            _UpperCAmelCase				:      str				=  (
                (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
            )  # use the highway of the last layer

      return outputs  # (loss), logits, (hidden_states), (attentions), entropy




'''simple docstring'''
import unittest

from transformers import AutoConfig, AutoTokenizer, BertConfig, TensorType, is_flax_available
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, slow


if is_flax_available():
					import jax

					from transformers.models.auto.modeling_flax_auto import FlaxAutoModel
					from transformers.models.bert.modeling_flax_bert import FlaxBertModel
					from transformers.models.roberta.modeling_flax_roberta import FlaxRobertaModel






@require_flax
class 				lowercase__      (				unittest.TestCase  ):





	@slow
	def       UpperCamelCase_    (					self   :							Union[str, Any]					):







							'''simple docstring'''
							for model_name in ["bert-base-cased", "bert-large-uncased"]:
													with self.subTest(lowerCamelCase__					):
																			_UpperCamelCase   :						Optional[int]												=  AutoConfig.from_pretrained(lowerCamelCase__					)
																			self.assertIsNotNone(lowerCamelCase__					)
																			self.assertIsInstance(lowerCamelCase__					,lowerCamelCase__					)

																			_UpperCamelCase   :						Dict												=  FlaxAutoModel.from_pretrained(lowerCamelCase__					)
																			self.assertIsNotNone(lowerCamelCase__					)
																			self.assertIsInstance(lowerCamelCase__					,lowerCamelCase__					)





	@slow
	def       UpperCamelCase_    (					self   :							List[Any]					):







							'''simple docstring'''
							for model_name in ["roberta-base", "roberta-large"]:
													with self.subTest(lowerCamelCase__					):
																			_UpperCamelCase   :						Any												=  AutoConfig.from_pretrained(lowerCamelCase__					)
																			self.assertIsNotNone(lowerCamelCase__					)
																			self.assertIsInstance(lowerCamelCase__					,lowerCamelCase__					)

																			_UpperCamelCase   :						Any												=  FlaxAutoModel.from_pretrained(lowerCamelCase__					)
																			self.assertIsNotNone(lowerCamelCase__					)
																			self.assertIsInstance(lowerCamelCase__					,lowerCamelCase__					)





	@slow
	def       UpperCamelCase_    (					self   :							Union[str, Any]					):







							'''simple docstring'''
							for model_name in ["bert-base-cased", "bert-large-uncased"]:
													_UpperCamelCase   :						Optional[int]												=  AutoTokenizer.from_pretrained(lowerCamelCase__					)
													_UpperCamelCase   :						List[Any]												=  FlaxBertModel.from_pretrained(lowerCamelCase__					)
													_UpperCamelCase   :						Optional[int]												=  tokenizer('Do you support jax jitted function?'					,return_tensors=TensorType.JAX					)

													@jax.jit
													def eval(**lowerCamelCase__   :							Union[str, Any]					):
																			return model(**lowerCamelCase__					)

													eval(**lowerCamelCase__					).block_until_ready()





	@slow
	def       UpperCamelCase_    (					self   :							Dict					):







							'''simple docstring'''
							for model_name in ["roberta-base", "roberta-large"]:
													_UpperCamelCase   :						Optional[int]												=  AutoTokenizer.from_pretrained(lowerCamelCase__					)
													_UpperCamelCase   :						Tuple												=  FlaxRobertaModel.from_pretrained(lowerCamelCase__					)
													_UpperCamelCase   :						Optional[Any]												=  tokenizer('Do you support jax jitted function?'					,return_tensors=TensorType.JAX					)

													@jax.jit
													def eval(**lowerCamelCase__   :							Union[str, Any]					):
																			return model(**lowerCamelCase__					)

													eval(**lowerCamelCase__					).block_until_ready()





	def       UpperCamelCase_    (					self   :							Union[str, Any]					):







							'''simple docstring'''
							with self.assertRaisesRegex(
							    lowerCamelCase__					,'bert-base is not a local folder and is not a valid model identifier'					):
													_UpperCamelCase   :						int												=  FlaxAutoModel.from_pretrained('bert-base'					)





	def       UpperCamelCase_    (					self   :							str					):







							'''simple docstring'''
							with self.assertRaisesRegex(
							    lowerCamelCase__					,R'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'					):
													_UpperCamelCase   :						Tuple												=  FlaxAutoModel.from_pretrained(lowerCamelCase__					,revision='aaaaaa'					)





	def       UpperCamelCase_    (					self   :							List[str]					):







							'''simple docstring'''
							with self.assertRaisesRegex(
							    lowerCamelCase__					,'hf-internal-testing/config-no-model does not appear to have a file named flax_model.msgpack'					,):
													_UpperCamelCase   :						List[Any]												=  FlaxAutoModel.from_pretrained('hf-internal-testing/config-no-model'					)





	def       UpperCamelCase_    (					self   :							List[str]					):







							'''simple docstring'''
							with self.assertRaisesRegex(lowerCamelCase__					,'Use `from_pt=True` to load this model'					):
													_UpperCamelCase   :						Tuple												=  FlaxAutoModel.from_pretrained('hf-internal-testing/tiny-bert-pt-only'					)


