



"""simple docstring"""

from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy
@add_start_docstrings(
    '''The RoBERTa Model transformer with early exiting (DeeRoBERTa). '''							,							_lowerCamelCase							,							)
class     lowerCAmelCase__						(	_lowerCamelCase ):


	'''simple docstring'''







	_SCREAMING_SNAKE_CASE	:		List[str]  						= RobertaConfig
	_SCREAMING_SNAKE_CASE	:		List[Any]  						= '''roberta'''





	def __init__(      self				:       str     ,       _SCREAMING_SNAKE_CASE				:       Optional[int]    )						->	str:





					"""simple docstring"""


					super().__init__(_SCREAMING_SNAKE_CASE    )

					SCREAMING_SNAKE_CASE						:							Optional[Any]							=     RobertaEmbeddings(_SCREAMING_SNAKE_CASE    )
					self.init_weights()




@add_start_docstrings(
    '''RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. '''							,							_lowerCamelCase							,							)
class     lowerCAmelCase__						(	_lowerCamelCase ):


	'''simple docstring'''







	_SCREAMING_SNAKE_CASE	:		Optional[int]  						= RobertaConfig
	_SCREAMING_SNAKE_CASE	:		List[Any]  						= '''roberta'''





	def __init__(      self				:       List[Any]     ,       _SCREAMING_SNAKE_CASE				:       str    )						->	Optional[int]:





					"""simple docstring"""


					super().__init__(_SCREAMING_SNAKE_CASE    )
					SCREAMING_SNAKE_CASE						:							Dict							=     config.num_labels
					SCREAMING_SNAKE_CASE						:							List[str]							=     config.num_hidden_layers

					SCREAMING_SNAKE_CASE						:							List[str]							=     DeeRobertaModel(_SCREAMING_SNAKE_CASE    )
					SCREAMING_SNAKE_CASE						:							int							=     nn.Dropout(config.hidden_dropout_prob    )
					SCREAMING_SNAKE_CASE						:							int							=     nn.Linear(config.hidden_size     ,       self.config.num_labels    )







	@add_start_docstrings_to_model_forward(_SCREAMING_SNAKE_CASE    )
	def 			_lowerCAmelCase     (      self				:       int     ,       _SCREAMING_SNAKE_CASE				:       str=None     ,       _SCREAMING_SNAKE_CASE				:       Optional[Any]=None     ,       _SCREAMING_SNAKE_CASE				:       Optional[int]=None     ,       _SCREAMING_SNAKE_CASE				:       Tuple=None     ,       _SCREAMING_SNAKE_CASE				:       Dict=None     ,       _SCREAMING_SNAKE_CASE				:       Optional[int]=None     ,       _SCREAMING_SNAKE_CASE				:       List[Any]=None     ,       _SCREAMING_SNAKE_CASE				:       List[str]=-1     ,       _SCREAMING_SNAKE_CASE				:       Dict=False     ,       )						->	List[str]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Any							=     self.num_layers
					try:
									SCREAMING_SNAKE_CASE						:							str							=     self.roberta(
									    _SCREAMING_SNAKE_CASE     ,       attention_mask=_SCREAMING_SNAKE_CASE     ,       token_type_ids=_SCREAMING_SNAKE_CASE     ,       position_ids=_SCREAMING_SNAKE_CASE     ,       head_mask=_SCREAMING_SNAKE_CASE     ,       inputs_embeds=_SCREAMING_SNAKE_CASE     ,       )

									SCREAMING_SNAKE_CASE						:							Dict							=     outputs[1]

									SCREAMING_SNAKE_CASE						:							Optional[Any]							=     self.dropout(_SCREAMING_SNAKE_CASE    )
									SCREAMING_SNAKE_CASE						:							str							=     self.classifier(_SCREAMING_SNAKE_CASE    )
									SCREAMING_SNAKE_CASE						:							List[str]							=     (logits,) + outputs[2:]  # add hidden states and attention if they are here
					except HighwayException as e:
									SCREAMING_SNAKE_CASE						:							Union[str, Any]							=     e.message
									SCREAMING_SNAKE_CASE						:							Optional[int]							=     e.exit_layer
									SCREAMING_SNAKE_CASE						:							Union[str, Any]							=     outputs[0]

					if not self.training:
									SCREAMING_SNAKE_CASE						:							List[Any]							=     entropy(_SCREAMING_SNAKE_CASE    )
									SCREAMING_SNAKE_CASE						:							Optional[int]							=     []
									SCREAMING_SNAKE_CASE						:							Any							=     []
					if labels is not None:
									if self.num_labels == 1:
													#  We are doing regression
													SCREAMING_SNAKE_CASE						:							str							=     MSELoss()
													SCREAMING_SNAKE_CASE						:							int							=     loss_fct(logits.view(-1    )     ,       labels.view(-1    )    )
									else:
													SCREAMING_SNAKE_CASE						:							str							=     CrossEntropyLoss()
													SCREAMING_SNAKE_CASE						:							Tuple							=     loss_fct(logits.view(-1     ,       self.num_labels    )     ,       labels.view(-1    )    )

									# work with highway exits
									SCREAMING_SNAKE_CASE						:							Optional[int]							=     []
									for highway_exit in outputs[-1]:
													SCREAMING_SNAKE_CASE						:							List[Any]							=     highway_exit[0]
													if not self.training:
																	highway_logits_all.append(_SCREAMING_SNAKE_CASE    )
																	highway_entropy.append(highway_exit[2]    )
													if self.num_labels == 1:
																	#  We are doing regression
																	SCREAMING_SNAKE_CASE						:							int							=     MSELoss()
																	SCREAMING_SNAKE_CASE						:							Dict							=     loss_fct(highway_logits.view(-1    )     ,       labels.view(-1    )    )
													else:
																	SCREAMING_SNAKE_CASE						:							Dict							=     CrossEntropyLoss()
																	SCREAMING_SNAKE_CASE						:							Optional[Any]							=     loss_fct(highway_logits.view(-1     ,       self.num_labels    )     ,       labels.view(-1    )    )
													highway_losses.append(_SCREAMING_SNAKE_CASE    )

									if train_highway:
													SCREAMING_SNAKE_CASE						:							List[str]							=     (sum(highway_losses[:-1]    ),) + outputs
													# exclude the final highway, of course
									else:
													SCREAMING_SNAKE_CASE						:							List[str]							=     (loss,) + outputs
					if not self.training:
									SCREAMING_SNAKE_CASE						:							Optional[int]							=     outputs + ((original_entropy, highway_entropy), exit_layer)
									if output_layer >= 0:
													SCREAMING_SNAKE_CASE						:							Optional[Any]							=     (
													    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
													)  # use the highway of the last layer

					return outputs  # (loss), logits, (hidden_states), (attentions), entropy










"""simple docstring"""

import argparse

import torch

from transformers import BertConfig, BertForPreTraining, load_tf_weights_in_bert
from transformers.utils import logging


logging.set_verbosity_info()
def  __snake_case							(	__A      :						Union[str, Any]							,		__A      :						Any							,		__A      :						Dict     )     ->       Tuple:






				'''simple docstring'''
				# Initialise PyTorch model
				SCREAMING_SNAKE_CASE						:							Union[str, Any]							=     BertConfig.from_json_file(__A     )
				print(F"""Building PyTorch model from configuration: {config}"""     )
				SCREAMING_SNAKE_CASE						:							List[Any]							=     BertForPreTraining(__A     )

				# Load weights from tf checkpoint
				load_tf_weights_in_bert(__A							,		__A							,		__A     )

				# Save pytorch-model
				print(F"""Save PyTorch model to {pytorch_dump_path}"""     )
				torch.save(model.state_dict()							,		__A     )


if __name__ == "__main__":
							A_						:    int				=      argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    '--tf_checkpoint_path', default=None, type=str, required=True, help='Path to the TensorFlow checkpoint path.'
							)
							parser.add_argument(
							    '--bert_config_file',
							    default=None,
							    type=str,
							    required=True,
							    help=(
							        'The config json file corresponding to the pre-trained BERT model. \n'
							        'This specifies the model architecture.'
							    ),
							)
							parser.add_argument(
							    '--pytorch_dump_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
							)
							A_						:    str				=      parser.parse_args()
							convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.bert_config_file, args.pytorch_dump_path)





