
import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEmbeddings,
    BertLayer,
    BertPooler,
    BertPreTrainedModel,
)



def      __lowerCAmelCase						(							UpperCAmelCase__				:      Optional[Any]       )  ->    str:
							lowerCamelCase_							  =					torch.exp(UpperCAmelCase__       )
							lowerCamelCase_							  =					torch.sum(UpperCAmelCase__			,				dim=1       )  # sum of exp(x_i)
							lowerCamelCase_							  =					torch.sum(x * exp_x			,				dim=1       )  # sum of x_i * exp(x_i)
							return torch.log(UpperCAmelCase__       ) - B / A





class       __A(				nn.Module       ):

					def __init__(	self					:     Optional[int]   ,					__UpperCamelCase					:     str   ):
												super().__init__()
												lowerCamelCase_							  =					config.output_attentions
												lowerCamelCase_							  =					config.output_hidden_states
												lowerCamelCase_							  =					nn.ModuleList([BertLayer(A__   ) for _ in range(config.num_hidden_layers   )]   )
												lowerCamelCase_							  =					nn.ModuleList([BertHighway(A__   ) for _ in range(config.num_hidden_layers   )]   )

												lowerCamelCase_							  =					[-1 for _ in range(config.num_hidden_layers   )]

					def    lowercase__      (	self					:     str   ,					__UpperCamelCase					:     Optional[Any]   ):
												if (type(A__   ) is float) or (type(A__   ) is int):
																			for i in range(len(self.early_exit_entropy   )   ):
																										lowerCamelCase_							  =					x
												else:
																			lowerCamelCase_							  =					x

					def    lowercase__      (	self					:     Optional[int]   ,					__UpperCamelCase					:     Union[str, Any]   ):
												lowerCamelCase_							  =					pooler.state_dict()
												for highway in self.highway:
																			for name, param in highway.pooler.state_dict().items():
																										param.copy_(loaded_model[name]   )



					def    lowercase__      (	self					:     str   ,					__UpperCamelCase					:     Dict   ,					__UpperCamelCase					:     Tuple=None   ,					__UpperCamelCase					:     Union[str, Any]=None   ,					__UpperCamelCase					:     Any=None   ,					__UpperCamelCase					:     Optional[Any]=None   ,					):
												lowerCamelCase_							  =					()
												lowerCamelCase_							  =					()
												lowerCamelCase_							  =					()
												for i, layer_module in enumerate(self.layer   ):
																			if self.output_hidden_states:
																										lowerCamelCase_							  =					all_hidden_states + (hidden_states,)

																			lowerCamelCase_							  =					layer_module(
																			    A__   ,					A__   ,					head_mask[i]   ,					A__   ,					A__   )
																			lowerCamelCase_							  =					layer_outputs[0]

																			if self.output_attentions:
																										lowerCamelCase_							  =					all_attentions + (layer_outputs[1],)

																			lowerCamelCase_							  =					(hidden_states,)
																			if self.output_hidden_states:
																										lowerCamelCase_							  =					current_outputs + (all_hidden_states,)
																			if self.output_attentions:
																										lowerCamelCase_							  =					current_outputs + (all_attentions,)

																			lowerCamelCase_							  =					self.highway[i](A__   )
																			# logits, pooled_output

																			if not self.training:
																										lowerCamelCase_							  =					highway_exit[0]
																										lowerCamelCase_							  =					entropy(A__   )
																										lowerCamelCase_							  =					highway_exit + (highway_entropy,)  # logits, hidden_states(?), entropy
																										lowerCamelCase_							  =					all_highway_exits + (highway_exit,)

																										if highway_entropy < self.early_exit_entropy[i]:
																																	lowerCamelCase_							  =					(highway_logits,) + current_outputs[1:] + (all_highway_exits,)
																																	raise HighwayException(A__   ,					i + 1   )
																			else:
																										lowerCamelCase_							  =					all_highway_exits + (highway_exit,)

																										# Add last layer
												if self.output_hidden_states:
																			lowerCamelCase_							  =					all_hidden_states + (hidden_states,)

												lowerCamelCase_							  =					(hidden_states,)
												if self.output_hidden_states:
																			lowerCamelCase_							  =					outputs + (all_hidden_states,)
												if self.output_attentions:
																			lowerCamelCase_							  =					outputs + (all_attentions,)

												lowerCamelCase_							  =					outputs + (all_highway_exits,)
												return outputs  # last-layer hidden state, (all hidden states), (all attentions), all highway exits





@add_start_docstrings(
    '''The Bert Model transformer with early exiting (DeeBERT). '''     ,		__a     ,		)
class       __A(				__a       ):

					def __init__(	self					:     List[Any]   ,					__UpperCamelCase					:     Dict   ):
												super().__init__(A__   )
												lowerCamelCase_							  =					config

												lowerCamelCase_							  =					BertEmbeddings(A__   )
												lowerCamelCase_							  =					DeeBertEncoder(A__   )
												lowerCamelCase_							  =					BertPooler(A__   )

												self.init_weights()

					def    lowercase__      (	self					:     Union[str, Any]   ):
												self.encoder.init_highway_pooler(self.pooler   )

					def    lowercase__      (	self					:     Union[str, Any]   ):
												return self.embeddings.word_embeddings

					def    lowercase__      (	self					:     Optional[int]   ,					__UpperCamelCase					:     Union[str, Any]   ):
												lowerCamelCase_							  =					value

					def    lowercase__      (	self					:     Any   ,					__UpperCamelCase					:     List[Any]   ):
												for layer, heads in heads_to_prune.items():
																			self.encoder.layer[layer].attention.prune_heads(A__   )



					@add_start_docstrings_to_model_forward(A__   )
					def    lowercase__      (	self					:     str   ,					__UpperCamelCase					:     int=None   ,					__UpperCamelCase					:     str=None   ,					__UpperCamelCase					:     Tuple=None   ,					__UpperCamelCase					:     Any=None   ,					__UpperCamelCase					:     List[Any]=None   ,					__UpperCamelCase					:     Any=None   ,					__UpperCamelCase					:     List[Any]=None   ,					__UpperCamelCase					:     int=None   ,					):
												if input_ids is not None and inputs_embeds is not None:
																			raise ValueError("""You cannot specify both input_ids and inputs_embeds at the same time"""   )
												elif input_ids is not None:
																			lowerCamelCase_							  =					input_ids.size()
												elif inputs_embeds is not None:
																			lowerCamelCase_							  =					inputs_embeds.size()[:-1]
												else:
																			raise ValueError("""You have to specify either input_ids or inputs_embeds"""   )

												lowerCamelCase_							  =					input_ids.device if input_ids is not None else inputs_embeds.device

												if attention_mask is None:
																			lowerCamelCase_							  =					torch.ones(A__   ,					device=A__   )
												if encoder_attention_mask is None:
																			lowerCamelCase_							  =					torch.ones(A__   ,					device=A__   )
												if token_type_ids is None:
																			lowerCamelCase_							  =					torch.zeros(A__   ,					dtype=torch.long   ,					device=A__   )

												# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
												# ourselves in which case we just need to make it broadcastable to all heads.
												lowerCamelCase_							  =					self.get_extended_attention_mask(A__   ,					A__   ,					A__   )

												# If a 2D ou 3D attention mask is provided for the cross-attention
												# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
												if encoder_attention_mask.dim() == 3:
																			lowerCamelCase_							  =					encoder_attention_mask[:, None, :, :]
												if encoder_attention_mask.dim() == 2:
																			lowerCamelCase_							  =					encoder_attention_mask[:, None, None, :]

												lowerCamelCase_							  =					encoder_extended_attention_mask.to(
												    dtype=next(self.parameters()   ).dtype   )  # fp16 compatibility
												lowerCamelCase_							  =					(1.0 - encoder_extended_attention_mask) * -1_0_0_0_0.0

												# Prepare head mask if needed
												# 1.0 in head_mask indicate we keep the head
												# attention_probs has shape bsz x n_heads x N x N
												# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
												# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
												lowerCamelCase_							  =					self.get_head_mask(A__   ,					self.config.num_hidden_layers   )

												lowerCamelCase_							  =					self.embeddings(
												    input_ids=A__   ,					position_ids=A__   ,					token_type_ids=A__   ,					inputs_embeds=A__   )
												lowerCamelCase_							  =					self.encoder(
												    A__   ,					attention_mask=A__   ,					head_mask=A__   ,					encoder_hidden_states=A__   ,					encoder_attention_mask=A__   ,					)
												lowerCamelCase_							  =					encoder_outputs[0]
												lowerCamelCase_							  =					self.pooler(A__   )

												lowerCamelCase_							  =					(
												    sequence_output,
												    pooled_output,
												) + encoder_outputs[
												    1:
												]  # add hidden_states and attentions if they are here
												return outputs  # sequence_output, pooled_output, (hidden_states), (attentions), highway exits





class       __A(				__a       ):

					def __init__(	self					:     Optional[int]   ,					__UpperCamelCase					:     str   ,					__UpperCamelCase					:     List[str]   ):
												lowerCamelCase_							  =					message
												lowerCamelCase_							  =					exit_layer  # start from 1!





class       __A(				nn.Module       ):

					def __init__(	self					:     List[str]   ,					__UpperCamelCase					:     Tuple   ):
												super().__init__()
												lowerCamelCase_							  =					BertPooler(A__   )
												lowerCamelCase_							  =					nn.Dropout(config.hidden_dropout_prob   )
												lowerCamelCase_							  =					nn.Linear(config.hidden_size   ,					config.num_labels   )



					def    lowercase__      (	self					:     Dict   ,					__UpperCamelCase					:     Optional[int]   ):
												# Pooler
												lowerCamelCase_							  =					encoder_outputs[0]
												lowerCamelCase_							  =					self.pooler(A__   )
												# "return" pooler_output

												# BertModel
												lowerCamelCase_							  =					(pooler_input, pooler_output) + encoder_outputs[1:]
												# "return" bmodel_output

												# Dropout and classification
												lowerCamelCase_							  =					bmodel_output[1]

												lowerCamelCase_							  =					self.dropout(A__   )
												lowerCamelCase_							  =					self.classifier(A__   )

												return logits, pooled_output





@add_start_docstrings(
    '''Bert Model (with early exiting - DeeBERT) with a classifier on top,
    also takes care of multi-layer training. '''     ,		__a     ,		)
class       __A(				__a       ):

					def __init__(	self					:     Any   ,					__UpperCamelCase					:     Optional[int]   ):
												super().__init__(A__   )
												lowerCamelCase_							  =					config.num_labels
												lowerCamelCase_							  =					config.num_hidden_layers

												lowerCamelCase_							  =					DeeBertModel(A__   )
												lowerCamelCase_							  =					nn.Dropout(config.hidden_dropout_prob   )
												lowerCamelCase_							  =					nn.Linear(config.hidden_size   ,					self.config.num_labels   )

												self.init_weights()



					@add_start_docstrings_to_model_forward(A__   )
					def    lowercase__      (	self					:     Union[str, Any]   ,					__UpperCamelCase					:     Tuple=None   ,					__UpperCamelCase					:     Dict=None   ,					__UpperCamelCase					:     Optional[Any]=None   ,					__UpperCamelCase					:     str=None   ,					__UpperCamelCase					:     List[str]=None   ,					__UpperCamelCase					:     Union[str, Any]=None   ,					__UpperCamelCase					:     Optional[int]=None   ,					__UpperCamelCase					:     List[Any]=-1   ,					__UpperCamelCase					:     Optional[int]=False   ,					):
												lowerCamelCase_							  =					self.num_layers
												try:
																			lowerCamelCase_							  =					self.bert(
																			    A__   ,					attention_mask=A__   ,					token_type_ids=A__   ,					position_ids=A__   ,					head_mask=A__   ,					inputs_embeds=A__   ,					)
																			# sequence_output, pooled_output, (hidden_states), (attentions), highway exits

																			lowerCamelCase_							  =					outputs[1]

																			lowerCamelCase_							  =					self.dropout(A__   )
																			lowerCamelCase_							  =					self.classifier(A__   )
																			lowerCamelCase_							  =					(logits,) + outputs[2:]  # add hidden states and attention if they are here
												except HighwayException as e:
																			lowerCamelCase_							  =					e.message
																			lowerCamelCase_							  =					e.exit_layer
																			lowerCamelCase_							  =					outputs[0]

												if not self.training:
																			lowerCamelCase_							  =					entropy(A__   )
																			lowerCamelCase_							  =					[]
																			lowerCamelCase_							  =					[]
												if labels is not None:
																			if self.num_labels == 1:
																										#  We are doing regression
																										lowerCamelCase_							  =					MSELoss()
																										lowerCamelCase_							  =					loss_fct(logits.view(-1   )   ,					labels.view(-1   )   )
																			else:
																										lowerCamelCase_							  =					CrossEntropyLoss()
																										lowerCamelCase_							  =					loss_fct(logits.view(-1   ,					self.num_labels   )   ,					labels.view(-1   )   )

																			# work with highway exits
																			lowerCamelCase_							  =					[]
																			for highway_exit in outputs[-1]:
																										lowerCamelCase_							  =					highway_exit[0]
																										if not self.training:
																																	highway_logits_all.append(A__   )
																																	highway_entropy.append(highway_exit[2]   )
																										if self.num_labels == 1:
																																	#  We are doing regression
																																	lowerCamelCase_							  =					MSELoss()
																																	lowerCamelCase_							  =					loss_fct(highway_logits.view(-1   )   ,					labels.view(-1   )   )
																										else:
																																	lowerCamelCase_							  =					CrossEntropyLoss()
																																	lowerCamelCase_							  =					loss_fct(highway_logits.view(-1   ,					self.num_labels   )   ,					labels.view(-1   )   )
																										highway_losses.append(A__   )

																			if train_highway:
																										lowerCamelCase_							  =					(sum(highway_losses[:-1]   ),) + outputs
																										# exclude the final highway, of course
																			else:
																										lowerCamelCase_							  =					(loss,) + outputs
												if not self.training:
																			lowerCamelCase_							  =					outputs + ((original_entropy, highway_entropy), exit_layer)
																			if output_layer >= 0:
																										lowerCamelCase_							  =					(
																										    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
																										)  # use the highway of the last layer

												return outputs  # (loss), logits, (hidden_states), (attentions), (highway_exits)







import os
import socket
from contextlib import contextmanager

import torch

from ..commands.config.default import write_basic_config  # noqa: F401
from ..state import PartialState
from .dataclasses import DistributedType
from .imports import is_deepspeed_available, is_tpu_available
from .transformer_engine import convert_model
from .versions import is_torch_version


if is_deepspeed_available():
		from deepspeed import DeepSpeedEngine

if is_tpu_available(check_device=False):
		import torch_xla.core.xla_model as xm



def      __lowerCAmelCase						(							UpperCAmelCase__				:      Any       )  ->    List[str]:
							if is_torch_version("""<"""			,				"""2.0.0"""       ) or not hasattr(UpperCAmelCase__			,				"""_dynamo"""       ):
														return False
							return isinstance(UpperCAmelCase__			,				torch._dynamo.eval_frame.OptimizedModule       )



def      __lowerCAmelCase						(							UpperCAmelCase__				:      str			,				UpperCAmelCase__				:      bool = True       )  ->    str:
							lowerCamelCase_							  =					(torch.nn.parallel.DistributedDataParallel, torch.nn.DataParallel)

							lowerCamelCase_							  =					is_compiled_module(UpperCAmelCase__       )
							if is_compiled:
														lowerCamelCase_							  =					model
														lowerCamelCase_							  =					model._orig_mod

							if is_deepspeed_available():
														options += (DeepSpeedEngine,)

							while isinstance(UpperCAmelCase__			,				UpperCAmelCase__       ):
														lowerCamelCase_							  =					model.module

							if not keep_fpaa_wrapper:
														lowerCamelCase_							  =					getattr(UpperCAmelCase__			,				"""forward"""       )
														lowerCamelCase_							  =					model.__dict__.pop("""_original_forward"""			,				UpperCAmelCase__       )
														if original_forward is not None:
																					while hasattr(UpperCAmelCase__			,				"""__wrapped__"""       ):
																												lowerCamelCase_							  =					forward.__wrapped__
																												if forward == original_forward:
																																			break
																					lowerCamelCase_							  =					forward
														if getattr(UpperCAmelCase__			,				"""_converted_to_transformer_engine"""			,				UpperCAmelCase__       ):
																					convert_model(UpperCAmelCase__			,				to_transformer_engine=UpperCAmelCase__       )

							if is_compiled:
														lowerCamelCase_							  =					model
														lowerCamelCase_							  =					compiled_model

							return model



def      __lowerCAmelCase						(							)  ->    Any:
							PartialState().wait_for_everyone()



def      __lowerCAmelCase						(							UpperCAmelCase__				:      Union[str, Any]			,				UpperCAmelCase__				:      Tuple       )  ->    Dict:
							if PartialState().distributed_type == DistributedType.TPU:
														xm.save(UpperCAmelCase__			,				UpperCAmelCase__       )
							elif PartialState().local_process_index == 0:
														torch.save(UpperCAmelCase__			,				UpperCAmelCase__       )



@contextmanager
def      __lowerCAmelCase						(							**UpperCAmelCase__				:      Any       )  ->    Dict:
							for key, value in kwargs.items():
														lowerCamelCase_							  =					str(UpperCAmelCase__       )

							yield

							for key in kwargs:
														if key.upper() in os.environ:
																					del os.environ[key.upper()]



def      __lowerCAmelCase						(							UpperCAmelCase__				:      Union[str, Any]       )  ->    str:
							if not hasattr(UpperCAmelCase__			,				"""__qualname__"""       ) and not hasattr(UpperCAmelCase__			,				"""__name__"""       ):
														lowerCamelCase_							  =					getattr(UpperCAmelCase__			,				"""__class__"""			,				UpperCAmelCase__       )
							if hasattr(UpperCAmelCase__			,				"""__qualname__"""       ):
														return obj.__qualname__
							if hasattr(UpperCAmelCase__			,				"""__name__"""       ):
														return obj.__name__
							return str(UpperCAmelCase__       )



def      __lowerCAmelCase						(							UpperCAmelCase__				:      str			,				UpperCAmelCase__				:      Union[str, Any]       )  ->    Dict:
							for key, value in source.items():
														if isinstance(UpperCAmelCase__			,				UpperCAmelCase__       ):
																					lowerCamelCase_							  =					destination.setdefault(UpperCAmelCase__			,				{}       )
																					merge_dicts(UpperCAmelCase__			,				UpperCAmelCase__       )
														else:
																					lowerCamelCase_							  =					value

							return destination



def      __lowerCAmelCase						(							UpperCAmelCase__				:      int = None       )  ->    bool:
							if port is None:
														lowerCamelCase_							  =					2_9_5_0_0
							with socket.socket(socket.AF_INET			,				socket.SOCK_STREAM       ) as s:
														return s.connect_ex(("""localhost""", port)       ) == 0





