

import importlib
import inspect
import json
import os
import re
import shutil
import sys
from pathlib import Path
from typing import Dict, Optional, Union
from urllib import request

from huggingface_hub import HfFolder, cached_download, hf_hub_download, model_info
from packaging import version

from .. import __version__
from . import DIFFUSERS_DYNAMIC_MODULE_NAME, HF_MODULES_CACHE, logging


UpperCAmelCase_							:					Tuple		 =   (
    '''https://raw.githubusercontent.com/huggingface/diffusers/{revision}/examples/community/{pipeline}.py'''
)


UpperCAmelCase_							:					Optional[Any]		 =   logging.get_logger(__name__)  # pylint: disable=invalid-name
def 							__SCREAMING_SNAKE_CASE     (   )					->				Union[str, Any]:
	__A :			int		    =			"""https://pypi.org/pypi/diffusers/json"""
	__A :			List[str]		    =			json.loads(request.urlopen(a__       ).read()       )["""releases"""].keys()
	return sorted(a__			,key=lambda a__       : version.Version(a__       )       )
def 							__SCREAMING_SNAKE_CASE     (   )					->				Tuple:
	# This function has already been executed if HF_MODULES_CACHE already is in the Python path.
	if HF_MODULES_CACHE in sys.path:
		return

	sys.path.append(a__       )
	os.makedirs(a__			,exist_ok=a__       )
	__A :			str		    =			Path(a__       ) / """__init__.py"""
	if not init_path.exists():
		init_path.touch()
def 							__SCREAMING_SNAKE_CASE     (   a__    :   Union[str, os.PathLike]       )					->				List[Any]:
	init_hf_modules()
	__A :			Union[str, Any]		    =			Path(a__       ) / name
	# If the parent module does not exist yet, recursively create it.
	if not dynamic_module_path.parent.exists():
		create_dynamic_module(dynamic_module_path.parent       )
	os.makedirs(a__			,exist_ok=a__       )
	__A :			Optional[Any]		    =			dynamic_module_path / """__init__.py"""
	if not init_path.exists():
		init_path.touch()
def 							__SCREAMING_SNAKE_CASE     (   a__    :   List[Any]       )					->				Optional[int]:
	with open(a__			,"""r"""			,encoding="""utf-8"""       ) as f:
		__A :			Optional[int]		    =			f.read()

	# Imports of the form `import .xxx`
	__A :			str		    =			re.findall("""^\s*import\s+\.(\S+)\s*$"""			,a__			,flags=re.MULTILINE       )
	# Imports of the form `from .xxx import yyy`
	relative_imports += re.findall("""^\s*from\s+\.(\S+)\s+import"""			,a__			,flags=re.MULTILINE       )
	# Unique-ify
	return list(set(a__       )       )
def 							__SCREAMING_SNAKE_CASE     (   a__    :   List[Any]       )					->				Union[str, Any]:
	__A :			List[str]		    =			False
	__A :			Any		    =			[module_file]
	__A :			Any		    =			[]

	# Let's recurse through all relative imports
	while not no_change:
		__A :			Dict		    =			[]
		for f in files_to_check:
			new_imports.extend(get_relative_imports(a__       )       )

		__A :			Optional[Any]		    =			Path(a__       ).parent
		__A :			Tuple		    =			[str(module_path / m       ) for m in new_imports]
		__A :			int		    =			[f for f in new_import_files if f not in all_relative_imports]
		__A :			int		    =			[f"""{f}.py""" for f in new_import_files]

		__A :			Tuple		    =			len(a__       ) == 0
		all_relative_imports.extend(a__       )

	return all_relative_imports
def 							__SCREAMING_SNAKE_CASE     (   a__    :   str       )					->				Optional[Any]:
	with open(a__			,"""r"""			,encoding="""utf-8"""       ) as f:
		__A :			Dict		    =			f.read()

	# Imports of the form `import xxx`
	__A :			Tuple		    =			re.findall("""^\s*import\s+(\S+)\s*$"""			,a__			,flags=re.MULTILINE       )
	# Imports of the form `from xxx import yyy`
	imports += re.findall("""^\s*from\s+(\S+)\s+import"""			,a__			,flags=re.MULTILINE       )
	# Only keep the top-level module
	__A :			Tuple		    =			[imp.split("""."""       )[0] for imp in imports if not imp.startswith("""."""       )]

	# Unique-ify and test we got them all
	__A :			Any		    =			list(set(a__       )       )
	__A :			Optional[int]		    =			[]
	for imp in imports:
		try:
			importlib.import_module(a__       )
		except ImportError:
			missing_packages.append(a__       )

	if len(a__       ) > 0:
		raise ImportError(
		    """This modeling file requires the following packages that were not found in your environment: """
		    f"""{", ".join(a__       )}. Run `pip install {" ".join(a__       )}`"""       )

	return get_relative_imports(a__       )
def 							__SCREAMING_SNAKE_CASE     (   a__    :   Optional[Any]			,a__    :   str       )					->				Optional[int]:
	__A :			Dict		    =			module_path.replace(os.path.sep			,"""."""       )
	__A :			Tuple		    =			importlib.import_module(a__       )

	if class_name is None:
		return find_pipeline_class(a__       )
	return getattr(a__			,a__       )
def 							__SCREAMING_SNAKE_CASE     (   a__    :   Optional[Any]       )					->				Union[str, Any]:
	from ..pipelines import DiffusionPipeline

	__A :			Any		    =			dict(inspect.getmembers(a__			,inspect.isclass       )       )

	__A :			Dict		    =			None
	for cls_name, cls in cls_members.items():
		if (
		    cls_name != DiffusionPipeline.__name__
		    and issubclass(cls			,a__       )
		    and cls.__module__.split("""."""       )[0] != "diffusers"
		):
			if pipeline_class is not None:
				raise ValueError(
				    f"""Multiple classes that inherit from {DiffusionPipeline.__name__} have been found:"""
				    f""" {pipeline_class.__name__}, and {cls_name}. Please make sure to define only one in"""
				    f""" {loaded_module}."""       )
			__A :			Optional[Any]		    =			cls

	return pipeline_class
def 							__SCREAMING_SNAKE_CASE     (   a__    :   Union[str, os.PathLike]			,a__    :   str			,a__    :   Optional[Union[str, os.PathLike]] = None			,a__    :   bool = False			,a__    :   bool = False			,a__    :   Optional[Dict[str, str]] = None			,a__    :   Optional[Union[bool, str]] = None			,a__    :   Optional[str] = None			,a__    :   bool = False			,)					->				Union[str, Any]:
	__A :			Any		    =			str(a__       )

	__A :			Optional[Any]		    =			os.path.join(a__			,a__       )

	if os.path.isfile(a__       ):
		__A :			Any		    =			module_file_or_url
		__A :			int		    =			"""local"""
	elif pretrained_model_name_or_path.count("""/"""       ) == 0:
		__A :			Tuple		    =			get_diffusers_versions()
		# cut ".dev0"
		__A :			str		    =			"""v""" + """.""".join(__version__.split("""."""       )[:3]       )

		# retrieve github version that matches
		if revision is None:
			__A :			Optional[Any]		    =			latest_version if latest_version[1:] in available_versions else """main"""
			logger.info(f"""Defaulting to latest_version: {revision}."""       )
		elif revision in available_versions:
			__A :			Any		    =			f"""v{revision}"""
		elif revision == "main":
			__A :			Dict		    =			revision
		else:
			raise ValueError(
			    f"""`custom_revision`: {revision} does not exist. Please make sure to choose one of"""
			    f""" {", ".join(available_versions + ["main"]       )}."""       )

		# community pipeline on GitHub
		__A :			Union[str, Any]		    =			COMMUNITY_PIPELINES_URL.format(revision=a__			,pipeline=a__       )
		try:
			__A :			Optional[Any]		    =			cached_download(
			    a__			,cache_dir=a__			,force_download=a__			,proxies=a__			,resume_download=a__			,local_files_only=a__			,use_auth_token=a__			,)
			__A :			Dict		    =			"""git"""
			__A :			str		    =			pretrained_model_name_or_path + """.py"""
		except EnvironmentError:
			logger.error(f"""Could not locate the {module_file} inside {pretrained_model_name_or_path}."""       )
			raise
	else:
		try:
			# Load from URL or cache if already cached
			__A :			List[str]		    =			hf_hub_download(
			    a__			,a__			,cache_dir=a__			,force_download=a__			,proxies=a__			,resume_download=a__			,local_files_only=a__			,use_auth_token=a__			,)
			__A :			Tuple		    =			os.path.join("""local"""			,"""--""".join(pretrained_model_name_or_path.split("""/"""       )       )       )
		except EnvironmentError:
			logger.error(f"""Could not locate the {module_file} inside {pretrained_model_name_or_path}."""       )
			raise

    # Check we have all the requirements in our environment
	__A :			Tuple		    =			check_imports(a__       )

	# Now we move the module inside our cached dynamic modules.
	__A :			str		    =			DIFFUSERS_DYNAMIC_MODULE_NAME + os.path.sep + submodule
	create_dynamic_module(a__       )
	__A :			List[Any]		    =			Path(a__       ) / full_submodule
	if submodule == "local" or submodule == "git":
		# We always copy local files (we could hash the file to see if there was a change, and give them the name of
		# that hash, to only copy when there is a modification but it seems overkill for now).
		# The only reason we do the copy is to avoid putting too many folders in sys.path.
		shutil.copy(a__			,submodule_path / module_file       )
		for module_needed in modules_needed:
			__A :			int		    =			f"""{module_needed}.py"""
			shutil.copy(os.path.join(a__			,a__       )			,submodule_path / module_needed       )
	else:
		# Get the commit hash
		# TODO: we will get this info in the etag soon, so retrieve it from there and not here.
		if isinstance(a__			,a__       ):
			__A :			List[Any]		    =			use_auth_token
		elif use_auth_token is True:
			__A :			Any		    =			HfFolder.get_token()
		else:
			__A :			Dict		    =			None

		__A :			Union[str, Any]		    =			model_info(a__			,revision=a__			,token=a__       ).sha

		# The module file will end up being placed in a subfolder with the git hash of the repo. This way we get the
		# benefit of versioning.
		__A :			Union[str, Any]		    =			submodule_path / commit_hash
		__A :			Union[str, Any]		    =			full_submodule + os.path.sep + commit_hash
		create_dynamic_module(a__       )

		if not (submodule_path / module_file).exists():
			shutil.copy(a__			,submodule_path / module_file       )
		# Make sure we also have every file with relative
		for module_needed in modules_needed:
			if not (submodule_path / module_needed).exists():
				get_cached_module_file(
				    a__			,f"""{module_needed}.py"""			,cache_dir=a__			,force_download=a__			,resume_download=a__			,proxies=a__			,use_auth_token=a__			,revision=a__			,local_files_only=a__			,)
	return os.path.join(a__			,a__       )







def 							__SCREAMING_SNAKE_CASE     (   a__    :   Union[str, os.PathLike]			,a__    :   str			,a__    :   Optional[str] = None			,a__    :   Optional[Union[str, os.PathLike]] = None			,a__    :   bool = False			,a__    :   bool = False			,a__    :   Optional[Dict[str, str]] = None			,a__    :   Optional[Union[bool, str]] = None			,a__    :   Optional[str] = None			,a__    :   bool = False			,**a__    :   Optional[Any]			,)					->				List[Any]:
	__A :			Optional[int]		    =			get_cached_module_file(
	    a__			,a__			,cache_dir=a__			,force_download=a__			,resume_download=a__			,proxies=a__			,use_auth_token=a__			,revision=a__			,local_files_only=a__			,)
	return get_class_in_module(a__			,final_module.replace(""".py"""			,""""""       )       )











import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEmbeddings,
    BertLayer,
    BertPooler,
    BertPreTrainedModel,
)


def 		lowerCamelCase__       (						_A				):







	'''simple docstring'''





	snake_case_						       =	torch.exp(_A				)
	snake_case_						       =	torch.sum(_A	,      dim=1				)  # sum of exp(x_i)
	snake_case_						       =	torch.sum(x * exp_x	,      dim=1				)  # sum of x_i * exp(x_i)
	return torch.log(_A				) - B / A
class 						UpperCAmelCase  (							nn.Module     ):







						'''simple docstring'''





						def __init__(		self				:	Optional[int]			,      __lowercase				:	List[Any]		):






							"""simple docstring"""






							super().__init__()
							snake_case_						       =	config.output_attentions
							snake_case_						       =	config.output_hidden_states
							snake_case_						       =	nn.ModuleList([BertLayer(__lowercase		) for _ in range(config.num_hidden_layers		)]		)
							snake_case_						       =	nn.ModuleList([BertHighway(__lowercase		) for _ in range(config.num_hidden_layers		)]		)

							snake_case_						       =	[-1 for _ in range(config.num_hidden_layers		)]






						def 			snake_case__		(		self				:	Union[str, Any]			,      __lowercase				:	Optional[int]		):






							"""simple docstring"""






							if (type(__lowercase		) is float) or (type(__lowercase		) is int):
								for i in range(len(self.early_exit_entropy		)		):
									snake_case_						       =	x
							else:
								snake_case_						       =	x






						def 			snake_case__		(		self				:	int			,      __lowercase				:	Tuple		):






							"""simple docstring"""






							snake_case_						       =	pooler.state_dict()
							for highway in self.highway:
								for name, param in highway.pooler.state_dict().items():
									param.copy_(loaded_model[name]		)






						def 			snake_case__		(		self				:	List[Any]			,      __lowercase				:	List[str]			,      __lowercase				:	List[Any]=None			,      __lowercase				:	Optional[int]=None			,      __lowercase				:	Optional[Any]=None			,      __lowercase				:	List[str]=None			,      ):






							"""simple docstring"""






							snake_case_						       =	()
							snake_case_						       =	()
							snake_case_						       =	()
							for i, layer_module in enumerate(self.layer		):
								if self.output_hidden_states:
									snake_case_						       =	all_hidden_states + (hidden_states,)

								snake_case_						       =	layer_module(
								    __lowercase			,      __lowercase			,      head_mask[i]			,      __lowercase			,      __lowercase		)
								snake_case_						       =	layer_outputs[0]

								if self.output_attentions:
									snake_case_						       =	all_attentions + (layer_outputs[1],)

								snake_case_						       =	(hidden_states,)
								if self.output_hidden_states:
									snake_case_						       =	current_outputs + (all_hidden_states,)
								if self.output_attentions:
									snake_case_						       =	current_outputs + (all_attentions,)

								snake_case_						       =	self.highway[i](__lowercase		)
								# logits, pooled_output

								if not self.training:
									snake_case_						       =	highway_exit[0]
									snake_case_						       =	entropy(__lowercase		)
									snake_case_						       =	highway_exit + (highway_entropy,)  # logits, hidden_states(?), entropy
									snake_case_						       =	all_highway_exits + (highway_exit,)

									if highway_entropy < self.early_exit_entropy[i]:
										snake_case_						       =	(highway_logits,) + current_outputs[1:] + (all_highway_exits,)
										raise HighwayException(__lowercase			,      i + 1		)
								else:
									snake_case_						       =	all_highway_exits + (highway_exit,)

        # Add last layer
							if self.output_hidden_states:
								snake_case_						       =	all_hidden_states + (hidden_states,)

							snake_case_						       =	(hidden_states,)
							if self.output_hidden_states:
								snake_case_						       =	outputs + (all_hidden_states,)
							if self.output_attentions:
								snake_case_						       =	outputs + (all_attentions,)

							snake_case_						       =	outputs + (all_highway_exits,)
							return outputs  # last-layer hidden state, (all hidden states), (all attentions), all highway exits







@add_start_docstrings(
    '''The Bert Model transformer with early exiting (DeeBERT). '''			,  UpperCAmelCase__			,  )
class 						UpperCAmelCase  (							UpperCAmelCase__     ):







						'''simple docstring'''





						def __init__(		self				:	Union[str, Any]			,      __lowercase				:	List[Any]		):






							"""simple docstring"""






							super().__init__(__lowercase		)
							snake_case_						       =	config

							snake_case_						       =	BertEmbeddings(__lowercase		)
							snake_case_						       =	DeeBertEncoder(__lowercase		)
							snake_case_						       =	BertPooler(__lowercase		)

							self.init_weights()






						def 			snake_case__		(		self				:	Tuple		):






							"""simple docstring"""






							self.encoder.init_highway_pooler(self.pooler		)






						def 			snake_case__		(		self				:	Tuple		):






							"""simple docstring"""






							return self.embeddings.word_embeddings






						def 			snake_case__		(		self				:	Optional[int]			,      __lowercase				:	Any		):






							"""simple docstring"""






							snake_case_						       =	value






						def 			snake_case__		(		self				:	List[Any]			,      __lowercase				:	Optional[int]		):






							"""simple docstring"""






							for layer, heads in heads_to_prune.items():
								self.encoder.layer[layer].attention.prune_heads(__lowercase		)






						@add_start_docstrings_to_model_forward(__lowercase		)
						def 			snake_case__		(		self				:	List[str]			,      __lowercase				:	List[str]=None			,      __lowercase				:	int=None			,      __lowercase				:	Tuple=None			,      __lowercase				:	Dict=None			,      __lowercase				:	str=None			,      __lowercase				:	Tuple=None			,      __lowercase				:	List[Any]=None			,      __lowercase				:	List[str]=None			,      ):






							"""simple docstring"""






							if input_ids is not None and inputs_embeds is not None:
								raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time"		)
							elif input_ids is not None:
								snake_case_						       =	input_ids.size()
							elif inputs_embeds is not None:
								snake_case_						       =	inputs_embeds.size()[:-1]
							else:
								raise ValueError("You have to specify either input_ids or inputs_embeds"		)

							snake_case_						       =	input_ids.device if input_ids is not None else inputs_embeds.device

							if attention_mask is None:
								snake_case_						       =	torch.ones(__lowercase			,      device=__lowercase		)
							if encoder_attention_mask is None:
								snake_case_						       =	torch.ones(__lowercase			,      device=__lowercase		)
							if token_type_ids is None:
								snake_case_						       =	torch.zeros(__lowercase			,      dtype=torch.long			,      device=__lowercase		)

							# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
							# ourselves in which case we just need to make it broadcastable to all heads.
							snake_case_						       =	self.get_extended_attention_mask(__lowercase			,      __lowercase			,      __lowercase		)

							# If a 2D ou 3D attention mask is provided for the cross-attention
							# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
							if encoder_attention_mask.dim() == 3:
								snake_case_						       =	encoder_attention_mask[:, None, :, :]
							if encoder_attention_mask.dim() == 2:
								snake_case_						       =	encoder_attention_mask[:, None, None, :]

							snake_case_						       =	encoder_extended_attention_mask.to(
							    dtype=next(self.parameters()		).dtype		)  # fp16 compatibility
							snake_case_						       =	(1.0 - encoder_extended_attention_mask) * -1_0000.0

							# Prepare head mask if needed
							# 1.0 in head_mask indicate we keep the head
							# attention_probs has shape bsz x n_heads x N x N
							# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
							# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
							snake_case_						       =	self.get_head_mask(__lowercase			,      self.config.num_hidden_layers		)

							snake_case_						       =	self.embeddings(
							    input_ids=__lowercase			,      position_ids=__lowercase			,      token_type_ids=__lowercase			,      inputs_embeds=__lowercase		)
							snake_case_						       =	self.encoder(
							    __lowercase			,      attention_mask=__lowercase			,      head_mask=__lowercase			,      encoder_hidden_states=__lowercase			,      encoder_attention_mask=__lowercase			,      )
							snake_case_						       =	encoder_outputs[0]
							snake_case_						       =	self.pooler(__lowercase		)

							snake_case_						       =	(
							    sequence_output,
							    pooled_output,
							) + encoder_outputs[
							    1:
							]  # add hidden_states and attentions if they are here
							return outputs  # sequence_output, pooled_output, (hidden_states), (attentions), highway exits







class 						UpperCAmelCase  (							UpperCAmelCase__     ):







						'''simple docstring'''





						def __init__(		self				:	Dict			,      __lowercase				:	Tuple			,      __lowercase				:	List[Any]		):






							"""simple docstring"""






							snake_case_						       =	message
							snake_case_						       =	exit_layer  # start from 1!







class 						UpperCAmelCase  (							nn.Module     ):







						'''simple docstring'''





						def __init__(		self				:	int			,      __lowercase				:	Optional[int]		):






							"""simple docstring"""






							super().__init__()
							snake_case_						       =	BertPooler(__lowercase		)
							snake_case_						       =	nn.Dropout(config.hidden_dropout_prob		)
							snake_case_						       =	nn.Linear(config.hidden_size			,      config.num_labels		)






						def 			snake_case__		(		self				:	List[str]			,      __lowercase				:	str		):






							"""simple docstring"""






							snake_case_						       =	encoder_outputs[0]
							snake_case_						       =	self.pooler(__lowercase		)
							# "return" pooler_output

							# BertModel
							snake_case_						       =	(pooler_input, pooler_output) + encoder_outputs[1:]
							# "return" bmodel_output

							# Dropout and classification
							snake_case_						       =	bmodel_output[1]

							snake_case_						       =	self.dropout(__lowercase		)
							snake_case_						       =	self.classifier(__lowercase		)

							return logits, pooled_output







@add_start_docstrings(
    '''Bert Model (with early exiting - DeeBERT) with a classifier on top,
    also takes care of multi-layer training. '''			,  UpperCAmelCase__			,  )
class 						UpperCAmelCase  (							UpperCAmelCase__     ):







						'''simple docstring'''





						def __init__(		self				:	Union[str, Any]			,      __lowercase				:	Union[str, Any]		):






							"""simple docstring"""






							super().__init__(__lowercase		)
							snake_case_						       =	config.num_labels
							snake_case_						       =	config.num_hidden_layers

							snake_case_						       =	DeeBertModel(__lowercase		)
							snake_case_						       =	nn.Dropout(config.hidden_dropout_prob		)
							snake_case_						       =	nn.Linear(config.hidden_size			,      self.config.num_labels		)

							self.init_weights()






						@add_start_docstrings_to_model_forward(__lowercase		)
						def 			snake_case__		(		self				:	Optional[int]			,      __lowercase				:	List[Any]=None			,      __lowercase				:	Dict=None			,      __lowercase				:	Optional[Any]=None			,      __lowercase				:	Optional[int]=None			,      __lowercase				:	List[Any]=None			,      __lowercase				:	Tuple=None			,      __lowercase				:	List[str]=None			,      __lowercase				:	int=-1			,      __lowercase				:	Tuple=False			,      ):






							"""simple docstring"""






							snake_case_						       =	self.num_layers
							try:
								snake_case_						       =	self.bert(
								    __lowercase			,      attention_mask=__lowercase			,      token_type_ids=__lowercase			,      position_ids=__lowercase			,      head_mask=__lowercase			,      inputs_embeds=__lowercase			,      )
								# sequence_output, pooled_output, (hidden_states), (attentions), highway exits

								snake_case_						       =	outputs[1]

								snake_case_						       =	self.dropout(__lowercase		)
								snake_case_						       =	self.classifier(__lowercase		)
								snake_case_						       =	(logits,) + outputs[2:]  # add hidden states and attention if they are here
							except HighwayException as e:
								snake_case_						       =	e.message
								snake_case_						       =	e.exit_layer
								snake_case_						       =	outputs[0]

							if not self.training:
								snake_case_						       =	entropy(__lowercase		)
								snake_case_						       =	[]
								snake_case_						       =	[]
							if labels is not None:
								if self.num_labels == 1:
									#  We are doing regression
									snake_case_						       =	MSELoss()
									snake_case_						       =	loss_fct(logits.view(-1		)			,      labels.view(-1		)		)
								else:
									snake_case_						       =	CrossEntropyLoss()
									snake_case_						       =	loss_fct(logits.view(-1			,      self.num_labels		)			,      labels.view(-1		)		)

								# work with highway exits
								snake_case_						       =	[]
								for highway_exit in outputs[-1]:
									snake_case_						       =	highway_exit[0]
									if not self.training:
										highway_logits_all.append(__lowercase		)
										highway_entropy.append(highway_exit[2]		)
									if self.num_labels == 1:
										#  We are doing regression
										snake_case_						       =	MSELoss()
										snake_case_						       =	loss_fct(highway_logits.view(-1		)			,      labels.view(-1		)		)
									else:
										snake_case_						       =	CrossEntropyLoss()
										snake_case_						       =	loss_fct(highway_logits.view(-1			,      self.num_labels		)			,      labels.view(-1		)		)
									highway_losses.append(__lowercase		)

								if train_highway:
									snake_case_						       =	(sum(highway_losses[:-1]		),) + outputs
									# exclude the final highway, of course
								else:
									snake_case_						       =	(loss,) + outputs
							if not self.training:
								snake_case_						       =	outputs + ((original_entropy, highway_entropy), exit_layer)
								if output_layer >= 0:
									snake_case_						       =	(
									    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
									)  # use the highway of the last layer

							return outputs  # (loss), logits, (hidden_states), (attentions), (highway_exits)
