



'''simple docstring'''






from __future__ import annotations

import inspect
import unittest
from typing import List, Tuple

from transformers import RegNetConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
						import tensorflow as tf

						from transformers import TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST, TFRegNetForImageClassification, TFRegNetModel


if is_vision_available():
						from PIL import Image

						from transformers import AutoImageProcessor

class        SCREAMING_SNAKE_CASE:







	"""simple docstring"""






	def __init__(		self    :  int	,    __snake_case    :  Any	,    __snake_case    :  Optional[int]=3	,    __snake_case    :  str=32	,    __snake_case    :  int=3	,    __snake_case    :  Any=10	,    __snake_case    :  Dict=[10, 20, 30, 40]	,    __snake_case    :  str=[1, 1, 2, 1]	,    __snake_case    :  Any=True	,    __snake_case    :  Optional[Any]=True	,    __snake_case    :  List[str]="relu"	,    __snake_case    :  List[str]=3	,    __snake_case    :  Union[str, Any]=None	,    )    ->	int:
								UpperCAmelCase				: Dict											=	parent
								UpperCAmelCase				: List[Any]											=	batch_size
								UpperCAmelCase				: Optional[Any]											=	image_size
								UpperCAmelCase				: Dict											=	num_channels
								UpperCAmelCase				: Tuple											=	embeddings_size
								UpperCAmelCase				: List[Any]											=	hidden_sizes
								UpperCAmelCase				: Optional[int]											=	depths
								UpperCAmelCase				: Optional[Any]											=	is_training
								UpperCAmelCase				: Any											=	use_labels
								UpperCAmelCase				: List[Any]											=	hidden_act
								UpperCAmelCase				: Union[str, Any]											=	num_labels
								UpperCAmelCase				: Optional[Any]											=	scope
								UpperCAmelCase				: Union[str, Any]											=	len(__snake_case	)







	def   A			(		self    :  Any	)    ->	Optional[Any]:
								UpperCAmelCase				: Optional[Any]											=	floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]	)

								UpperCAmelCase				: Dict											=	None
								if self.use_labels:
															UpperCAmelCase				: int											=	ids_tensor([self.batch_size]	,    self.num_labels	)

								UpperCAmelCase				: Tuple											=	self.get_config()
								return config, pixel_values, labels







	def   A			(		self    :  Union[str, Any]	)    ->	Tuple:
								return RegNetConfig(
								    num_channels=self.num_channels	,    embeddings_size=self.embeddings_size	,    hidden_sizes=self.hidden_sizes	,    depths=self.depths	,    hidden_act=self.hidden_act	,    num_labels=self.num_labels	,    )







	def   A			(		self    :  Tuple	,    __snake_case    :  List[Any]	,    __snake_case    :  Tuple	,    __snake_case    :  Any	)    ->	Optional[Any]:
								UpperCAmelCase				: Union[str, Any]											=	TFRegNetModel(config=__snake_case	)
								UpperCAmelCase				: Any											=	model(__snake_case	,    training=__snake_case	)
								# expected last hidden states: B, C, H // 32, W // 32
								self.parent.assertEqual(
								    result.last_hidden_state.shape	,    (self.batch_size, self.hidden_sizes[-1], self.image_size // 32, self.image_size // 32)	,    )







	def   A			(		self    :  Optional[int]	,    __snake_case    :  Tuple	,    __snake_case    :  str	,    __snake_case    :  Optional[Any]	)    ->	Optional[Any]:
								UpperCAmelCase				: Optional[Any]											=	self.num_labels
								UpperCAmelCase				: Any											=	TFRegNetForImageClassification(__snake_case	)
								UpperCAmelCase				: int											=	model(__snake_case	,    labels=__snake_case	,    training=__snake_case	)
								self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.num_labels)	)







	def   A			(		self    :  str	)    ->	Tuple:
								UpperCAmelCase				: Union[str, Any]											=	self.prepare_config_and_inputs()
								UpperCAmelCase							,	UpperCAmelCase							,	UpperCAmelCase				: Union[str, Any]											=	config_and_inputs
								UpperCAmelCase				: List[Any]											=	{'''pixel_values''': pixel_values}
								return config, inputs_dict

@require_tf
class        SCREAMING_SNAKE_CASE(							A__							,	A__							,	unittest.TestCase    ):







	"""simple docstring"""






	lowerCamelCase__    				= (TFRegNetModel, TFRegNetForImageClassification) if is_tf_available() else ()
	lowerCamelCase__    				= (
	    {"""feature-extraction""": TFRegNetModel, """image-classification""": TFRegNetForImageClassification}
	    if is_tf_available()
	    else {}
	)

	lowerCamelCase__    				= False
	lowerCamelCase__    				= False
	lowerCamelCase__    				= False
	lowerCamelCase__    				= False
	lowerCamelCase__    				= False







	def   A			(		self    :  Optional[int]	)    ->	List[str]:
								UpperCAmelCase				: Any											=	TFRegNetModelTester(self	)
								UpperCAmelCase				: Any											=	ConfigTester(self	,    config_class=__snake_case	,    has_text_modality=__snake_case	)







	def   A			(		self    :  List[Any]	)    ->	List[Any]:
								return







	@unittest.skip(reason='''RegNet does not use inputs_embeds'''	)
	def   A			(		self    :  List[Any]	)    ->	Union[str, Any]:
								pass







	@unittest.skipIf(
	    not is_tf_available() or len(tf.config.list_physical_devices('''GPU'''	)	) == 0	,    reason='''TF does not support backprop for grouped convolutions on CPU.'''	,    )
	@slow
	def   A			(		self    :  List[Any]	)    ->	Dict:
								super().test_keras_fit()







	@unittest.skip(reason='''RegNet does not support input and output embeddings'''	)
	def   A			(		self    :  List[str]	)    ->	int:
								pass







	def   A			(		self    :  int	)    ->	int:
								UpperCAmelCase							,	UpperCAmelCase				: int											=	self.model_tester.prepare_config_and_inputs_for_common()

								for model_class in self.all_model_classes:
															UpperCAmelCase				: Union[str, Any]											=	model_class(__snake_case	)
															UpperCAmelCase				: Optional[int]											=	inspect.signature(model.call	)
															# signature.parameters is an OrderedDict => so arg_names order is deterministic
															UpperCAmelCase				: Optional[Any]											=	[*signature.parameters.keys()]

															UpperCAmelCase				: str											=	['''pixel_values''']
															self.assertListEqual(arg_names[:1]	,    __snake_case	)







	def   A			(		self    :  Tuple	)    ->	List[Any]:
								UpperCAmelCase				: Dict											=	self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*__snake_case	)







	def   A			(		self    :  List[str]	)    ->	Tuple:
								def check_hidden_states_output(__snake_case    :  Union[str, Any]	,    __snake_case    :  List[str]	,    __snake_case    :  int	):
															UpperCAmelCase				: Union[str, Any]											=	model_class(__snake_case	)
															UpperCAmelCase				: Optional[Any]											=	model(**self._prepare_for_class(__snake_case	,    __snake_case	)	,    training=__snake_case	)

															UpperCAmelCase				: str											=	outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

															UpperCAmelCase				: Any											=	self.model_tester.num_stages
															self.assertEqual(len(__snake_case	)	,    expected_num_stages + 1	)

															# RegNet's feature maps are of shape (batch_size, num_channels, height, width)
															self.assertListEqual(
															    list(hidden_states[0].shape[-2:]	)	,    [self.model_tester.image_size // 2, self.model_tester.image_size // 2]	,    )

								UpperCAmelCase							,	UpperCAmelCase				: Any											=	self.model_tester.prepare_config_and_inputs_for_common()
								UpperCAmelCase				: int											=	['''basic''', '''bottleneck''']
								for model_class in self.all_model_classes:
															for layer_type in layers_type:
																						UpperCAmelCase				: List[str]											=	layer_type
																						UpperCAmelCase				: Union[str, Any]											=	True
																						check_hidden_states_output(__snake_case	,    __snake_case	,    __snake_case	)

																						# check that output_hidden_states also work using config
																						del inputs_dict["output_hidden_states"]
																						UpperCAmelCase				: Dict											=	True

																						check_hidden_states_output(__snake_case	,    __snake_case	,    __snake_case	)







	def   A			(		self    :  int	)    ->	Union[str, Any]:
								UpperCAmelCase							,	UpperCAmelCase				: List[Any]											=	self.model_tester.prepare_config_and_inputs_for_common()

								def check_equivalence(__snake_case    :  Optional[Any]	,    __snake_case    :  Tuple	,    __snake_case    :  Optional[int]	,    __snake_case    :  Union[str, Any]={}	):
															UpperCAmelCase				: int											=	model(__snake_case	,    return_dict=__snake_case	,    **__snake_case	)
															UpperCAmelCase				: Any											=	model(__snake_case	,    return_dict=__snake_case	,    **__snake_case	).to_tuple()

															def recursive_check(__snake_case    :  int	,    __snake_case    :  List[str]	):
																						if isinstance(__snake_case	,    (List, Tuple)	):
																													for tuple_iterable_value, dict_iterable_value in zip(__snake_case	,    __snake_case	):
																																				recursive_check(__snake_case	,    __snake_case	)
																						elif tuple_object is None:
																													return
																						else:
																													self.assertTrue(
																													    all(tf.equal(__snake_case	,    __snake_case	)	)	,    msg=(
																													        '''Tuple and dict output are not equal. Difference:'''
																													        F""" {tf.math.reduce_max(tf.abs(tuple_object - dict_object	)	)}"""
																													    )	,    )

																						recursive_check(__snake_case	,    __snake_case	)

								for model_class in self.all_model_classes:
															UpperCAmelCase				: Dict											=	model_class(__snake_case	)

															UpperCAmelCase				: Union[str, Any]											=	self._prepare_for_class(__snake_case	,    __snake_case	)
															UpperCAmelCase				: Tuple											=	self._prepare_for_class(__snake_case	,    __snake_case	)
															check_equivalence(__snake_case	,    __snake_case	,    __snake_case	)

															UpperCAmelCase				: int											=	self._prepare_for_class(__snake_case	,    __snake_case	,    return_labels=__snake_case	)
															UpperCAmelCase				: str											=	self._prepare_for_class(__snake_case	,    __snake_case	,    return_labels=__snake_case	)
															check_equivalence(__snake_case	,    __snake_case	,    __snake_case	)

															UpperCAmelCase				: List[Any]											=	self._prepare_for_class(__snake_case	,    __snake_case	)
															UpperCAmelCase				: Optional[int]											=	self._prepare_for_class(__snake_case	,    __snake_case	)
															check_equivalence(__snake_case	,    __snake_case	,    __snake_case	,    {'''output_hidden_states''': True}	)

															UpperCAmelCase				: Any											=	self._prepare_for_class(__snake_case	,    __snake_case	,    return_labels=__snake_case	)
															UpperCAmelCase				: Optional[Any]											=	self._prepare_for_class(__snake_case	,    __snake_case	,    return_labels=__snake_case	)
															check_equivalence(__snake_case	,    __snake_case	,    __snake_case	,    {'''output_hidden_states''': True}	)







	def   A			(		self    :  Dict	)    ->	Dict:
								UpperCAmelCase				: Union[str, Any]											=	self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_image_classification(*__snake_case	)







	@slow
	def   A			(		self    :  int	)    ->	Any:
								for model_name in TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
															UpperCAmelCase				: List[str]											=	TFRegNetModel.from_pretrained(__snake_case	)
															self.assertIsNotNone(__snake_case	)


def 				snake_case_  (      )   ->     Any:
							UpperCAmelCase				: Optional[int]											=	Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''						)
							return image




@require_tf
@require_vision
class        SCREAMING_SNAKE_CASE(							unittest.TestCase    ):







	"""simple docstring"""






	@cached_property
	def   A			(		self    :  Dict	)    ->	Tuple:
								return (
								    AutoImageProcessor.from_pretrained(TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[0]	)
								    if is_vision_available()
								    else None
								)







	@slow
	def   A			(		self    :  Optional[Any]	)    ->	List[str]:
								UpperCAmelCase				: int											=	TFRegNetForImageClassification.from_pretrained(TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[0]	)

								UpperCAmelCase				: Tuple											=	self.default_image_processor
								UpperCAmelCase				: Tuple											=	prepare_img()
								UpperCAmelCase				: int											=	image_processor(images=__snake_case	,    return_tensors='''tf'''	)

								# forward pass
								UpperCAmelCase				: Union[str, Any]											=	model(**__snake_case	,    training=__snake_case	)

								# verify the logits
								UpperCAmelCase				: Any											=	tf.TensorShape((1, 1000)	)
								self.assertEqual(outputs.logits.shape	,    __snake_case	)

								UpperCAmelCase				: str											=	tf.constant([-0.41_80, -1.50_51, -3.48_36]	)

								tf.debugging.assert_near(outputs.logits[0, :3]	,    __snake_case	,    atol=1E-4	)










'''simple docstring'''






import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


UpperCamelCase__:     List[str]										=    logging.getLogger(__name__)

class        SCREAMING_SNAKE_CASE(							A__    ):







	"""simple docstring"""






	def   A			(		self    :  str	,    __snake_case    :  Tuple	,    __snake_case    :  int	,    __snake_case    :  List[Any]=None	,    __snake_case    :  List[Any]=None	)    ->	Optional[int]:
								UpperCAmelCase				: Union[str, Any]											=	self.layer[current_layer](__snake_case	,    __snake_case	,    head_mask[current_layer]	)

								UpperCAmelCase				: Optional[Any]											=	layer_outputs[0]

								return hidden_states

@add_start_docstrings(
    """The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top."""							,	A__							,	)
class        SCREAMING_SNAKE_CASE(							A__    ):







	"""simple docstring"""






	def __init__(		self    :  int	,    __snake_case    :  Any	)    ->	Any:
								super().__init__(__snake_case	)

								UpperCAmelCase				: Any											=	BertEncoderWithPabee(__snake_case	)

								self.init_weights()
								UpperCAmelCase				: Optional[Any]											=	0
								UpperCAmelCase				: List[str]											=	0
								UpperCAmelCase				: Tuple											=	0

								UpperCAmelCase				: List[str]											=	0







	def   A			(		self    :  List[str]	,    __snake_case    :  List[str]	)    ->	List[Any]:
								UpperCAmelCase				: Union[str, Any]											=	threshold







	def   A			(		self    :  str	,    __snake_case    :  List[str]	)    ->	Optional[Any]:
								UpperCAmelCase				: List[str]											=	patience







	def   A			(		self    :  Dict	)    ->	str:
								UpperCAmelCase				: List[Any]											=	0
								UpperCAmelCase				: Dict											=	0







	def   A			(		self    :  Optional[Any]	)    ->	Optional[int]:
								UpperCAmelCase				: Dict											=	self.inference_layers_num / self.inference_instances_num
								UpperCAmelCase				: int											=	(
								    F"""*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="""
								    F""" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"""
								)
								print(__snake_case	)







	@add_start_docstrings_to_model_forward(__snake_case	)
	def   A			(		self    :  Union[str, Any]	,    __snake_case    :  Optional[Any]=None	,    __snake_case    :  str=None	,    __snake_case    :  int=None	,    __snake_case    :  Optional[Any]=None	,    __snake_case    :  Tuple=None	,    __snake_case    :  Tuple=None	,    __snake_case    :  Tuple=None	,    __snake_case    :  Optional[Any]=None	,    __snake_case    :  Union[str, Any]=None	,    __snake_case    :  Optional[int]=None	,    __snake_case    :  Optional[int]=False	,    )    ->	List[str]:

								if input_ids is not None and inputs_embeds is not None:
															raise ValueError('''You cannot specify both input_ids and inputs_embeds at the same time'''	)
								elif input_ids is not None:
															UpperCAmelCase				: Union[str, Any]											=	input_ids.size()
								elif inputs_embeds is not None:
															UpperCAmelCase				: List[Any]											=	inputs_embeds.size()[:-1]
								else:
															raise ValueError('''You have to specify either input_ids or inputs_embeds'''	)

								UpperCAmelCase				: Optional[Any]											=	input_ids.device if input_ids is not None else inputs_embeds.device

								if attention_mask is None:
															UpperCAmelCase				: Optional[Any]											=	torch.ones(__snake_case	,    device=__snake_case	)
								if token_type_ids is None:
															UpperCAmelCase				: Tuple											=	torch.zeros(__snake_case	,    dtype=torch.long	,    device=__snake_case	)

								# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
								# ourselves in which case we just need to make it broadcastable to all heads.
								UpperCAmelCase				: torch.Tensor											=	self.get_extended_attention_mask(__snake_case	,    __snake_case	,    __snake_case	)

								# If a 2D ou 3D attention mask is provided for the cross-attention
								# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
								if self.config.is_decoder and encoder_hidden_states is not None:
															UpperCAmelCase							,	UpperCAmelCase							,	UpperCAmelCase				: Any											=	encoder_hidden_states.size()
															UpperCAmelCase				: List[Any]											=	(encoder_batch_size, encoder_sequence_length)
															if encoder_attention_mask is None:
																						UpperCAmelCase				: str											=	torch.ones(__snake_case	,    device=__snake_case	)
															UpperCAmelCase				: Tuple											=	self.invert_attention_mask(__snake_case	)
								else:
															UpperCAmelCase				: List[str]											=	None

								# Prepare head mask if needed
								# 1.0 in head_mask indicate we keep the head
								# attention_probs has shape bsz x n_heads x N x N
								# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
								# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
								UpperCAmelCase				: List[Any]											=	self.get_head_mask(__snake_case	,    self.config.num_hidden_layers	)

								UpperCAmelCase				: Any											=	self.embeddings(
								    input_ids=__snake_case	,    position_ids=__snake_case	,    token_type_ids=__snake_case	,    inputs_embeds=__snake_case	)
								UpperCAmelCase				: Optional[int]											=	embedding_output

								if self.training:
															UpperCAmelCase				: str											=	[]
															for i in range(self.config.num_hidden_layers	):
																						UpperCAmelCase				: Optional[int]											=	self.encoder.adaptive_forward(
																						    __snake_case	,    current_layer=__snake_case	,    attention_mask=__snake_case	,    head_mask=__snake_case	)

																						UpperCAmelCase				: List[str]											=	self.pooler(__snake_case	)
																						UpperCAmelCase				: Dict											=	output_layers[i](output_dropout(__snake_case	)	)
																						res.append(__snake_case	)
								elif self.patience == 0:  # Use all layers for inference
															UpperCAmelCase				: Any											=	self.encoder(
															    __snake_case	,    attention_mask=__snake_case	,    head_mask=__snake_case	,    encoder_hidden_states=__snake_case	,    encoder_attention_mask=__snake_case	,    )
															UpperCAmelCase				: Dict											=	self.pooler(encoder_outputs[0]	)
															UpperCAmelCase				: List[Any]											=	[output_layers[self.config.num_hidden_layers - 1](__snake_case	)]
								else:
															UpperCAmelCase				: Tuple											=	0
															UpperCAmelCase				: List[str]											=	None
															UpperCAmelCase				: Optional[Any]											=	0
															for i in range(self.config.num_hidden_layers	):
																						calculated_layer_num += 1
																						UpperCAmelCase				: Optional[Any]											=	self.encoder.adaptive_forward(
																						    __snake_case	,    current_layer=__snake_case	,    attention_mask=__snake_case	,    head_mask=__snake_case	)

																						UpperCAmelCase				: List[str]											=	self.pooler(__snake_case	)
																						UpperCAmelCase				: Optional[int]											=	output_layers[i](__snake_case	)
																						if regression:
																													UpperCAmelCase				: Union[str, Any]											=	logits.detach()
																													if patient_result is not None:
																																				UpperCAmelCase				: List[str]											=	patient_result.detach()
																													if (patient_result is not None) and torch.abs(patient_result - labels	) < self.regression_threshold:
																																				patient_counter += 1
																													else:
																																				UpperCAmelCase				: Dict											=	0
																						else:
																													UpperCAmelCase				: Union[str, Any]											=	logits.detach().argmax(dim=1	)
																													if patient_result is not None:
																																				UpperCAmelCase				: str											=	patient_result.detach().argmax(dim=1	)
																													if (patient_result is not None) and torch.all(labels.eq(__snake_case	)	):
																																				patient_counter += 1
																													else:
																																				UpperCAmelCase				: Union[str, Any]											=	0

																						UpperCAmelCase				: List[str]											=	logits
																						if patient_counter == self.patience:
																													break
															UpperCAmelCase				: List[Any]											=	[patient_result]
															self.inference_layers_num += calculated_layer_num
															self.inference_instances_num += 1

								return res

@add_start_docstrings(
    """Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. """							,	A__							,	)
class        SCREAMING_SNAKE_CASE(							A__    ):







	"""simple docstring"""






	def __init__(		self    :  str	,    __snake_case    :  str	)    ->	Optional[Any]:
								super().__init__(__snake_case	)
								UpperCAmelCase				: Optional[int]											=	config.num_labels

								UpperCAmelCase				: Optional[Any]											=	BertModelWithPabee(__snake_case	)
								UpperCAmelCase				: Tuple											=	nn.Dropout(config.hidden_dropout_prob	)
								UpperCAmelCase				: Dict											=	nn.ModuleList(
								    [nn.Linear(config.hidden_size	,    self.config.num_labels	) for _ in range(config.num_hidden_layers	)]	)

								self.init_weights()







	@add_start_docstrings_to_model_forward(__snake_case	)
	def   A			(		self    :  Any	,    __snake_case    :  Dict=None	,    __snake_case    :  str=None	,    __snake_case    :  List[Any]=None	,    __snake_case    :  List[Any]=None	,    __snake_case    :  Union[str, Any]=None	,    __snake_case    :  int=None	,    __snake_case    :  str=None	,    )    ->	Union[str, Any]:
								UpperCAmelCase				: Optional[int]											=	self.bert(
								    input_ids=__snake_case	,    attention_mask=__snake_case	,    token_type_ids=__snake_case	,    position_ids=__snake_case	,    head_mask=__snake_case	,    inputs_embeds=__snake_case	,    output_dropout=self.dropout	,    output_layers=self.classifiers	,    regression=self.num_labels == 1	,    )

								UpperCAmelCase				: List[str]											=	(logits[-1],)

								if labels is not None:
															UpperCAmelCase				: Tuple											=	None
															UpperCAmelCase				: Any											=	0
															for ix, logits_item in enumerate(__snake_case	):
																						if self.num_labels == 1:
																													#  We are doing regression
																													UpperCAmelCase				: Optional[Any]											=	MSELoss()
																													UpperCAmelCase				: Optional[Any]											=	loss_fct(logits_item.view(-1	)	,    labels.view(-1	)	)
																						else:
																													UpperCAmelCase				: Dict											=	CrossEntropyLoss()
																													UpperCAmelCase				: Dict											=	loss_fct(logits_item.view(-1	,    self.num_labels	)	,    labels.view(-1	)	)
																						if total_loss is None:
																													UpperCAmelCase				: Dict											=	loss
																						else:
																													total_loss += loss * (ix + 1)
																						total_weights += ix + 1
															UpperCAmelCase				: Any											=	(total_loss / total_weights,) + outputs

								return outputs





