

import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin



class        A__(a_,  unittest.TestCase							):
						"""simple docstring"""






						_A    :				Dict       					=			CanineTokenizer
						_A    :				Any       					=			False





						def 			UpperCamelCase__					(    self    )     -> Optional[Any]:
									super().setUp()
									a_						:						Optional[int] 							= CanineTokenizer()
									tokenizer.save_pretrained(self.tmpdirname    )





						@cached_property
						def 			UpperCamelCase__					(    self    )     -> Union[str, Any]:
									return CanineTokenizer.from_pretrained("""google/canine-s"""    )





						def 			UpperCamelCase__					(    self  ,      **_lowercase    )     -> CanineTokenizer:
									a_						:						List[str] 							= self.tokenizer_class.from_pretrained(self.tmpdirname  ,      **_lowercase    )
									a_						:						Optional[Any] 							= 1_024
									return tokenizer





						@require_torch
						def 			UpperCamelCase__					(    self    )     -> Optional[int]:
									a_						:						Optional[int] 							= self.canine_tokenizer
									a_						:						Any 							= ["""Life is like a box of chocolates.""", """You never know what you're gonna get."""]
									# fmt: off
									a_						:						Tuple 							= [57_344, 76, 105, 102, 101, 32, 105, 115, 32, 108, 105, 107, 101, 32, 97, 32, 98, 111, 120, 32, 111, 102, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 115, 46, 57_345, 0, 0, 0, 0]
									# fmt: on
									a_						:						Tuple 							= tokenizer(_lowercase  ,      padding=_lowercase  ,      return_tensors="""pt"""    )
									self.assertIsInstance(_lowercase  ,      _lowercase    )

									a_						:						Any 							= list(batch.input_ids.numpy()[0]    )

									self.assertListEqual(_lowercase  ,      _lowercase    )

									self.assertEqual((2, 39)  ,      batch.input_ids.shape    )
									self.assertEqual((2, 39)  ,      batch.attention_mask.shape    )





						@require_torch
						def 			UpperCamelCase__					(    self    )     -> List[str]:
									a_						:						str 							= self.canine_tokenizer
									a_						:						int 							= ["""Once there was a man.""", """He wrote a test in HuggingFace Tranformers."""]
									a_						:						Optional[int] 							= tokenizer(_lowercase  ,      padding=_lowercase  ,      return_tensors="""pt"""    )
									# check if input_ids, attention_mask and token_type_ids are returned
									self.assertIn("""input_ids"""  ,      _lowercase    )
									self.assertIn("""attention_mask"""  ,      _lowercase    )
									self.assertIn("""token_type_ids"""  ,      _lowercase    )





						@require_torch
						def 			UpperCamelCase__					(    self    )     -> Dict:
									a_						:						str 							= self.canine_tokenizer
									a_						:						str 							= [
									    """What's the weater?""",
									    """It's about 25 degrees.""",
									]
									a_						:						Any 							= tokenizer(
									    text_target=_lowercase  ,      max_length=32  ,      padding="""max_length"""  ,      truncation=_lowercase  ,      return_tensors="""pt"""    )
									self.assertEqual(32  ,      targets["""input_ids"""].shape[1]    )





						def 			UpperCamelCase__					(    self    )     -> Optional[int]:
									# safety check on max_len default value so we are sure the test works
									a_						:						List[str] 							= self.get_tokenizers()
									for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''    ):
															self.assertNotEqual(tokenizer.model_max_length  ,      42    )

        # Now let's start the test
									a_						:						Union[str, Any] 							= self.get_tokenizers()
									for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''    ):
															# Isolate this from the other tests because we save additional tokens/etc
															a_						:						Tuple 							= tempfile.mkdtemp()

															a_						:						Optional[int] 							= """ He is very happy, UNwant\u00E9d,running"""
															a_						:						int 							= tokenizer.encode(_lowercase  ,      add_special_tokens=_lowercase    )
															tokenizer.save_pretrained(_lowercase    )

															a_						:						str 							= tokenizer.__class__.from_pretrained(_lowercase    )
															a_						:						int 							= after_tokenizer.encode(_lowercase  ,      add_special_tokens=_lowercase    )
															self.assertListEqual(_lowercase  ,      _lowercase    )

															shutil.rmtree(_lowercase    )

									a_						:						Optional[int] 							= self.get_tokenizers(model_max_length=42    )
									for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''    ):
															# Isolate this from the other tests because we save additional tokens/etc
															a_						:						Optional[Any] 							= tempfile.mkdtemp()

															a_						:						Optional[Any] 							= """ He is very happy, UNwant\u00E9d,running"""

															a_						:						Any 							= tokenizer.additional_special_tokens

															# We can add a new special token for Canine as follows:
															a_						:						List[str] 							= chr(0Xe0_07    )
															additional_special_tokens.append(_lowercase    )
															tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}    )
															a_						:						Tuple 							= tokenizer.encode(_lowercase  ,      add_special_tokens=_lowercase    )
															tokenizer.save_pretrained(_lowercase    )

															a_						:						str 							= tokenizer.__class__.from_pretrained(_lowercase    )
															a_						:						Tuple 							= after_tokenizer.encode(_lowercase  ,      add_special_tokens=_lowercase    )
															self.assertListEqual(_lowercase  ,      _lowercase    )
															self.assertIn(_lowercase  ,      after_tokenizer.additional_special_tokens    )
															self.assertEqual(after_tokenizer.model_max_length  ,      42    )

															a_						:						List[Any] 							= tokenizer.__class__.from_pretrained(_lowercase  ,      model_max_length=43    )
															self.assertEqual(tokenizer.model_max_length  ,      43    )

															shutil.rmtree(_lowercase    )





						def 			UpperCamelCase__					(    self    )     -> Any:
									a_						:						str 							= self.get_tokenizers(do_lower_case=_lowercase    )
									for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''    ):
															a_							,					a_						:						Tuple 							= self.get_clean_sequence(_lowercase    )

															# a special token for Canine can be defined as follows:
															a_						:						str 							= 0Xe0_05
															a_						:						Union[str, Any] 							= chr(_lowercase    )

															tokenizer.add_special_tokens({"""cls_token""": special_token}    )
															a_						:						List[str] 							= tokenizer.encode(_lowercase  ,      add_special_tokens=_lowercase    )
															self.assertEqual(len(_lowercase    )  ,      1    )

															a_						:						Dict 							= tokenizer.decode(ids + encoded_special_token  ,      clean_up_tokenization_spaces=_lowercase    )
															a_						:						Union[str, Any] 							= tokenizer.encode(_lowercase  ,      add_special_tokens=_lowercase    )

															a_						:						int 							= tokenizer.encode(_lowercase  ,      add_special_tokens=_lowercase    )
															a_						:						str 							= tokenizer.encode(_lowercase  ,      add_special_tokens=_lowercase    )
															self.assertEqual(_lowercase  ,      input_encoded + special_token_id    )

															a_						:						Dict 							= tokenizer.decode(_lowercase  ,      skip_special_tokens=_lowercase    )
															self.assertTrue(special_token not in decoded    )





						def 			UpperCamelCase__					(    self    )     -> str:
									a_						:						Optional[Any] 							= self.get_tokenizers(do_lower_case=_lowercase    )
									for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''    ):
															a_						:						Optional[Any] 							= chr(0Xe0_05    )
															a_						:						str 							= chr(0Xe0_06    )

															# `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
															tokenizer.add_tokens([SPECIAL_TOKEN_1]  ,      special_tokens=_lowercase    )
															# `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
															# which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
															tokenizer.add_special_tokens({"""additional_special_tokens""": [SPECIAL_TOKEN_2]}    )

															a_						:						Optional[Any] 							= tokenizer.tokenize(_lowercase    )
															a_						:						Any 							= tokenizer.tokenize(_lowercase    )

															self.assertEqual(len(_lowercase    )  ,      1    )
															self.assertEqual(len(_lowercase    )  ,      1    )
															self.assertEqual(token_a[0]  ,      _lowercase    )
															self.assertEqual(token_a[0]  ,      _lowercase    )





						@require_tokenizers
						def 			UpperCamelCase__					(    self    )     -> List[str]:
									a_						:						Union[str, Any] 							= self.get_tokenizers(do_lower_case=_lowercase    )
									for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''    ):
															# a special token for Canine can be defined as follows:
															a_						:						str 							= 0Xe0_06
															a_						:						Dict 							= chr(_lowercase    )

															a_						:						Any 							= AddedToken(_lowercase  ,      lstrip=_lowercase    )
															tokenizer.add_special_tokens({"""additional_special_tokens""": [new_token]}    )

															with tempfile.TemporaryDirectory() as tmp_dir_name:
																		tokenizer.save_pretrained(_lowercase    )
																		tokenizer.from_pretrained(_lowercase    )





						def 			UpperCamelCase__					(    self    )     -> Dict:
									a_						:						Dict 							= []
									if self.test_slow_tokenizer:
												tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())    )

									if self.test_rust_tokenizer:
												tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())    )

									for tokenizer_class, tokenizer_utils in tokenizer_list:
												with tempfile.TemporaryDirectory() as tmp_dir:
															tokenizer_utils.save_pretrained(_lowercase    )

															with open(os.path.join(_lowercase  ,      """special_tokens_map.json"""    )  ,      encoding="""utf-8"""    ) as json_file:
																		a_						:						Tuple 							= json.load(_lowercase    )

															with open(os.path.join(_lowercase  ,      """tokenizer_config.json"""    )  ,      encoding="""utf-8"""    ) as json_file:
																		a_						:						Dict 							= json.load(_lowercase    )

															# a special token for Canine can be defined as follows:
															a_						:						Optional[Any] 							= 0Xe0_06
															a_						:						Dict 							= chr(_lowercase    )

															a_						:						List[str] 							= [new_token_a]
															a_						:						List[Any] 							= [new_token_a]

															with open(os.path.join(_lowercase  ,      """special_tokens_map.json"""    )  ,      """w"""  ,      encoding="""utf-8"""    ) as outfile:
																		json.dump(_lowercase  ,      _lowercase    )
															with open(os.path.join(_lowercase  ,      """tokenizer_config.json"""    )  ,      """w"""  ,      encoding="""utf-8"""    ) as outfile:
																		json.dump(_lowercase  ,      _lowercase    )

															# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
															# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
															# "special_tokens_map.json" files
															a_						:						str 							= tokenizer_class.from_pretrained(_lowercase  ,      extra_ids=0    )
															self.assertIn(_lowercase  ,      tokenizer_without_change_in_init.additional_special_tokens    )
															# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
															self.assertEqual(
															    [new_token_a]  ,      tokenizer_without_change_in_init.convert_ids_to_tokens(
															        tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]    )    )  ,      )

															a_						:						Any 							= 0Xe0_07
															a_						:						Dict 							= chr(_lowercase    )
															# Now we test that we can change the value of additional_special_tokens in the from_pretrained
															a_						:						Union[str, Any] 							= [AddedToken(_lowercase  ,      lstrip=_lowercase    )]
															a_						:						List[Any] 							= tokenizer_class.from_pretrained(
															    _lowercase  ,      additional_special_tokens=_lowercase  ,      extra_ids=0    )

															self.assertIn(_lowercase  ,      tokenizer.additional_special_tokens    )
															# self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
															self.assertEqual(
															    [new_token_a]  ,      tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]    )    )    )





						@require_tokenizers
						def 			UpperCamelCase__					(    self    )     -> List[str]:
									a_						:						Tuple 							= self.get_tokenizers(do_lower_case=_lowercase    )
									for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''    ):
															a_						:						Tuple 							= """hello world"""
															if self.space_between_special_tokens:
																		a_						:						Dict 							= """[CLS] hello world [SEP]"""
															else:
																		a_						:						Dict 							= input
															a_						:						int 							= tokenizer.encode(_lowercase  ,      add_special_tokens=_lowercase    )
															a_						:						Any 							= tokenizer.decode(_lowercase  ,      spaces_between_special_tokens=self.space_between_special_tokens    )
															self.assertIn(_lowercase  ,      [output, output.lower()]    )





						def 			UpperCamelCase__					(    self    )     -> Union[str, Any]:
									a_						:						Optional[int] 							= self.get_tokenizers()
									for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''    ):
															a_						:						Union[str, Any] 							= [
															    """bos_token""",
															    """eos_token""",
															    """unk_token""",
															    """sep_token""",
															    """pad_token""",
															    """cls_token""",
															    """mask_token""",
															]

									a_						:						Optional[Any] 							= """a"""
									a_						:						List[Any] 							= ord(_lowercase    )

									for attr in attributes_list:
												setattr(_lowercase  ,      attr + """_id"""  ,      _lowercase    )
												self.assertEqual(getattr(_lowercase  ,      _lowercase    )  ,      _lowercase    )
												self.assertEqual(getattr(_lowercase  ,      attr + """_id"""    )  ,      _lowercase    )

												setattr(_lowercase  ,      attr + """_id"""  ,      _lowercase    )
												self.assertEqual(getattr(_lowercase  ,      _lowercase    )  ,      _lowercase    )
												self.assertEqual(getattr(_lowercase  ,      attr + """_id"""    )  ,      _lowercase    )

									setattr(_lowercase  ,      """additional_special_tokens_ids"""  ,      []    )
									self.assertListEqual(getattr(_lowercase  ,      """additional_special_tokens"""    )  ,      []    )
									self.assertListEqual(getattr(_lowercase  ,      """additional_special_tokens_ids"""    )  ,      []    )

									a_						:						str 							= 0Xe0_06
									a_						:						str 							= chr(_lowercase    )
									setattr(_lowercase  ,      """additional_special_tokens_ids"""  ,      [additional_special_token_id]    )
									self.assertListEqual(getattr(_lowercase  ,      """additional_special_tokens"""    )  ,      [additional_special_token]    )
									self.assertListEqual(getattr(_lowercase  ,      """additional_special_tokens_ids"""    )  ,      [additional_special_token_id]    )





						def 			UpperCamelCase__					(    self    )     -> Any:
									pass





						def 			UpperCamelCase__					(    self    )     -> int:
									pass





						def 			UpperCamelCase__					(    self    )     -> int:
									pass





						def 			UpperCamelCase__					(    self    )     -> int:
									pass





						def 			UpperCamelCase__					(    self    )     -> Dict:
									pass





						def 			UpperCamelCase__					(    self    )     -> Optional[int]:
									pass





						def 			UpperCamelCase__					(    self    )     -> List[Any]:
									pass





						def 			UpperCamelCase__					(    self    )     -> Union[str, Any]:
									pass





import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


__snake_case				:      Optional[int]   		=			logging.getLogger(__name__)



class        A__(a_							):
						"""simple docstring"""






						def 			UpperCamelCase__					(    self  ,      _lowercase  ,      _lowercase  ,      _lowercase=None  ,      _lowercase=None    )     -> Tuple:
									a_						:						str 							= self.layer[current_layer](_lowercase  ,      _lowercase  ,      head_mask[current_layer]    )

									a_						:						str 							= layer_outputs[0]

									return hidden_states







@add_start_docstrings(
    '''The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top.''',  a_,  )
class        A__(a_							):
						"""simple docstring"""






						def __init__(    self  ,      _lowercase    )     -> List[str]:
									super().__init__(_lowercase    )

									a_						:						Tuple 							= BertEncoderWithPabee(_lowercase    )

									self.init_weights()
									a_						:						int 							= 0
									a_						:						Any 							= 0
									a_						:						Tuple 							= 0

									a_						:						Optional[int] 							= 0





						def 			UpperCamelCase__					(    self  ,      _lowercase    )     -> Tuple:
									a_						:						Dict 							= threshold





						def 			UpperCamelCase__					(    self  ,      _lowercase    )     -> List[Any]:
									a_						:						Optional[int] 							= patience





						def 			UpperCamelCase__					(    self    )     -> Dict:
									a_						:						str 							= 0
									a_						:						Optional[int] 							= 0





						def 			UpperCamelCase__					(    self    )     -> List[Any]:
									a_						:						Union[str, Any] 							= self.inference_layers_num / self.inference_instances_num
									a_						:						Optional[int] 							= (
									    F'''*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='''
									    F''' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'''
									)
									print(_lowercase    )





						@add_start_docstrings_to_model_forward(_lowercase    )
						def 			UpperCamelCase__					(    self  ,      _lowercase=None  ,      _lowercase=None  ,      _lowercase=None  ,      _lowercase=None  ,      _lowercase=None  ,      _lowercase=None  ,      _lowercase=None  ,      _lowercase=None  ,      _lowercase=None  ,      _lowercase=None  ,      _lowercase=False  ,      )     -> str:

									if input_ids is not None and inputs_embeds is not None:
												raise ValueError("""You cannot specify both input_ids and inputs_embeds at the same time"""    )
									elif input_ids is not None:
												a_						:						Dict 							= input_ids.size()
									elif inputs_embeds is not None:
												a_						:						Dict 							= inputs_embeds.size()[:-1]
									else:
												raise ValueError("""You have to specify either input_ids or inputs_embeds"""    )

									a_						:						Optional[Any] 							= input_ids.device if input_ids is not None else inputs_embeds.device

									if attention_mask is None:
												a_						:						Tuple 							= torch.ones(_lowercase  ,      device=_lowercase    )
									if token_type_ids is None:
												a_						:						List[str] 							= torch.zeros(_lowercase  ,      dtype=torch.long  ,      device=_lowercase    )

									# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
									# ourselves in which case we just need to make it broadcastable to all heads.
									a_						:						torch.Tensor 							= self.get_extended_attention_mask(_lowercase  ,      _lowercase  ,      _lowercase    )

									# If a 2D ou 3D attention mask is provided for the cross-attention
									# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
									if self.config.is_decoder and encoder_hidden_states is not None:
												a_							,					a_							,					a_						:						int 							= encoder_hidden_states.size()
												a_						:						List[str] 							= (encoder_batch_size, encoder_sequence_length)
												if encoder_attention_mask is None:
															a_						:						Tuple 							= torch.ones(_lowercase  ,      device=_lowercase    )
												a_						:						List[Any] 							= self.invert_attention_mask(_lowercase    )
									else:
												a_						:						Optional[Any] 							= None

									# Prepare head mask if needed
									# 1.0 in head_mask indicate we keep the head
									# attention_probs has shape bsz x n_heads x N x N
									# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
									# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
									a_						:						List[Any] 							= self.get_head_mask(_lowercase  ,      self.config.num_hidden_layers    )

									a_						:						List[str] 							= self.embeddings(
									    input_ids=_lowercase  ,      position_ids=_lowercase  ,      token_type_ids=_lowercase  ,      inputs_embeds=_lowercase    )
									a_						:						List[Any] 							= embedding_output

									if self.training:
												a_						:						Any 							= []
												for i in range(self.config.num_hidden_layers    ):
															a_						:						int 							= self.encoder.adaptive_forward(
															    _lowercase  ,      current_layer=_lowercase  ,      attention_mask=_lowercase  ,      head_mask=_lowercase    )

															a_						:						List[Any] 							= self.pooler(_lowercase    )
															a_						:						Optional[int] 							= output_layers[i](output_dropout(_lowercase    )    )
															res.append(_lowercase    )
									elif self.patience == 0:  # Use all layers for inference
												a_						:						Union[str, Any] 							= self.encoder(
												    _lowercase  ,      attention_mask=_lowercase  ,      head_mask=_lowercase  ,      encoder_hidden_states=_lowercase  ,      encoder_attention_mask=_lowercase  ,      )
												a_						:						Union[str, Any] 							= self.pooler(encoder_outputs[0]    )
												a_						:						List[str] 							= [output_layers[self.config.num_hidden_layers - 1](_lowercase    )]
									else:
												a_						:						Any 							= 0
												a_						:						Dict 							= None
												a_						:						Tuple 							= 0
												for i in range(self.config.num_hidden_layers    ):
															calculated_layer_num += 1
															a_						:						Optional[Any] 							= self.encoder.adaptive_forward(
															    _lowercase  ,      current_layer=_lowercase  ,      attention_mask=_lowercase  ,      head_mask=_lowercase    )

															a_						:						Optional[int] 							= self.pooler(_lowercase    )
															a_						:						int 							= output_layers[i](_lowercase    )
															if regression:
																		a_						:						Dict 							= logits.detach()
																		if patient_result is not None:
																					a_						:						Optional[Any] 							= patient_result.detach()
																		if (patient_result is not None) and torch.abs(patient_result - labels    ) < self.regression_threshold:
																					patient_counter += 1
																		else:
																					a_						:						Optional[Any] 							= 0
															else:
																		a_						:						str 							= logits.detach().argmax(dim=1    )
																		if patient_result is not None:
																					a_						:						str 							= patient_result.detach().argmax(dim=1    )
																		if (patient_result is not None) and torch.all(labels.eq(_lowercase    )    ):
																					patient_counter += 1
																		else:
																					a_						:						Tuple 							= 0

															a_						:						Union[str, Any] 							= logits
															if patient_counter == self.patience:
																		break
												a_						:						str 							= [patient_result]
												self.inference_layers_num += calculated_layer_num
												self.inference_instances_num += 1

									return res







@add_start_docstrings(
    '''Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. ''',  a_,  )
class        A__(a_							):
						"""simple docstring"""






						def __init__(    self  ,      _lowercase    )     -> str:
									super().__init__(_lowercase    )
									a_						:						str 							= config.num_labels

									a_						:						Optional[Any] 							= BertModelWithPabee(_lowercase    )
									a_						:						int 							= nn.Dropout(config.hidden_dropout_prob    )
									a_						:						str 							= nn.ModuleList(
									    [nn.Linear(config.hidden_size  ,      self.config.num_labels    ) for _ in range(config.num_hidden_layers    )]    )

									self.init_weights()





						@add_start_docstrings_to_model_forward(_lowercase    )
						def 			UpperCamelCase__					(    self  ,      _lowercase=None  ,      _lowercase=None  ,      _lowercase=None  ,      _lowercase=None  ,      _lowercase=None  ,      _lowercase=None  ,      _lowercase=None  ,      )     -> Tuple:
									a_						:						Optional[Any] 							= self.bert(
									    input_ids=_lowercase  ,      attention_mask=_lowercase  ,      token_type_ids=_lowercase  ,      position_ids=_lowercase  ,      head_mask=_lowercase  ,      inputs_embeds=_lowercase  ,      output_dropout=self.dropout  ,      output_layers=self.classifiers  ,      regression=self.num_labels == 1  ,      )

									a_						:						Optional[Any] 							= (logits[-1],)

									if labels is not None:
												a_						:						int 							= None
												a_						:						Union[str, Any] 							= 0
												for ix, logits_item in enumerate(_lowercase    ):
															if self.num_labels == 1:
																		#  We are doing regression
																		a_						:						Any 							= MSELoss()
																		a_						:						Union[str, Any] 							= loss_fct(logits_item.view(-1    )  ,      labels.view(-1    )    )
															else:
																		a_						:						Any 							= CrossEntropyLoss()
																		a_						:						int 							= loss_fct(logits_item.view(-1  ,      self.num_labels    )  ,      labels.view(-1    )    )
															if total_loss is None:
																		a_						:						str 							= loss
															else:
																		total_loss += loss * (ix + 1)
															total_weights += ix + 1
												a_						:						Any 							= (total_loss / total_weights,) + outputs

									return outputs


