




import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing the experiment tracking capability,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################

lowercase__							:	Tuple											=		16
lowercase__							:	List[Any]											=		32


def 		lowerCamelCase__       (						_A	,      _A = 16				):







	'''simple docstring'''





	snake_case_						       =	AutoTokenizer.from_pretrained("bert-base-cased"				)
	snake_case_						       =	load_dataset("glue"	,      "mrpc"				)

	def tokenize_function(_A				):
		# max_length=None => use the model max length (it's actually the default)
		snake_case_						       =	tokenizer(examples["sentence1"]	,      examples["sentence2"]	,      truncation=_A	,      max_length=_A				)
		return outputs

	# Apply the method we just defined to all the examples in all the splits of the dataset
	# starting with the main process first:
	with accelerator.main_process_first():
		snake_case_						       =	datasets.map(
		    _A	,      batched=_A	,      remove_columns=["idx", "sentence1", "sentence2"]	,      )

	# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
	# transformers library
	snake_case_						       =	tokenized_datasets.rename_column("label"	,      "labels"				)

	def collate_fn(_A				):
		# On TPU it's best to pad everything to the same length or training will be very slow.
		snake_case_						       =	128 if accelerator.distributed_type == DistributedType.TPU else None
		# When using mixed precision we want round multiples of 8/16
		if accelerator.mixed_precision == "fp8":
			snake_case_						       =	16
		elif accelerator.mixed_precision != "no":
			snake_case_						       =	8
		else:
			snake_case_						       =	None

		return tokenizer.pad(
		    _A	,      padding="longest"	,      max_length=_A	,      pad_to_multiple_of=_A	,      return_tensors="pt"	,      )

	# Instantiate dataloaders.
	snake_case_						       =	DataLoader(
	    tokenized_datasets["train"]	,      shuffle=_A	,      collate_fn=_A	,      batch_size=_A				)
	snake_case_						       =	DataLoader(
	    tokenized_datasets["validation"]	,      shuffle=_A	,      collate_fn=_A	,      batch_size=_A				)

	return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("TESTING_MOCKED_DATALOADERS", None) == "1":
		from accelerate.test_utils.training import mocked_dataloaders

		lowercase__							:	Union[str, Any]											=		mocked_dataloaders  # noqa: F811


def 		lowerCamelCase__       (						_A	,      _A				):







	'''simple docstring'''





	if os.environ.get("TESTING_MOCKED_DATALOADERS"	,      _A				) == "1":
		snake_case_						       =	2
	# Initialize Accelerator

	# New Code #
	# We pass in "all" to `log_with` to grab all available trackers in the environment
	# Note: If using a custom `Tracker` class, should be passed in here such as:
	# >>> log_with = ["all", MyCustomTrackerClassInstance()]
	if args.with_tracking:
		snake_case_						       =	Accelerator(
		    cpu=args.cpu	,      mixed_precision=args.mixed_precision	,      log_with="all"	,      project_dir=args.project_dir				)
	else:
		snake_case_						       =	Accelerator(cpu=args.cpu	,      mixed_precision=args.mixed_precision				)
	# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
	snake_case_						       =	config["lr"]
	snake_case_						       =	int(config["num_epochs"]				)
	snake_case_						       =	int(config["seed"]				)
	snake_case_						       =	int(config["batch_size"]				)
	set_seed(_A				)

	snake_case_       , snake_case_						       =	get_dataloaders(_A	,      _A				)
	snake_case_						       =	evaluate.load("glue"	,      "mrpc"				)

	# If the batch size is too big we use gradient accumulation
	snake_case_						       =	1
	if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
		snake_case_						       =	batch_size // MAX_GPU_BATCH_SIZE
		snake_case_						       =	MAX_GPU_BATCH_SIZE

	# Instantiate the model (we build the model here so that the seed also control new weights initialization)
	snake_case_						       =	AutoModelForSequenceClassification.from_pretrained("bert-base-cased"	,      return_dict=_A				)

	# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
	# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
	# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
	snake_case_						       =	model.to(accelerator.device				)

	# Instantiate optimizer
	snake_case_						       =	AdamW(params=model.parameters()	,      lr=_A				)

	# Instantiate scheduler
	snake_case_						       =	get_linear_schedule_with_warmup(
	    optimizer=_A	,      num_warmup_steps=100	,      num_training_steps=(len(_A				) * num_epochs) // gradient_accumulation_steps	,      )

	# Prepare everything
	# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
	# prepare method.
	snake_case_       , snake_case_       , snake_case_       , snake_case_       , snake_case_						       =	accelerator.prepare(
	    _A	,      _A	,      _A	,      _A	,      _A				)

	# New Code #
	# We need to initialize the trackers we use. Overall configurations can also be stored
	if args.with_tracking:
		snake_case_						       =	os.path.split(_A				)[-1].split("."				)[0]
		accelerator.init_trackers(_A	,      _A				)

	# Now we train the model
	for epoch in range(_A				):
		model.train()
		# New Code #
		# For our tracking example, we will log the total loss of each epoch
		if args.with_tracking:
			snake_case_						       =	0
		for step, batch in enumerate(_A				):
			# We could avoid this line since we set the accelerator with `device_placement=True`.
			batch.to(accelerator.device				)
			snake_case_						       =	model(**_A				)
			snake_case_						       =	outputs.loss
			# New Code #
			if args.with_tracking:
				total_loss += loss.detach().float()
			snake_case_						       =	loss / gradient_accumulation_steps
			accelerator.backward(_A				)
			if step % gradient_accumulation_steps == 0:
				optimizer.step()
				lr_scheduler.step()
				optimizer.zero_grad()

		model.eval()
		for step, batch in enumerate(_A				):
			# We could avoid this line since we set the accelerator with `device_placement=True` (the default).
			batch.to(accelerator.device				)
			with torch.no_grad():
				snake_case_						       =	model(**_A				)
			snake_case_						       =	outputs.logits.argmax(dim=-1				)
			snake_case_       , snake_case_						       =	accelerator.gather_for_metrics((predictions, batch["labels"])				)
			metric.add_batch(
			    predictions=_A	,      references=_A	,      )

		snake_case_						       =	metric.compute()
		# Use accelerator.print to print only on the main process.
		accelerator.print(f"epoch {epoch}:"	,      _A				)

		# New Code #
		# To actually log, we call `Accelerator.log`
		# The values passed can be of `str`, `int`, `float` or `dict` of `str` to `float`/`int`
		if args.with_tracking:
			accelerator.log(
			    {
			        "accuracy": eval_metric["accuracy"],
			        "f1": eval_metric["f1"],
			        "train_loss": total_loss.item() / len(_A				),
			        "epoch": epoch,
			    }	,      step=_A	,      )

    # New Code #
    # When a run is finished, you should call `accelerator.end_training()`
    # to close all of the open trackers
	if args.with_tracking:
		accelerator.end_training()


def 		lowerCamelCase__       (						):







	'''simple docstring'''





	snake_case_						       =	argparse.ArgumentParser(description="Simple example of training script."				)
	parser.add_argument(
	    "--mixed_precision"	,      type=_A	,      default=_A	,      choices=["no", "fp16", "bf16", "fp8"]	,      help="Whether to use mixed precision. Choose"
	    "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."
	    "and an Nvidia Ampere GPU."	,      )
	parser.add_argument("--cpu"	,      action="store_true"	,      help="If passed, will train on the CPU."				)
	parser.add_argument(
	    "--with_tracking"	,      action="store_true"	,      help="Whether to load in all available experiment trackers from the environment and use them for logging."	,      )
	parser.add_argument(
	    "--project_dir"	,      type=_A	,      default="logs"	,      help="Location on where to store experiment tracking logs` and relevent project information"	,      )
	snake_case_						       =	parser.parse_args()
	snake_case_						       =	{"lr": 2E-5, "num_epochs": 3, "seed": 42, "batch_size": 16}
	training_function(_A	,      _A				)


if __name__ == "__main__":
		main()






import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEmbeddings,
    BertLayer,
    BertPooler,
    BertPreTrainedModel,
)


def 		lowerCamelCase__       (						_A				):







	'''simple docstring'''





	snake_case_						       =	torch.exp(_A				)
	snake_case_						       =	torch.sum(_A	,      dim=1				)  # sum of exp(x_i)
	snake_case_						       =	torch.sum(x * exp_x	,      dim=1				)  # sum of x_i * exp(x_i)
	return torch.log(_A				) - B / A
class 						UpperCAmelCase  (							nn.Module     ):







						'''simple docstring'''





						def __init__(		self				:	Optional[int]			,      __lowercase				:	List[Any]		):






							"""simple docstring"""






							super().__init__()
							snake_case_						       =	config.output_attentions
							snake_case_						       =	config.output_hidden_states
							snake_case_						       =	nn.ModuleList([BertLayer(__lowercase		) for _ in range(config.num_hidden_layers		)]		)
							snake_case_						       =	nn.ModuleList([BertHighway(__lowercase		) for _ in range(config.num_hidden_layers		)]		)

							snake_case_						       =	[-1 for _ in range(config.num_hidden_layers		)]






						def 			snake_case__		(		self				:	Union[str, Any]			,      __lowercase				:	Optional[int]		):






							"""simple docstring"""






							if (type(__lowercase		) is float) or (type(__lowercase		) is int):
								for i in range(len(self.early_exit_entropy		)		):
									snake_case_						       =	x
							else:
								snake_case_						       =	x






						def 			snake_case__		(		self				:	int			,      __lowercase				:	Tuple		):






							"""simple docstring"""






							snake_case_						       =	pooler.state_dict()
							for highway in self.highway:
								for name, param in highway.pooler.state_dict().items():
									param.copy_(loaded_model[name]		)






						def 			snake_case__		(		self				:	List[Any]			,      __lowercase				:	List[str]			,      __lowercase				:	List[Any]=None			,      __lowercase				:	Optional[int]=None			,      __lowercase				:	Optional[Any]=None			,      __lowercase				:	List[str]=None			,      ):






							"""simple docstring"""






							snake_case_						       =	()
							snake_case_						       =	()
							snake_case_						       =	()
							for i, layer_module in enumerate(self.layer		):
								if self.output_hidden_states:
									snake_case_						       =	all_hidden_states + (hidden_states,)

								snake_case_						       =	layer_module(
								    __lowercase			,      __lowercase			,      head_mask[i]			,      __lowercase			,      __lowercase		)
								snake_case_						       =	layer_outputs[0]

								if self.output_attentions:
									snake_case_						       =	all_attentions + (layer_outputs[1],)

								snake_case_						       =	(hidden_states,)
								if self.output_hidden_states:
									snake_case_						       =	current_outputs + (all_hidden_states,)
								if self.output_attentions:
									snake_case_						       =	current_outputs + (all_attentions,)

								snake_case_						       =	self.highway[i](__lowercase		)
								# logits, pooled_output

								if not self.training:
									snake_case_						       =	highway_exit[0]
									snake_case_						       =	entropy(__lowercase		)
									snake_case_						       =	highway_exit + (highway_entropy,)  # logits, hidden_states(?), entropy
									snake_case_						       =	all_highway_exits + (highway_exit,)

									if highway_entropy < self.early_exit_entropy[i]:
										snake_case_						       =	(highway_logits,) + current_outputs[1:] + (all_highway_exits,)
										raise HighwayException(__lowercase			,      i + 1		)
								else:
									snake_case_						       =	all_highway_exits + (highway_exit,)

        # Add last layer
							if self.output_hidden_states:
								snake_case_						       =	all_hidden_states + (hidden_states,)

							snake_case_						       =	(hidden_states,)
							if self.output_hidden_states:
								snake_case_						       =	outputs + (all_hidden_states,)
							if self.output_attentions:
								snake_case_						       =	outputs + (all_attentions,)

							snake_case_						       =	outputs + (all_highway_exits,)
							return outputs  # last-layer hidden state, (all hidden states), (all attentions), all highway exits







@add_start_docstrings(
    '''The Bert Model transformer with early exiting (DeeBERT). '''			,  UpperCAmelCase__			,  )
class 						UpperCAmelCase  (							UpperCAmelCase__     ):







						'''simple docstring'''





						def __init__(		self				:	Union[str, Any]			,      __lowercase				:	List[Any]		):






							"""simple docstring"""






							super().__init__(__lowercase		)
							snake_case_						       =	config

							snake_case_						       =	BertEmbeddings(__lowercase		)
							snake_case_						       =	DeeBertEncoder(__lowercase		)
							snake_case_						       =	BertPooler(__lowercase		)

							self.init_weights()






						def 			snake_case__		(		self				:	Tuple		):






							"""simple docstring"""






							self.encoder.init_highway_pooler(self.pooler		)






						def 			snake_case__		(		self				:	Tuple		):






							"""simple docstring"""






							return self.embeddings.word_embeddings






						def 			snake_case__		(		self				:	Optional[int]			,      __lowercase				:	Any		):






							"""simple docstring"""






							snake_case_						       =	value






						def 			snake_case__		(		self				:	List[Any]			,      __lowercase				:	Optional[int]		):






							"""simple docstring"""






							for layer, heads in heads_to_prune.items():
								self.encoder.layer[layer].attention.prune_heads(__lowercase		)






						@add_start_docstrings_to_model_forward(__lowercase		)
						def 			snake_case__		(		self				:	List[str]			,      __lowercase				:	List[str]=None			,      __lowercase				:	int=None			,      __lowercase				:	Tuple=None			,      __lowercase				:	Dict=None			,      __lowercase				:	str=None			,      __lowercase				:	Tuple=None			,      __lowercase				:	List[Any]=None			,      __lowercase				:	List[str]=None			,      ):






							"""simple docstring"""






							if input_ids is not None and inputs_embeds is not None:
								raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time"		)
							elif input_ids is not None:
								snake_case_						       =	input_ids.size()
							elif inputs_embeds is not None:
								snake_case_						       =	inputs_embeds.size()[:-1]
							else:
								raise ValueError("You have to specify either input_ids or inputs_embeds"		)

							snake_case_						       =	input_ids.device if input_ids is not None else inputs_embeds.device

							if attention_mask is None:
								snake_case_						       =	torch.ones(__lowercase			,      device=__lowercase		)
							if encoder_attention_mask is None:
								snake_case_						       =	torch.ones(__lowercase			,      device=__lowercase		)
							if token_type_ids is None:
								snake_case_						       =	torch.zeros(__lowercase			,      dtype=torch.long			,      device=__lowercase		)

							# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
							# ourselves in which case we just need to make it broadcastable to all heads.
							snake_case_						       =	self.get_extended_attention_mask(__lowercase			,      __lowercase			,      __lowercase		)

							# If a 2D ou 3D attention mask is provided for the cross-attention
							# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
							if encoder_attention_mask.dim() == 3:
								snake_case_						       =	encoder_attention_mask[:, None, :, :]
							if encoder_attention_mask.dim() == 2:
								snake_case_						       =	encoder_attention_mask[:, None, None, :]

							snake_case_						       =	encoder_extended_attention_mask.to(
							    dtype=next(self.parameters()		).dtype		)  # fp16 compatibility
							snake_case_						       =	(1.0 - encoder_extended_attention_mask) * -1_0000.0

							# Prepare head mask if needed
							# 1.0 in head_mask indicate we keep the head
							# attention_probs has shape bsz x n_heads x N x N
							# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
							# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
							snake_case_						       =	self.get_head_mask(__lowercase			,      self.config.num_hidden_layers		)

							snake_case_						       =	self.embeddings(
							    input_ids=__lowercase			,      position_ids=__lowercase			,      token_type_ids=__lowercase			,      inputs_embeds=__lowercase		)
							snake_case_						       =	self.encoder(
							    __lowercase			,      attention_mask=__lowercase			,      head_mask=__lowercase			,      encoder_hidden_states=__lowercase			,      encoder_attention_mask=__lowercase			,      )
							snake_case_						       =	encoder_outputs[0]
							snake_case_						       =	self.pooler(__lowercase		)

							snake_case_						       =	(
							    sequence_output,
							    pooled_output,
							) + encoder_outputs[
							    1:
							]  # add hidden_states and attentions if they are here
							return outputs  # sequence_output, pooled_output, (hidden_states), (attentions), highway exits







class 						UpperCAmelCase  (							UpperCAmelCase__     ):







						'''simple docstring'''





						def __init__(		self				:	Dict			,      __lowercase				:	Tuple			,      __lowercase				:	List[Any]		):






							"""simple docstring"""






							snake_case_						       =	message
							snake_case_						       =	exit_layer  # start from 1!







class 						UpperCAmelCase  (							nn.Module     ):







						'''simple docstring'''





						def __init__(		self				:	int			,      __lowercase				:	Optional[int]		):






							"""simple docstring"""






							super().__init__()
							snake_case_						       =	BertPooler(__lowercase		)
							snake_case_						       =	nn.Dropout(config.hidden_dropout_prob		)
							snake_case_						       =	nn.Linear(config.hidden_size			,      config.num_labels		)






						def 			snake_case__		(		self				:	List[str]			,      __lowercase				:	str		):






							"""simple docstring"""






							snake_case_						       =	encoder_outputs[0]
							snake_case_						       =	self.pooler(__lowercase		)
							# "return" pooler_output

							# BertModel
							snake_case_						       =	(pooler_input, pooler_output) + encoder_outputs[1:]
							# "return" bmodel_output

							# Dropout and classification
							snake_case_						       =	bmodel_output[1]

							snake_case_						       =	self.dropout(__lowercase		)
							snake_case_						       =	self.classifier(__lowercase		)

							return logits, pooled_output







@add_start_docstrings(
    '''Bert Model (with early exiting - DeeBERT) with a classifier on top,
    also takes care of multi-layer training. '''			,  UpperCAmelCase__			,  )
class 						UpperCAmelCase  (							UpperCAmelCase__     ):







						'''simple docstring'''





						def __init__(		self				:	Union[str, Any]			,      __lowercase				:	Union[str, Any]		):






							"""simple docstring"""






							super().__init__(__lowercase		)
							snake_case_						       =	config.num_labels
							snake_case_						       =	config.num_hidden_layers

							snake_case_						       =	DeeBertModel(__lowercase		)
							snake_case_						       =	nn.Dropout(config.hidden_dropout_prob		)
							snake_case_						       =	nn.Linear(config.hidden_size			,      self.config.num_labels		)

							self.init_weights()






						@add_start_docstrings_to_model_forward(__lowercase		)
						def 			snake_case__		(		self				:	Optional[int]			,      __lowercase				:	List[Any]=None			,      __lowercase				:	Dict=None			,      __lowercase				:	Optional[Any]=None			,      __lowercase				:	Optional[int]=None			,      __lowercase				:	List[Any]=None			,      __lowercase				:	Tuple=None			,      __lowercase				:	List[str]=None			,      __lowercase				:	int=-1			,      __lowercase				:	Tuple=False			,      ):






							"""simple docstring"""






							snake_case_						       =	self.num_layers
							try:
								snake_case_						       =	self.bert(
								    __lowercase			,      attention_mask=__lowercase			,      token_type_ids=__lowercase			,      position_ids=__lowercase			,      head_mask=__lowercase			,      inputs_embeds=__lowercase			,      )
								# sequence_output, pooled_output, (hidden_states), (attentions), highway exits

								snake_case_						       =	outputs[1]

								snake_case_						       =	self.dropout(__lowercase		)
								snake_case_						       =	self.classifier(__lowercase		)
								snake_case_						       =	(logits,) + outputs[2:]  # add hidden states and attention if they are here
							except HighwayException as e:
								snake_case_						       =	e.message
								snake_case_						       =	e.exit_layer
								snake_case_						       =	outputs[0]

							if not self.training:
								snake_case_						       =	entropy(__lowercase		)
								snake_case_						       =	[]
								snake_case_						       =	[]
							if labels is not None:
								if self.num_labels == 1:
									#  We are doing regression
									snake_case_						       =	MSELoss()
									snake_case_						       =	loss_fct(logits.view(-1		)			,      labels.view(-1		)		)
								else:
									snake_case_						       =	CrossEntropyLoss()
									snake_case_						       =	loss_fct(logits.view(-1			,      self.num_labels		)			,      labels.view(-1		)		)

								# work with highway exits
								snake_case_						       =	[]
								for highway_exit in outputs[-1]:
									snake_case_						       =	highway_exit[0]
									if not self.training:
										highway_logits_all.append(__lowercase		)
										highway_entropy.append(highway_exit[2]		)
									if self.num_labels == 1:
										#  We are doing regression
										snake_case_						       =	MSELoss()
										snake_case_						       =	loss_fct(highway_logits.view(-1		)			,      labels.view(-1		)		)
									else:
										snake_case_						       =	CrossEntropyLoss()
										snake_case_						       =	loss_fct(highway_logits.view(-1			,      self.num_labels		)			,      labels.view(-1		)		)
									highway_losses.append(__lowercase		)

								if train_highway:
									snake_case_						       =	(sum(highway_losses[:-1]		),) + outputs
									# exclude the final highway, of course
								else:
									snake_case_						       =	(loss,) + outputs
							if not self.training:
								snake_case_						       =	outputs + ((original_entropy, highway_entropy), exit_layer)
								if output_layer >= 0:
									snake_case_						       =	(
									    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
									)  # use the highway of the last layer

							return outputs  # (loss), logits, (hidden_states), (attentions), (highway_exits)
