




import inspect

import jax
import jax.lax as lax
import jax.numpy as jnp

from ..utils import add_start_docstrings
from ..utils.logging import get_logger


lowercase__							:	Dict											=		get_logger(__name__)


lowercase__							:	Optional[Any]											=		R"\n    Args:\n        input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`):\n            Indices of input sequence tokens in the vocabulary.\n\n            Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and\n            [`PreTrainedTokenizer.__call__`] for details.\n\n            [What are input IDs?](../glossary#input-ids)\n        scores (`jnp.ndarray` of shape `(batch_size, config.vocab_size)`):\n            Prediction scores of a language modeling head. These can be logits for each vocabulary when not using beam\n            search or log softmax for each vocabulary token when using beam search\n        kwargs (`Dict[str, Any]`, *optional*):\n            Additional logits processor specific kwargs.\n\n    Return:\n        `jnp.ndarray` of shape `(batch_size, config.vocab_size)`: The processed prediction scores.\n\n"
class 						UpperCAmelCase  :







						'''simple docstring'''





						@add_start_docstrings(__lowercase		)
						def __call__(		self				:	Optional[Any]			,      __lowercase				:	jnp.ndarray			,      __lowercase				:	jnp.ndarray		):






							"""simple docstring"""






							raise NotImplementedError(
							    f"{self.__class__} is an abstract class. Only classes inheriting this class can be called."		)







class 						UpperCAmelCase  :







						'''simple docstring'''





						@add_start_docstrings(__lowercase		)
						def __call__(		self				:	Optional[Any]			,      __lowercase				:	jnp.ndarray			,      __lowercase				:	jnp.ndarray		):






							"""simple docstring"""






							raise NotImplementedError(
							    f"{self.__class__} is an abstract class. Only classes inheriting this class can be called."		)







class 						UpperCAmelCase  (							UpperCAmelCase__     ):







						'''simple docstring'''





						@add_start_docstrings(__lowercase		)
						def __call__(		self				:	Union[str, Any]			,      __lowercase				:	jnp.ndarray			,      __lowercase				:	jnp.ndarray			,      __lowercase				:	int			,      **__lowercase				:	str		):






							"""simple docstring"""






							for processor in self:
								snake_case_						       =	inspect.signature(processor.__call__		).parameters
								if len(__lowercase		) > 3:
									if not all(arg in kwargs for arg in list(function_args.keys()		)[2:]		):
										raise ValueError(
										    f"Make sure that all the required parameters: {list(function_args.keys()		)} for "
										    f"{processor.__class__} are passed to the logits processor."		)
									snake_case_						       =	processor(__lowercase			,      __lowercase			,      __lowercase			,      **__lowercase		)
								else:
									snake_case_						       =	processor(__lowercase			,      __lowercase			,      __lowercase		)
							return scores







class 						UpperCAmelCase  (							UpperCAmelCase__     ):







						'''simple docstring'''





						def __init__(		self				:	Tuple			,      __lowercase				:	float		):






							"""simple docstring"""






							if not isinstance(__lowercase			,      __lowercase		) or not (temperature > 0):
								raise ValueError(f"`temperature` has to be a strictly positive float, but is {temperature}"		)

							snake_case_						       =	temperature






						def __call__(		self				:	Dict			,      __lowercase				:	jnp.ndarray			,      __lowercase				:	jnp.ndarray			,      __lowercase				:	int		):






							"""simple docstring"""






							snake_case_						       =	scores / self.temperature
							return scores







class 						UpperCAmelCase  (							UpperCAmelCase__     ):







						'''simple docstring'''





						def __init__(		self				:	Union[str, Any]			,      __lowercase				:	float			,      __lowercase				:	float = -float("Inf"		)			,      __lowercase				:	int = 1		):






							"""simple docstring"""






							if not isinstance(__lowercase			,      __lowercase		) or (top_p < 0 or top_p > 1.0):
								raise ValueError(f"`top_p` has to be a float > 0 and < 1, but is {top_p}"		)
							if not isinstance(__lowercase			,      __lowercase		) or (min_tokens_to_keep < 1):
								raise ValueError(f"`min_tokens_to_keep` has to be a positive integer, but is {min_tokens_to_keep}"		)

							snake_case_						       =	top_p
							snake_case_						       =	filter_value
							snake_case_						       =	min_tokens_to_keep






						def __call__(		self				:	List[Any]			,      __lowercase				:	jnp.ndarray			,      __lowercase				:	jnp.ndarray			,      __lowercase				:	int		):






							"""simple docstring"""






							snake_case_       , snake_case_						       =	lax.top_k(__lowercase			,      scores.shape[-1]		)

							snake_case_						       =	jnp.full_like(__lowercase			,      self.filter_value		)
							snake_case_						       =	jax.nn.softmax(__lowercase			,      axis=-1		).cumsum(axis=-1		)
							snake_case_						       =	cumulative_probs < self.top_p

							# include the token that is higher than top_p as well
							snake_case_						       =	jnp.roll(__lowercase			,      1		)
							score_mask |= score_mask.at[:, 0].set(__lowercase		)

							# min tokens to keep
							snake_case_						       =	score_mask.at[:, : self.min_tokens_to_keep].set(__lowercase		)

							snake_case_						       =	jnp.where(__lowercase			,      __lowercase			,      __lowercase		)
							snake_case_						       =	jax.lax.sort_key_val(__lowercase			,      __lowercase		)[-1]

							return next_scores







class 						UpperCAmelCase  (							UpperCAmelCase__     ):







						'''simple docstring'''





						def __init__(		self				:	List[str]			,      __lowercase				:	int			,      __lowercase				:	float = -float("Inf"		)			,      __lowercase				:	int = 1		):






							"""simple docstring"""






							if not isinstance(__lowercase			,      __lowercase		) or top_k <= 0:
								raise ValueError(f"`top_k` has to be a strictly positive integer, but is {top_k}"		)

							snake_case_						       =	max(__lowercase			,      __lowercase		)
							snake_case_						       =	filter_value






						def __call__(		self				:	Any			,      __lowercase				:	jnp.ndarray			,      __lowercase				:	jnp.ndarray			,      __lowercase				:	int		):






							"""simple docstring"""






							snake_case_       , snake_case_						       =	scores.shape
							snake_case_						       =	jnp.full(batch_size * vocab_size			,      self.filter_value		)

							snake_case_						       =	min(self.top_k			,      scores.shape[-1]		)  # Safety check
							snake_case_       , snake_case_						       =	lax.top_k(__lowercase			,      __lowercase		)
							snake_case_						       =	jnp.broadcast_to((jnp.arange(__lowercase		) * vocab_size)[:, None]			,      (batch_size, topk)		).flatten()
							snake_case_						       =	topk_scores.flatten()
							snake_case_						       =	topk_indices.flatten() + shift

							snake_case_						       =	next_scores_flat.at[topk_indices_flat].set(__lowercase		)
							snake_case_						       =	next_scores_flat.reshape(__lowercase			,      __lowercase		)
							return next_scores







class 						UpperCAmelCase  (							UpperCAmelCase__     ):







						'''simple docstring'''





						def __init__(		self				:	Optional[int]			,      __lowercase				:	int		):






							"""simple docstring"""






							snake_case_						       =	bos_token_id






						def __call__(		self				:	Optional[Any]			,      __lowercase				:	jnp.ndarray			,      __lowercase				:	jnp.ndarray			,      __lowercase				:	int		):






							"""simple docstring"""






							snake_case_						       =	jnp.full(scores.shape			,      -float("inf"		)		)

							snake_case_						       =	1 - jnp.bool_(cur_len - 1		)

							snake_case_						       =	jnp.where(__lowercase			,      new_scores.at[:, self.bos_token_id].set(0		)			,      __lowercase		)

							return scores







class 						UpperCAmelCase  (							UpperCAmelCase__     ):







						'''simple docstring'''





						def __init__(		self				:	List[Any]			,      __lowercase				:	int			,      __lowercase				:	int		):






							"""simple docstring"""






							snake_case_						       =	max_length
							snake_case_						       =	eos_token_id






						def __call__(		self				:	Any			,      __lowercase				:	jnp.ndarray			,      __lowercase				:	jnp.ndarray			,      __lowercase				:	int		):






							"""simple docstring"""






							snake_case_						       =	jnp.full(scores.shape			,      -float("inf"		)		)

							snake_case_						       =	1 - jnp.bool_(cur_len - self.max_length + 1		)

							snake_case_						       =	jnp.where(__lowercase			,      new_scores.at[:, self.eos_token_id].set(0		)			,      __lowercase		)

							return scores







class 						UpperCAmelCase  (							UpperCAmelCase__     ):







						'''simple docstring'''





						def __init__(		self				:	str			,      __lowercase				:	int			,      __lowercase				:	int		):






							"""simple docstring"""






							if not isinstance(__lowercase			,      __lowercase		) or min_length < 0:
								raise ValueError(f"`min_length` has to be a positive integer, but is {min_length}"		)

							if not isinstance(__lowercase			,      __lowercase		) or eos_token_id < 0:
								raise ValueError(f"`eos_token_id` has to be a positive integer, but is {eos_token_id}"		)

							snake_case_						       =	min_length
							snake_case_						       =	eos_token_id






						def __call__(		self				:	int			,      __lowercase				:	jnp.ndarray			,      __lowercase				:	jnp.ndarray			,      __lowercase				:	int		):






							"""simple docstring"""






							snake_case_						       =	1 - jnp.clip(cur_len - self.min_length			,      0			,      1		)

							snake_case_						       =	jnp.where(__lowercase			,      scores.at[:, self.eos_token_id].set(-float("inf"		)		)			,      __lowercase		)

							return scores







class 						UpperCAmelCase  (							UpperCAmelCase__     ):







						'''simple docstring'''





						def __init__(		self				:	Optional[int]			,      __lowercase				:	Dict			,      __lowercase				:	str		):






							"""simple docstring"""






							snake_case_						       =	list(__lowercase		)
							snake_case_						       =	begin_index






						def __call__(		self				:	Optional[Any]			,      __lowercase				:	Tuple			,      __lowercase				:	Union[str, Any]			,      __lowercase				:	int		):






							"""simple docstring"""






							snake_case_						       =	1 - jnp.bool_(cur_len - self.begin_index		)

							snake_case_						       =	jnp.where(__lowercase			,      scores.at[:, self.begin_suppress_tokens].set(-float("inf"		)		)			,      __lowercase		)

							return scores







class 						UpperCAmelCase  (							UpperCAmelCase__     ):







						'''simple docstring'''





						def __init__(		self				:	str			,      __lowercase				:	list		):






							"""simple docstring"""






							snake_case_						       =	list(__lowercase		)






						def __call__(		self				:	List[str]			,      __lowercase				:	jnp.ndarray			,      __lowercase				:	jnp.ndarray			,      __lowercase				:	int		):






							"""simple docstring"""






							snake_case_						       =	scores.at[..., self.suppress_tokens].set(-float("inf"		)		)

							return scores







class 						UpperCAmelCase  (							UpperCAmelCase__     ):







						'''simple docstring'''





						def __init__(		self				:	Optional[Any]			,      __lowercase				:	str		):






							"""simple docstring"""






							snake_case_						       =	dict(__lowercase		)
							# Converts the dictionary of format {index: token} containing the tokens to be forced to an array, where the
							# index of the array corresponds to the index of the token to be forced, for XLA compatibility.
							# Indexes without forced tokens will have a negative value.
							snake_case_						       =	jnp.ones((max(force_token_map.keys()		) + 1)			,      dtype=jnp.intaa		) * -1
							for index, token in force_token_map.items():
								if token is not None:
									snake_case_						       =	force_token_array.at[index].set(__lowercase		)
							snake_case_						       =	jnp.intaa(__lowercase		)






						def __call__(		self				:	Optional[int]			,      __lowercase				:	jnp.ndarray			,      __lowercase				:	jnp.ndarray			,      __lowercase				:	int		):






							"""simple docstring"""






							def _force_token(__lowercase				:	Optional[Any]		):
								snake_case_						       =	scores.shape[0]
								snake_case_						       =	self.force_token_array[generation_idx]

								snake_case_						       =	jnp.ones_like(__lowercase			,      dtype=scores.dtype		) * -float("inf"		)
								snake_case_						       =	jnp.zeros((batch_size, 1)			,      dtype=scores.dtype		)
								snake_case_						       =	lax.dynamic_update_slice(__lowercase			,      __lowercase			,      (0, current_token)		)
								return new_scores

							snake_case_						       =	lax.cond(
							    cur_len >= self.force_token_array.shape[0]			,      lambda: scores			,      lambda: lax.cond(
							        self.force_token_array[cur_len] >= 0			,      lambda: _force_token(__lowercase		)			,      lambda: scores			,      )			,      )
							return scores







class 						UpperCAmelCase  (							UpperCAmelCase__     ):







						'''simple docstring'''





						def __init__(		self				:	List[str]			,      __lowercase				:	Optional[Any]			,      __lowercase				:	Dict			,      __lowercase				:	str		):






							"""simple docstring"""






							snake_case_						       =	generate_config.eos_token_id
							snake_case_						       =	generate_config.no_timestamps_token_id
							snake_case_						       =	generate_config.no_timestamps_token_id + 1

							snake_case_						       =	decoder_input_length + 1

							if generate_config.is_multilingual:
								# room for language token and task token
								self.begin_index += 2
							if hasattr(__lowercase			,      "max_initial_timestamp_index"		):
								snake_case_						       =	generate_config.max_initial_timestamp_index
							else:
								snake_case_						       =	model_config.vocab_size
							if self.max_initial_timestamp_index is None:
								snake_case_						       =	model_config.vocab_size






						def __call__(		self				:	Any			,      __lowercase				:	Union[str, Any]			,      __lowercase				:	Any			,      __lowercase				:	List[Any]		):






							"""simple docstring"""






							snake_case_						       =	scores.at[:, self.no_timestamps_token_id].set(-float("inf"		)		)

							def handle_pairs(__lowercase				:	Union[str, Any]			,      __lowercase				:	Tuple		):
								snake_case_						       =	jnp.where((cur_len - self.begin_index) >= 1			,      __lowercase			,      __lowercase		)
								snake_case_						       =	jnp.where(
								    input_ids_k[cur_len - 1] >= self.timestamp_begin			,      True and last_was_timestamp			,      __lowercase			,      )

								snake_case_						       =	jnp.where((cur_len - self.begin_index) < 2			,      __lowercase			,      __lowercase		)
								snake_case_						       =	jnp.where(
								    input_ids_k[cur_len - 2] >= self.timestamp_begin			,      __lowercase			,      __lowercase			,      )

								return jnp.where(
								    __lowercase			,      jnp.where(
								        penultimate_was_timestamp > 0			,      scores_k.at[self.timestamp_begin :].set(-float("inf"		)		)			,      scores_k.at[: self.eos_token_id].set(-float("inf"		)		)			,      )			,      __lowercase			,      )

							snake_case_						       =	jax.vmap(__lowercase		)(__lowercase			,      __lowercase		)

							snake_case_						       =	jnp.where(cur_len == self.begin_index			,      __lowercase			,      __lowercase		)
							snake_case_						       =	jnp.where(
							    self.max_initial_timestamp_index is not None			,      True and apply_max_initial_timestamp			,      __lowercase			,      )

							snake_case_						       =	self.timestamp_begin + self.max_initial_timestamp_index

							snake_case_						       =	jnp.where(
							    __lowercase			,      scores.at[:, last_allowed + 1 :].set(-float("inf"		)		)			,      __lowercase			,      )

							# if sum of probability over timestamps is above any other token, sample timestamp
							snake_case_						       =	jax.nn.log_softmax(__lowercase			,      axis=-1		)

							def handle_cumulative_probs(__lowercase				:	Optional[int]			,      __lowercase				:	Dict		):
								snake_case_						       =	jax.nn.logsumexp(logprobs_k[self.timestamp_begin :]			,      axis=-1		)
								snake_case_						       =	jnp.max(logprobs_k[: self.timestamp_begin]		)
								return jnp.where(
								    timestamp_logprob > max_text_token_logprob			,      scores_k.at[: self.timestamp_begin].set(-float("inf"		)		)			,      __lowercase			,      )

							snake_case_						       =	jax.vmap(__lowercase		)(__lowercase			,      __lowercase		)

							return scores






import argparse
import json
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils.deepspeed import DummyOptim, DummyScheduler


lowercase__							:	Dict											=		16
lowercase__							:	str											=		32


def 		lowerCamelCase__       (						_A	,      _A = 16	,      _A = "bert-base-cased"				):







	'''simple docstring'''





	snake_case_						       =	AutoTokenizer.from_pretrained(_A				)
	snake_case_						       =	load_dataset("glue"	,      "mrpc"				)

	def tokenize_function(_A				):
		# max_length=None => use the model max length (it's actually the default)
		snake_case_						       =	tokenizer(examples["sentence1"]	,      examples["sentence2"]	,      truncation=_A	,      max_length=_A				)
		return outputs

	# Apply the method we just defined to all the examples in all the splits of the dataset
	snake_case_						       =	datasets.map(
	    _A	,      batched=_A	,      remove_columns=["idx", "sentence1", "sentence2"]	,      load_from_cache_file=_A				)

	# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
	# transformers library
	snake_case_						       =	tokenized_datasets.rename_column("label"	,      "labels"				)

	def collate_fn(_A				):
		# On TPU it's best to pad everything to the same length or training will be very slow.
		if accelerator.distributed_type == DistributedType.TPU:
			return tokenizer.pad(_A	,      padding="max_length"	,      max_length=128	,      return_tensors="pt"				)
		return tokenizer.pad(_A	,      padding="longest"	,      return_tensors="pt"				)

	# Instantiate dataloaders.
	snake_case_						       =	DataLoader(
	    tokenized_datasets["train"]	,      shuffle=_A	,      collate_fn=_A	,      batch_size=_A				)
	snake_case_						       =	DataLoader(
	    tokenized_datasets["validation"]	,      shuffle=_A	,      collate_fn=_A	,      batch_size=_A				)

	return train_dataloader, eval_dataloader


def 		lowerCamelCase__       (						_A	,      _A				):







	'''simple docstring'''





	snake_case_						       =	Accelerator()

	# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
	snake_case_						       =	config["lr"]
	snake_case_						       =	int(config["num_epochs"]				)
	snake_case_						       =	int(config["seed"]				)
	snake_case_						       =	int(config["batch_size"]				)
	snake_case_						       =	args.model_name_or_path

	set_seed(_A				)
	snake_case_       , snake_case_						       =	get_dataloaders(_A	,      _A	,      _A				)

	# Instantiate the model (we build the model here so that the seed also control new weights initialization)
	snake_case_						       =	AutoModelForSequenceClassification.from_pretrained(_A	,      return_dict=_A				)

	# Instantiate optimizer
	snake_case_						       =	(
	    AdamW
	    if accelerator.state.deepspeed_plugin is None
	    or "optimizer" not in accelerator.state.deepspeed_plugin.deepspeed_config
	    else DummyOptim
	)
	snake_case_						       =	optimizer_cls(params=model.parameters()	,      lr=_A				)

	if accelerator.state.deepspeed_plugin is not None:
		snake_case_						       =	accelerator.state.deepspeed_plugin.deepspeed_config[
		    "gradient_accumulation_steps"
		]
	else:
		snake_case_						       =	1
	snake_case_						       =	(len(_A				) * num_epochs) // gradient_accumulation_steps

	# Instantiate scheduler
	if (
	    accelerator.state.deepspeed_plugin is None
	    or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config
	):
		snake_case_						       =	get_linear_schedule_with_warmup(
		    optimizer=_A	,      num_warmup_steps=0	,      num_training_steps=_A	,      )
	else:
		snake_case_						       =	DummyScheduler(_A	,      total_num_steps=_A	,      warmup_num_steps=0				)

	# Prepare everything
	# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
	# prepare method.
	snake_case_       , snake_case_       , snake_case_       , snake_case_       , snake_case_						       =	accelerator.prepare(
	    _A	,      _A	,      _A	,      _A	,      _A				)

	# We need to keep track of how many total steps we have iterated over
	snake_case_						       =	0
	# We also need to keep track of the stating epoch so files are named properly
	snake_case_						       =	0

	# Now we train the model
	snake_case_						       =	evaluate.load("glue"	,      "mrpc"				)
	snake_case_						       =	0
	snake_case_						       =	{}
	for epoch in range(_A	,      _A				):
		model.train()
		for step, batch in enumerate(_A				):
			snake_case_						       =	model(**_A				)
			snake_case_						       =	outputs.loss
			snake_case_						       =	loss / gradient_accumulation_steps
			accelerator.backward(_A				)
			if step % gradient_accumulation_steps == 0:
				optimizer.step()
				lr_scheduler.step()
				optimizer.zero_grad()

			overall_step += 1

		model.eval()
		snake_case_						       =	0
		for step, batch in enumerate(_A				):
			# We could avoid this line since we set the accelerator with `device_placement=True`.
			batch.to(accelerator.device				)
			with torch.no_grad():
				snake_case_						       =	model(**_A				)
			snake_case_						       =	outputs.logits.argmax(dim=-1				)
			# It is slightly faster to call this once, than multiple times
			snake_case_       , snake_case_						       =	accelerator.gather(
			    (predictions, batch["labels"])				)  # If we are in a multiprocess environment, the last batch has duplicates
			if accelerator.use_distributed:
				if step == len(_A				) - 1:
					snake_case_						       =	predictions[: len(eval_dataloader.dataset				) - samples_seen]
					snake_case_						       =	references[: len(eval_dataloader.dataset				) - samples_seen]
				else:
					samples_seen += references.shape[0]
			metric.add_batch(
			    predictions=_A	,      references=_A	,      )

		snake_case_						       =	metric.compute()
		# Use accelerator.print to print only on the main process.
		accelerator.print(f"epoch {epoch}:"	,      _A				)
		snake_case_						       =	eval_metric["accuracy"]

		if best_performance < eval_metric["accuracy"]:
			snake_case_						       =	eval_metric["accuracy"]

	if args.performance_lower_bound is not None:
		assert (
		    args.performance_lower_bound <= best_performance
		), f"Best performance metric {best_performance} is lower than the lower bound {args.performance_lower_bound}"

	accelerator.wait_for_everyone()
	if accelerator.is_main_process:
		with open(os.path.join(args.output_dir	,      "all_results.json"				)	,      "w"				) as f:
			json.dump(_A	,      _A				)


def 		lowerCamelCase__       (						):







	'''simple docstring'''





	snake_case_						       =	argparse.ArgumentParser(description="Simple example of training script tracking peak GPU memory usage."				)
	parser.add_argument(
	    "--model_name_or_path"	,      type=_A	,      default="bert-base-cased"	,      help="Path to pretrained model or model identifier from huggingface.co/models."	,      required=_A	,      )
	parser.add_argument(
	    "--output_dir"	,      type=_A	,      default="."	,      help="Optional save directory where all checkpoint folders will be stored. Default is the current working directory."	,      )
	parser.add_argument(
	    "--performance_lower_bound"	,      type=_A	,      default=_A	,      help="Optional lower bound for the performance metric. If set, the training will throw error when the performance metric drops below this value."	,      )
	parser.add_argument(
	    "--num_epochs"	,      type=_A	,      default=3	,      help="Number of train epochs."	,      )
	snake_case_						       =	parser.parse_args()
	snake_case_						       =	{"lr": 2E-5, "num_epochs": args.num_epochs, "seed": 42, "batch_size": 16}
	training_function(_A	,      _A				)


if __name__ == "__main__":
		main()
