




import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


lowercase__							:	Dict											=		logging.getLogger(__name__)

lowercase__							:	List[str]											=		tf.data.AUTOTUNE


def 		lowerCamelCase__       (						):







	'''simple docstring'''





	snake_case_						       =	argparse.ArgumentParser(description="Train a masked language model on TPU."				)
	parser.add_argument(
	    "--pretrained_model_config"	,      type=_A	,      default="roberta-base"	,      help="The model config to use. Note that we don't copy the model's weights, only the config!"	,      )
	parser.add_argument(
	    "--tokenizer"	,      type=_A	,      default="unigram-tokenizer-wikitext"	,      help="The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model's vocab size."	,      )

	parser.add_argument(
	    "--per_replica_batch_size"	,      type=_A	,      default=8	,      help="Batch size per TPU core."	,      )

	parser.add_argument(
	    "--no_tpu"	,      action="store_true"	,      help="If set, run on CPU and don't try to initialize a TPU. Useful for debugging on non-TPU instances."	,      )

	parser.add_argument(
	    "--tpu_name"	,      type=_A	,      help="Name of TPU resource to initialize. Should be blank on Colab, and 'local' on TPU VMs."	,      default="local"	,      )

	parser.add_argument(
	    "--tpu_zone"	,      type=_A	,      help="Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes."	,      )

	parser.add_argument(
	    "--gcp_project"	,      type=_A	,      help="Google cloud project name. Only used for non-Colab TPU nodes."				)

	parser.add_argument(
	    "--bfloat16"	,      action="store_true"	,      help="Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU."	,      )

	parser.add_argument(
	    "--train_dataset"	,      type=_A	,      help="Path to training dataset to load. If the path begins with `gs://`"
	    " then the dataset will be loaded from a Google Cloud Storage bucket."	,      )

	parser.add_argument(
	    "--shuffle_buffer_size"	,      type=_A	,      default=2**18	,      help="Size of the shuffle buffer (in samples)"	,      )

	parser.add_argument(
	    "--eval_dataset"	,      type=_A	,      help="Path to evaluation dataset to load. If the path begins with `gs://`"
	    " then the dataset will be loaded from a Google Cloud Storage bucket."	,      )

	parser.add_argument(
	    "--num_epochs"	,      type=_A	,      default=1	,      help="Number of epochs to train for."	,      )

	parser.add_argument(
	    "--learning_rate"	,      type=_A	,      default=1E-4	,      help="Learning rate to use for training."	,      )

	parser.add_argument(
	    "--weight_decay_rate"	,      type=_A	,      default=1E-3	,      help="Weight decay rate to use for training."	,      )

	parser.add_argument(
	    "--max_length"	,      type=_A	,      default=512	,      help="Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py"	,      )

	parser.add_argument(
	    "--mlm_probability"	,      type=_A	,      default=0.15	,      help="Fraction of tokens to mask during training."	,      )

	parser.add_argument("--output_dir"	,      type=_A	,      required=_A	,      help="Path to save model checkpoints to."				)
	parser.add_argument("--hub_model_id"	,      type=_A	,      help="Model ID to upload to on the Hugging Face Hub."				)

	snake_case_						       =	parser.parse_args()
	return args


def 		lowerCamelCase__       (						_A				):







	'''simple docstring'''





	try:
		if args.tpu_name:
			snake_case_						       =	tf.distribute.cluster_resolver.TPUClusterResolver(
			    args.tpu_name	,      zone=args.tpu_zone	,      project=args.gcp_project				)
		else:
			snake_case_						       =	tf.distribute.cluster_resolver.TPUClusterResolver()
	except ValueError:
		raise RuntimeError(
		    "Couldn't connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or "
		    "--gcp_project. When running on a TPU VM, use --tpu_name local."				)

	tf.config.experimental_connect_to_cluster(_A				)
	tf.tpu.experimental.initialize_tpu_system(_A				)

	return tpu


def 		lowerCamelCase__       (						_A				):







	'''simple docstring'''





	snake_case_						       =	0
	for file in file_list:
		snake_case_						       =	file.split("/"				)[-1]
		snake_case_						       =	re.search(R"-\d+-(\d+)\.tfrecord"	,      _A				).group(1				)
		snake_case_						       =	int(_A				)
		num_samples += sample_count

	return num_samples


def 		lowerCamelCase__       (						_A	,      _A	,      _A	,      _A	,      _A	,      _A=None				):







	'''simple docstring'''





	snake_case_						       =	count_samples(_A				)
	snake_case_						       =	tf.data.Dataset.from_tensor_slices(_A				)
	if shuffle:
		snake_case_						       =	dataset.shuffle(len(_A				)				)
	snake_case_						       =	tf.data.TFRecordDataset(_A	,      num_parallel_reads=_A				)
	# TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
	snake_case_						       =	dataset.apply(tf.data.experimental.assert_cardinality(_A				)				)
	snake_case_						       =	dataset.map(_A	,      num_parallel_calls=_A				)
	if shuffle:
		assert shuffle_buffer_size is not None
		snake_case_						       =	dataset.shuffle(args.shuffle_buffer_size				)
	snake_case_						       =	dataset.batch(_A	,      drop_remainder=_A				)
	snake_case_						       =	dataset.map(_A	,      num_parallel_calls=_A				)
	snake_case_						       =	dataset.prefetch(_A				)
	return dataset


def 		lowerCamelCase__       (						_A				):







	'''simple docstring'''





	if not args.no_tpu:
		snake_case_						       =	initialize_tpu(_A				)
		snake_case_						       =	tf.distribute.TPUStrategy(_A				)
	else:
		snake_case_						       =	tf.distribute.OneDeviceStrategy(device="/gpu:0"				)

	if args.bfloataa:
		tf.keras.mixed_precision.set_global_policy("mixed_bfloat16"				)

	snake_case_						       =	AutoTokenizer.from_pretrained(args.tokenizer				)
	snake_case_						       =	AutoConfig.from_pretrained(args.pretrained_model_config				)
	snake_case_						       =	tokenizer.vocab_size

	snake_case_						       =	tf.io.gfile.glob(os.path.join(args.train_dataset	,      "*.tfrecord"				)				)
	if not training_records:
		raise ValueError(f"No .tfrecord files found in {args.train_dataset}."				)
	snake_case_						       =	tf.io.gfile.glob(os.path.join(args.eval_dataset	,      "*.tfrecord"				)				)
	if not eval_records:
		raise ValueError(f"No .tfrecord files found in {args.eval_dataset}."				)

	snake_case_						       =	count_samples(_A				)

	snake_case_						       =	num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
	snake_case_						       =	steps_per_epoch * args.num_epochs

	with strategy.scope():
		snake_case_						       =	TFAutoModelForMaskedLM.from_config(_A				)
		model(model.dummy_inputs				)  # Pass some dummy inputs through the model to ensure all the weights are built
		snake_case_       , snake_case_						       =	create_optimizer(
		    num_train_steps=_A	,      num_warmup_steps=total_train_steps // 20	,      init_lr=args.learning_rate	,      weight_decay_rate=args.weight_decay_rate	,      )

		# Transformers models compute the right loss for their task by default when labels are passed, and will
		# use this for training unless you specify your own loss function in compile().
		model.compile(optimizer=_A	,      metrics=["accuracy"]				)

	def decode_fn(_A				):
		snake_case_						       =	{
		    "input_ids": tf.io.FixedLenFeature(dtype=tf.intaa	,      shape=(args.max_length,)				),
		    "attention_mask": tf.io.FixedLenFeature(dtype=tf.intaa	,      shape=(args.max_length,)				),
		}
		return tf.io.parse_single_example(_A	,      _A				)

	# Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
	# use their methods in our data pipeline.
	snake_case_						       =	DataCollatorForLanguageModeling(
	    tokenizer=_A	,      mlm_probability=args.mlm_probability	,      mlm=_A	,      return_tensors="tf"				)

	def mask_with_collator(_A				):
		# TF really needs an isin() function
		snake_case_						       =	(
		    ~tf.cast(batch["attention_mask"]	,      tf.bool				)
		    | (batch["input_ids"] == tokenizer.cls_token_id)
		    | (batch["input_ids"] == tokenizer.sep_token_id)
		)
		snake_case_       , snake_case_						       =	data_collator.tf_mask_tokens(
		    batch["input_ids"]	,      vocab_size=len(_A				)	,      mask_token_id=tokenizer.mask_token_id	,      special_tokens_mask=_A	,      )
		return batch

	snake_case_						       =	args.per_replica_batch_size * strategy.num_replicas_in_sync

	snake_case_						       =	prepare_dataset(
	    _A	,      decode_fn=_A	,      mask_fn=_A	,      batch_size=_A	,      shuffle=_A	,      shuffle_buffer_size=args.shuffle_buffer_size	,      )

	snake_case_						       =	prepare_dataset(
	    _A	,      decode_fn=_A	,      mask_fn=_A	,      batch_size=_A	,      shuffle=_A	,      )

	snake_case_						       =	[]
	if args.hub_model_id:
		callbacks.append(
		    PushToHubCallback(output_dir=args.output_dir	,      hub_model_id=args.hub_model_id	,      tokenizer=_A				)				)

	model.fit(
	    _A	,      validation_data=_A	,      epochs=args.num_epochs	,      callbacks=_A	,      )

	model.save_pretrained(args.output_dir				)


if __name__ == "__main__":
		lowercase__							:	List[str]											=		parse_args()
		main(args)






import argparse
import json
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils.deepspeed import DummyOptim, DummyScheduler


lowercase__							:	Dict											=		16
lowercase__							:	str											=		32


def 		lowerCamelCase__       (						_A	,      _A = 16	,      _A = "bert-base-cased"				):







	'''simple docstring'''





	snake_case_						       =	AutoTokenizer.from_pretrained(_A				)
	snake_case_						       =	load_dataset("glue"	,      "mrpc"				)

	def tokenize_function(_A				):
		# max_length=None => use the model max length (it's actually the default)
		snake_case_						       =	tokenizer(examples["sentence1"]	,      examples["sentence2"]	,      truncation=_A	,      max_length=_A				)
		return outputs

	# Apply the method we just defined to all the examples in all the splits of the dataset
	snake_case_						       =	datasets.map(
	    _A	,      batched=_A	,      remove_columns=["idx", "sentence1", "sentence2"]	,      load_from_cache_file=_A				)

	# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
	# transformers library
	snake_case_						       =	tokenized_datasets.rename_column("label"	,      "labels"				)

	def collate_fn(_A				):
		# On TPU it's best to pad everything to the same length or training will be very slow.
		if accelerator.distributed_type == DistributedType.TPU:
			return tokenizer.pad(_A	,      padding="max_length"	,      max_length=128	,      return_tensors="pt"				)
		return tokenizer.pad(_A	,      padding="longest"	,      return_tensors="pt"				)

	# Instantiate dataloaders.
	snake_case_						       =	DataLoader(
	    tokenized_datasets["train"]	,      shuffle=_A	,      collate_fn=_A	,      batch_size=_A				)
	snake_case_						       =	DataLoader(
	    tokenized_datasets["validation"]	,      shuffle=_A	,      collate_fn=_A	,      batch_size=_A				)

	return train_dataloader, eval_dataloader


def 		lowerCamelCase__       (						_A	,      _A				):







	'''simple docstring'''





	snake_case_						       =	Accelerator()

	# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
	snake_case_						       =	config["lr"]
	snake_case_						       =	int(config["num_epochs"]				)
	snake_case_						       =	int(config["seed"]				)
	snake_case_						       =	int(config["batch_size"]				)
	snake_case_						       =	args.model_name_or_path

	set_seed(_A				)
	snake_case_       , snake_case_						       =	get_dataloaders(_A	,      _A	,      _A				)

	# Instantiate the model (we build the model here so that the seed also control new weights initialization)
	snake_case_						       =	AutoModelForSequenceClassification.from_pretrained(_A	,      return_dict=_A				)

	# Instantiate optimizer
	snake_case_						       =	(
	    AdamW
	    if accelerator.state.deepspeed_plugin is None
	    or "optimizer" not in accelerator.state.deepspeed_plugin.deepspeed_config
	    else DummyOptim
	)
	snake_case_						       =	optimizer_cls(params=model.parameters()	,      lr=_A				)

	if accelerator.state.deepspeed_plugin is not None:
		snake_case_						       =	accelerator.state.deepspeed_plugin.deepspeed_config[
		    "gradient_accumulation_steps"
		]
	else:
		snake_case_						       =	1
	snake_case_						       =	(len(_A				) * num_epochs) // gradient_accumulation_steps

	# Instantiate scheduler
	if (
	    accelerator.state.deepspeed_plugin is None
	    or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config
	):
		snake_case_						       =	get_linear_schedule_with_warmup(
		    optimizer=_A	,      num_warmup_steps=0	,      num_training_steps=_A	,      )
	else:
		snake_case_						       =	DummyScheduler(_A	,      total_num_steps=_A	,      warmup_num_steps=0				)

	# Prepare everything
	# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
	# prepare method.
	snake_case_       , snake_case_       , snake_case_       , snake_case_       , snake_case_						       =	accelerator.prepare(
	    _A	,      _A	,      _A	,      _A	,      _A				)

	# We need to keep track of how many total steps we have iterated over
	snake_case_						       =	0
	# We also need to keep track of the stating epoch so files are named properly
	snake_case_						       =	0

	# Now we train the model
	snake_case_						       =	evaluate.load("glue"	,      "mrpc"				)
	snake_case_						       =	0
	snake_case_						       =	{}
	for epoch in range(_A	,      _A				):
		model.train()
		for step, batch in enumerate(_A				):
			snake_case_						       =	model(**_A				)
			snake_case_						       =	outputs.loss
			snake_case_						       =	loss / gradient_accumulation_steps
			accelerator.backward(_A				)
			if step % gradient_accumulation_steps == 0:
				optimizer.step()
				lr_scheduler.step()
				optimizer.zero_grad()

			overall_step += 1

		model.eval()
		snake_case_						       =	0
		for step, batch in enumerate(_A				):
			# We could avoid this line since we set the accelerator with `device_placement=True`.
			batch.to(accelerator.device				)
			with torch.no_grad():
				snake_case_						       =	model(**_A				)
			snake_case_						       =	outputs.logits.argmax(dim=-1				)
			# It is slightly faster to call this once, than multiple times
			snake_case_       , snake_case_						       =	accelerator.gather(
			    (predictions, batch["labels"])				)  # If we are in a multiprocess environment, the last batch has duplicates
			if accelerator.use_distributed:
				if step == len(_A				) - 1:
					snake_case_						       =	predictions[: len(eval_dataloader.dataset				) - samples_seen]
					snake_case_						       =	references[: len(eval_dataloader.dataset				) - samples_seen]
				else:
					samples_seen += references.shape[0]
			metric.add_batch(
			    predictions=_A	,      references=_A	,      )

		snake_case_						       =	metric.compute()
		# Use accelerator.print to print only on the main process.
		accelerator.print(f"epoch {epoch}:"	,      _A				)
		snake_case_						       =	eval_metric["accuracy"]

		if best_performance < eval_metric["accuracy"]:
			snake_case_						       =	eval_metric["accuracy"]

	if args.performance_lower_bound is not None:
		assert (
		    args.performance_lower_bound <= best_performance
		), f"Best performance metric {best_performance} is lower than the lower bound {args.performance_lower_bound}"

	accelerator.wait_for_everyone()
	if accelerator.is_main_process:
		with open(os.path.join(args.output_dir	,      "all_results.json"				)	,      "w"				) as f:
			json.dump(_A	,      _A				)


def 		lowerCamelCase__       (						):







	'''simple docstring'''





	snake_case_						       =	argparse.ArgumentParser(description="Simple example of training script tracking peak GPU memory usage."				)
	parser.add_argument(
	    "--model_name_or_path"	,      type=_A	,      default="bert-base-cased"	,      help="Path to pretrained model or model identifier from huggingface.co/models."	,      required=_A	,      )
	parser.add_argument(
	    "--output_dir"	,      type=_A	,      default="."	,      help="Optional save directory where all checkpoint folders will be stored. Default is the current working directory."	,      )
	parser.add_argument(
	    "--performance_lower_bound"	,      type=_A	,      default=_A	,      help="Optional lower bound for the performance metric. If set, the training will throw error when the performance metric drops below this value."	,      )
	parser.add_argument(
	    "--num_epochs"	,      type=_A	,      default=3	,      help="Number of train epochs."	,      )
	snake_case_						       =	parser.parse_args()
	snake_case_						       =	{"lr": 2E-5, "num_epochs": args.num_epochs, "seed": 42, "batch_size": 16}
	training_function(_A	,      _A				)


if __name__ == "__main__":
		main()
