


import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


_lowercase       :      Dict	=	logging.getLogger(__name__)

_lowercase       :      Optional[int]	=	tf.data.AUTOTUNE





def   _lowerCAmelCase				(   ) ->   List[Any]:





	"""simple docstring"""







	A								=				argparse.ArgumentParser(description="""Train a masked language model on TPU."""						)
	parser.add_argument(
	    """--pretrained_model_config"""							,					type=UpperCamelCase__							,					default="""roberta-base"""							,					help="""The model config to use. Note that we don't copy the model's weights, only the config!"""							,					)
	parser.add_argument(
	    """--tokenizer"""							,					type=UpperCamelCase__							,					default="""unigram-tokenizer-wikitext"""							,					help="""The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model's vocab size."""							,					)

	parser.add_argument(
	    """--per_replica_batch_size"""							,					type=UpperCamelCase__							,					default=8							,					help="""Batch size per TPU core."""							,					)

	parser.add_argument(
	    """--no_tpu"""							,					action="""store_true"""							,					help="""If set, run on CPU and don't try to initialize a TPU. Useful for debugging on non-TPU instances."""							,					)

	parser.add_argument(
	    """--tpu_name"""							,					type=UpperCamelCase__							,					help="""Name of TPU resource to initialize. Should be blank on Colab, and 'local' on TPU VMs."""							,					default="""local"""							,					)

	parser.add_argument(
	    """--tpu_zone"""							,					type=UpperCamelCase__							,					help="""Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes."""							,					)

	parser.add_argument(
	    """--gcp_project"""							,					type=UpperCamelCase__							,					help="""Google cloud project name. Only used for non-Colab TPU nodes."""						)

	parser.add_argument(
	    """--bfloat16"""							,					action="""store_true"""							,					help="""Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU."""							,					)

	parser.add_argument(
	    """--train_dataset"""							,					type=UpperCamelCase__							,					help="""Path to training dataset to load. If the path begins with `gs://`"""
	    """ then the dataset will be loaded from a Google Cloud Storage bucket."""							,					)

	parser.add_argument(
	    """--shuffle_buffer_size"""							,					type=UpperCamelCase__							,					default=2**18							,					help="""Size of the shuffle buffer (in samples)"""							,					)

	parser.add_argument(
	    """--eval_dataset"""							,					type=UpperCamelCase__							,					help="""Path to evaluation dataset to load. If the path begins with `gs://`"""
	    """ then the dataset will be loaded from a Google Cloud Storage bucket."""							,					)

	parser.add_argument(
	    """--num_epochs"""							,					type=UpperCamelCase__							,					default=1							,					help="""Number of epochs to train for."""							,					)

	parser.add_argument(
	    """--learning_rate"""							,					type=UpperCamelCase__							,					default=1e-4							,					help="""Learning rate to use for training."""							,					)

	parser.add_argument(
	    """--weight_decay_rate"""							,					type=UpperCamelCase__							,					default=1e-3							,					help="""Weight decay rate to use for training."""							,					)

	parser.add_argument(
	    """--max_length"""							,					type=UpperCamelCase__							,					default=5_12							,					help="""Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py"""							,					)

	parser.add_argument(
	    """--mlm_probability"""							,					type=UpperCamelCase__							,					default=0.15							,					help="""Fraction of tokens to mask during training."""							,					)

	parser.add_argument("""--output_dir"""							,					type=UpperCamelCase__							,					required=UpperCamelCase__							,					help="""Path to save model checkpoints to."""						)
	parser.add_argument("""--hub_model_id"""							,					type=UpperCamelCase__							,					help="""Model ID to upload to on the Hugging Face Hub."""						)

	A								=				parser.parse_args()
	return args





def   _lowerCAmelCase				(   UpperCamelCase__:     Dict						) ->   str:





	"""simple docstring"""







	try:
		if args.tpu_name:
			A								=				tf.distribute.cluster_resolver.TPUClusterResolver(
			    args.tpu_name							,					zone=args.tpu_zone							,					project=args.gcp_project						)
		else:
			A								=				tf.distribute.cluster_resolver.TPUClusterResolver()
	except ValueError:
		raise RuntimeError(
		    """Couldn't connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or """
		    """--gcp_project. When running on a TPU VM, use --tpu_name local."""						)

	tf.config.experimental_connect_to_cluster(UpperCamelCase__						)
	tf.tpu.experimental.initialize_tpu_system(UpperCamelCase__						)

	return tpu





def   _lowerCAmelCase				(   UpperCamelCase__:     List[str]						) ->   str:





	"""simple docstring"""







	A								=				0
	for file in file_list:
		A								=				file.split("""/"""						)[-1]
		A								=				re.search(r"""-\d+-(\d+)\.tfrecord"""							,					UpperCamelCase__						).group(1						)
		A								=				int(UpperCamelCase__						)
		num_samples += sample_count

	return num_samples





def   _lowerCAmelCase				(   UpperCamelCase__:     Union[str, Any]							,					UpperCamelCase__:     Dict							,					UpperCamelCase__:     str							,					UpperCamelCase__:     Union[str, Any]							,					UpperCamelCase__:     Optional[int]							,					UpperCamelCase__:     int=None						) ->   Tuple:





	"""simple docstring"""







	A								=				count_samples(UpperCamelCase__						)
	A								=				tf.data.Dataset.from_tensor_slices(UpperCamelCase__						)
	if shuffle:
		A								=				dataset.shuffle(len(UpperCamelCase__						)						)
	A								=				tf.data.TFRecordDataset(UpperCamelCase__							,					num_parallel_reads=UpperCamelCase__						)
	# TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
	A								=				dataset.apply(tf.data.experimental.assert_cardinality(UpperCamelCase__						)						)
	A								=				dataset.map(UpperCamelCase__							,					num_parallel_calls=UpperCamelCase__						)
	if shuffle:
		assert shuffle_buffer_size is not None
		A								=				dataset.shuffle(args.shuffle_buffer_size						)
	A								=				dataset.batch(UpperCamelCase__							,					drop_remainder=UpperCamelCase__						)
	A								=				dataset.map(UpperCamelCase__							,					num_parallel_calls=UpperCamelCase__						)
	A								=				dataset.prefetch(UpperCamelCase__						)
	return dataset





def   _lowerCAmelCase				(   UpperCamelCase__:     Any						) ->   Optional[int]:





	"""simple docstring"""







	if not args.no_tpu:
		A								=				initialize_tpu(UpperCamelCase__						)
		A								=				tf.distribute.TPUStrategy(UpperCamelCase__						)
	else:
		A								=				tf.distribute.OneDeviceStrategy(device="""/gpu:0"""						)

	if args.bfloataa:
		tf.keras.mixed_precision.set_global_policy("""mixed_bfloat16"""						)

	A								=				AutoTokenizer.from_pretrained(args.tokenizer						)
	A								=				AutoConfig.from_pretrained(args.pretrained_model_config						)
	A								=				tokenizer.vocab_size

	A								=				tf.io.gfile.glob(os.path.join(args.train_dataset							,					"""*.tfrecord"""						)						)
	if not training_records:
		raise ValueError(f'No .tfrecord files found in {args.train_dataset}.'						)
	A								=				tf.io.gfile.glob(os.path.join(args.eval_dataset							,					"""*.tfrecord"""						)						)
	if not eval_records:
		raise ValueError(f'No .tfrecord files found in {args.eval_dataset}.'						)

	A								=				count_samples(UpperCamelCase__						)

	A								=				num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
	A								=				steps_per_epoch * args.num_epochs

	with strategy.scope():
		A								=				TFAutoModelForMaskedLM.from_config(UpperCamelCase__						)
		model(model.dummy_inputs						)  # Pass some dummy inputs through the model to ensure all the weights are built
		A					,			A								=				create_optimizer(
		    num_train_steps=UpperCamelCase__							,					num_warmup_steps=total_train_steps // 20							,					init_lr=args.learning_rate							,					weight_decay_rate=args.weight_decay_rate							,					)

		# Transformers models compute the right loss for their task by default when labels are passed, and will
		# use this for training unless you specify your own loss function in compile().
		model.compile(optimizer=UpperCamelCase__							,					metrics=["""accuracy"""]						)

	def decode_fn(UpperCamelCase__:     Optional[Any]						):
		A								=				{
		    """input_ids""": tf.io.FixedLenFeature(dtype=tf.intaa							,					shape=(args.max_length,)						),
		    """attention_mask""": tf.io.FixedLenFeature(dtype=tf.intaa							,					shape=(args.max_length,)						),
		}
		return tf.io.parse_single_example(UpperCamelCase__							,					UpperCamelCase__						)

	# Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
	# use their methods in our data pipeline.
	A								=				DataCollatorForLanguageModeling(
	    tokenizer=UpperCamelCase__							,					mlm_probability=args.mlm_probability							,					mlm=UpperCamelCase__							,					return_tensors="""tf"""						)

	def mask_with_collator(UpperCamelCase__:     List[str]						):
		# TF really needs an isin() function
		A								=				(
		    ~tf.cast(batch["""attention_mask"""]							,					tf.bool						)
		    | (batch["""input_ids"""] == tokenizer.cls_token_id)
		    | (batch["""input_ids"""] == tokenizer.sep_token_id)
		)
		A					,			A								=				data_collator.tf_mask_tokens(
		    batch["""input_ids"""]							,					vocab_size=len(UpperCamelCase__						)							,					mask_token_id=tokenizer.mask_token_id							,					special_tokens_mask=UpperCamelCase__							,					)
		return batch

	A								=				args.per_replica_batch_size * strategy.num_replicas_in_sync

	A								=				prepare_dataset(
	    UpperCamelCase__							,					decode_fn=UpperCamelCase__							,					mask_fn=UpperCamelCase__							,					batch_size=UpperCamelCase__							,					shuffle=UpperCamelCase__							,					shuffle_buffer_size=args.shuffle_buffer_size							,					)

	A								=				prepare_dataset(
	    UpperCamelCase__							,					decode_fn=UpperCamelCase__							,					mask_fn=UpperCamelCase__							,					batch_size=UpperCamelCase__							,					shuffle=UpperCamelCase__							,					)

	A								=				[]
	if args.hub_model_id:
		callbacks.append(
		    PushToHubCallback(output_dir=args.output_dir							,					hub_model_id=args.hub_model_id							,					tokenizer=UpperCamelCase__						)						)

	model.fit(
	    UpperCamelCase__							,					validation_data=UpperCamelCase__							,					epochs=args.num_epochs							,					callbacks=UpperCamelCase__							,					)

	model.save_pretrained(args.output_dir						)


if __name__ == "__main__":
		_lowercase       :      str	=	parse_args()
		main(args)







from sklearn.metrics import recall_score

import datasets


_lowercase       :      Any	=	"\nRecall is the fraction of the positive examples that were correctly labeled by the model as positive. It can be computed with the equation:\nRecall = TP / (TP + FN)\nWhere TP is the true positives and FN is the false negatives.\n"


_lowercase       :      int	=	"\nArgs:\n- **predictions** (`list` of `int`): The predicted labels.\n- **references** (`list` of `int`): The ground truth labels.\n- **labels** (`list` of `int`): The set of labels to include when `average` is not set to `binary`, and their order when average is `None`. Labels present in the data can be excluded in this input, for example to calculate a multiclass average ignoring a majority negative class, while labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in y_true and y_pred are used in sorted order. Defaults to None.\n- **pos_label** (`int`): The class label to use as the 'positive class' when calculating the recall. Defaults to `1`.\n- **average** (`string`): This parameter is required for multiclass/multilabel targets. If None, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`.\n    - `'binary'`: Only report results for the class specified by `pos_label`. This is applicable only if the target labels and predictions are binary.\n    - `'micro'`: Calculate metrics globally by counting the total true positives, false negatives, and false positives.\n    - `'macro'`: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account.\n    - `'weighted'`: Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. Note that it can result in an F-score that is not between precision and recall.\n    - `'samples'`: Calculate metrics for each instance, and find their average (only meaningful for multilabel classification).\n- **sample_weight** (`list` of `float`): Sample weights Defaults to `None`.\n- **zero_division** (): Sets the value to return when there is a zero division. Defaults to .\n    - `'warn'`: If there is a zero division, the return value is `0`, but warnings are also raised.\n    - `0`: If there is a zero division, the return value is `0`.\n    - `1`: If there is a zero division, the return value is `1`.\n\nReturns:\n- **recall** (`float`, or `array` of `float`): Either the general recall score, or the recall scores for individual classes, depending on the values input to `labels` and `average`. Minimum possible value is 0. Maximum possible value is 1. A higher recall means that more of the positive examples have been labeled correctly. Therefore, a higher recall is generally considered better.\n\nExamples:\n\n    Example 1-A simple example with some errors\n        >>> recall_metric = datasets.load_metric('recall')\n        >>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1])\n        >>> print(results)\n        {'recall': 0.6666666666666666}\n\n    Example 2-The same example as Example 1, but with `pos_label=0` instead of the default `pos_label=1`.\n        >>> recall_metric = datasets.load_metric('recall')\n        >>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1], pos_label=0)\n        >>> print(results)\n        {'recall': 0.5}\n\n    Example 3-The same example as Example 1, but with `sample_weight` included.\n        >>> recall_metric = datasets.load_metric('recall')\n        >>> sample_weight = [0.9, 0.2, 0.9, 0.3, 0.8]\n        >>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1], sample_weight=sample_weight)\n        >>> print(results)\n        {'recall': 0.55}\n\n    Example 4-A multiclass example, using different averages.\n        >>> recall_metric = datasets.load_metric('recall')\n        >>> predictions = [0, 2, 1, 0, 0, 1]\n        >>> references = [0, 1, 2, 0, 1, 2]\n        >>> results = recall_metric.compute(predictions=predictions, references=references, average='macro')\n        >>> print(results)\n        {'recall': 0.3333333333333333}\n        >>> results = recall_metric.compute(predictions=predictions, references=references, average='micro')\n        >>> print(results)\n        {'recall': 0.3333333333333333}\n        >>> results = recall_metric.compute(predictions=predictions, references=references, average='weighted')\n        >>> print(results)\n        {'recall': 0.3333333333333333}\n        >>> results = recall_metric.compute(predictions=predictions, references=references, average=None)\n        >>> print(results)\n        {'recall': array([1., 0., 0.])}\n"


_lowercase       :      Dict	=	"\n@article{scikit-learn, title={Scikit-learn: Machine Learning in {P}ython}, author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, journal={Journal of Machine Learning Research}, volume={12}, pages={2825--2830}, year={2011}\n"






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION    ,		_KWARGS_DESCRIPTION				)
class       _UpperCamelCase    (					datasets.Metric				):

				"""simple docstring"""







				def      _UpperCAmelCase					(						self   )	->				Tuple:
					return datasets.MetricInfo(
					    description=_DESCRIPTION				,   citation=_CITATION				,   inputs_description=_KWARGS_DESCRIPTION				,   features=datasets.Features(
					        {
					            """predictions""": datasets.Sequence(datasets.Value("""int32"""   )   ),
					            """references""": datasets.Sequence(datasets.Value("""int32"""   )   ),
					        }
					        if self.config_name == """multilabel"""
					        else {
					            """predictions""": datasets.Value("""int32"""   ),
					            """references""": datasets.Value("""int32"""   ),
					        }   )				,   reference_urls=["""https://scikit-learn.org/stable/modules/generated/sklearn.metrics.recall_score.html"""]				,   )

				def      _UpperCAmelCase					(						self				,   a__				,   a__				,   a__=None				,   a__=1				,   a__="binary"				,   a__=None				,   a__="warn"				,   )	->				Any:
					A								=				recall_score(
					    a__				,   a__				,   labels=a__				,   pos_label=a__				,   average=a__				,   sample_weight=a__				,   zero_division=a__				,   )
					return {"recall": float(a__   ) if score.size == 1 else score}



