



'''simple docstring'''





import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


UpperCAmelCase    					=   logging.getLogger(__name__)

UpperCAmelCase    					=   tf.data.AUTOTUNE

def      A (       ):
						snake_case       :     int =    argparse.ArgumentParser(description='''Train a masked language model on TPU.'''       )
						parser.add_argument(
						    '''--pretrained_model_config'''    ,							type=A_    ,							default='''roberta-base'''    ,							help='''The model config to use. Note that we don\'t copy the model\'s weights, only the config!'''    ,							)
						parser.add_argument(
						    '''--tokenizer'''    ,							type=A_    ,							default='''unigram-tokenizer-wikitext'''    ,							help='''The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model\'s vocab size.'''    ,							)

						parser.add_argument(
						    '''--per_replica_batch_size'''    ,							type=A_    ,							default=8    ,							help='''Batch size per TPU core.'''    ,							)

						parser.add_argument(
						    '''--no_tpu'''    ,							action='''store_true'''    ,							help='''If set, run on CPU and don\'t try to initialize a TPU. Useful for debugging on non-TPU instances.'''    ,							)

						parser.add_argument(
						    '''--tpu_name'''    ,							type=A_    ,							help='''Name of TPU resource to initialize. Should be blank on Colab, and \'local\' on TPU VMs.'''    ,							default='''local'''    ,							)

						parser.add_argument(
						    '''--tpu_zone'''    ,							type=A_    ,							help='''Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes.'''    ,							)

						parser.add_argument(
						    '''--gcp_project'''    ,							type=A_    ,							help='''Google cloud project name. Only used for non-Colab TPU nodes.'''       )

						parser.add_argument(
						    '''--bfloat16'''    ,							action='''store_true'''    ,							help='''Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU.'''    ,							)

						parser.add_argument(
						    '''--train_dataset'''    ,							type=A_    ,							help='''Path to training dataset to load. If the path begins with `gs://`'''
						    ''' then the dataset will be loaded from a Google Cloud Storage bucket.'''    ,							)

						parser.add_argument(
						    '''--shuffle_buffer_size'''    ,							type=A_    ,							default=2**18    ,							help='''Size of the shuffle buffer (in samples)'''    ,							)

						parser.add_argument(
						    '''--eval_dataset'''    ,							type=A_    ,							help='''Path to evaluation dataset to load. If the path begins with `gs://`'''
						    ''' then the dataset will be loaded from a Google Cloud Storage bucket.'''    ,							)

						parser.add_argument(
						    '''--num_epochs'''    ,							type=A_    ,							default=1    ,							help='''Number of epochs to train for.'''    ,							)

						parser.add_argument(
						    '''--learning_rate'''    ,							type=A_    ,							default=1e-4    ,							help='''Learning rate to use for training.'''    ,							)

						parser.add_argument(
						    '''--weight_decay_rate'''    ,							type=A_    ,							default=1e-3    ,							help='''Weight decay rate to use for training.'''    ,							)

						parser.add_argument(
						    '''--max_length'''    ,							type=A_    ,							default=512    ,							help='''Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py'''    ,							)

						parser.add_argument(
						    '''--mlm_probability'''    ,							type=A_    ,							default=0.15    ,							help='''Fraction of tokens to mask during training.'''    ,							)

						parser.add_argument('''--output_dir'''    ,							type=A_    ,							required=A_    ,							help='''Path to save model checkpoints to.'''       )
						parser.add_argument('''--hub_model_id'''    ,							type=A_    ,							help='''Model ID to upload to on the Hugging Face Hub.'''       )

						snake_case       :     Optional[int] =    parser.parse_args()
						return args

def      A (       A_							:					List[str]       ):
						try:
												if args.tpu_name:
																		snake_case       :     Tuple =    tf.distribute.cluster_resolver.TPUClusterResolver(
																		    args.tpu_name    ,							zone=args.tpu_zone    ,							project=args.gcp_project       )
												else:
																		snake_case       :     Union[str, Any] =    tf.distribute.cluster_resolver.TPUClusterResolver()
						except ValueError:
												raise RuntimeError(
												    '''Couldn\'t connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or '''
												    '''--gcp_project. When running on a TPU VM, use --tpu_name local.'''       )

						tf.config.experimental_connect_to_cluster(A_       )
						tf.tpu.experimental.initialize_tpu_system(A_       )

						return tpu

def      A (       A_							:					Optional[int]       ):
						snake_case       :     Optional[int] =    0
						for file in file_list:
												snake_case       :     str =    file.split('''/'''       )[-1]
												snake_case       :     str =    re.search(R'''-\d+-(\d+)\.tfrecord'''    ,							A_       ).group(1       )
												snake_case       :     Tuple =    int(A_       )
												num_samples += sample_count

						return num_samples

def      A (       A_							:					str    ,							A_							:					str    ,							A_							:					List[Any]    ,							A_							:					Any    ,							A_							:					Optional[Any]    ,							A_							:					List[Any]=None       ):
						snake_case       :     List[str] =    count_samples(A_       )
						snake_case       :     List[Any] =    tf.data.Dataset.from_tensor_slices(A_       )
						if shuffle:
												snake_case       :     List[Any] =    dataset.shuffle(len(A_       )       )
						snake_case       :     Tuple =    tf.data.TFRecordDataset(A_    ,							num_parallel_reads=A_       )
						# TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
						snake_case       :     Optional[Any] =    dataset.apply(tf.data.experimental.assert_cardinality(A_       )       )
						snake_case       :     Any =    dataset.map(A_    ,							num_parallel_calls=A_       )
						if shuffle:
												assert shuffle_buffer_size is not None
												snake_case       :     str =    dataset.shuffle(args.shuffle_buffer_size       )
						snake_case       :     Tuple =    dataset.batch(A_    ,							drop_remainder=A_       )
						snake_case       :     List[Any] =    dataset.map(A_    ,							num_parallel_calls=A_       )
						snake_case       :     List[str] =    dataset.prefetch(A_       )
						return dataset

def      A (       A_							:					Union[str, Any]       ):
						if not args.no_tpu:
												snake_case       :     Optional[Any] =    initialize_tpu(A_       )
												snake_case       :     Dict =    tf.distribute.TPUStrategy(A_       )
						else:
												snake_case       :     Tuple =    tf.distribute.OneDeviceStrategy(device='''/gpu:0'''       )

						if args.bfloataa:
												tf.keras.mixed_precision.set_global_policy('''mixed_bfloat16'''       )

						snake_case       :     List[Any] =    AutoTokenizer.from_pretrained(args.tokenizer       )
						snake_case       :     Dict =    AutoConfig.from_pretrained(args.pretrained_model_config       )
						snake_case       :     List[str] =    tokenizer.vocab_size

						snake_case       :     Dict =    tf.io.gfile.glob(os.path.join(args.train_dataset    ,							'''*.tfrecord'''       )       )
						if not training_records:
												raise ValueError(F"""No .tfrecord files found in {args.train_dataset}."""       )
						snake_case       :     List[Any] =    tf.io.gfile.glob(os.path.join(args.eval_dataset    ,							'''*.tfrecord'''       )       )
						if not eval_records:
												raise ValueError(F"""No .tfrecord files found in {args.eval_dataset}."""       )

						snake_case       :     str =    count_samples(A_       )

						snake_case       :     Tuple =    num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
						snake_case       :     str =    steps_per_epoch * args.num_epochs

						with strategy.scope():
												snake_case       :     Optional[int] =    TFAutoModelForMaskedLM.from_config(A_       )
												model(model.dummy_inputs       )  # Pass some dummy inputs through the model to ensure all the weights are built
												snake_case       :     List[str] =    create_optimizer(
												    num_train_steps=A_    ,							num_warmup_steps=total_train_steps // 20    ,							init_lr=args.learning_rate    ,							weight_decay_rate=args.weight_decay_rate    ,							)

												# Transformers models compute the right loss for their task by default when labels are passed, and will
												# use this for training unless you specify your own loss function in compile().
												model.compile(optimizer=A_    ,							metrics=['''accuracy''']       )

						def decode_fn(A_							:					Any       ):
												snake_case       :     Optional[int] =    {
												    '''input_ids''': tf.io.FixedLenFeature(dtype=tf.intaa    ,							shape=(args.max_length,)       ),
												    '''attention_mask''': tf.io.FixedLenFeature(dtype=tf.intaa    ,							shape=(args.max_length,)       ),
												}
												return tf.io.parse_single_example(A_    ,							A_       )

						# Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
						# use their methods in our data pipeline.
						snake_case       :     Optional[Any] =    DataCollatorForLanguageModeling(
						    tokenizer=A_    ,							mlm_probability=args.mlm_probability    ,							mlm=A_    ,							return_tensors='''tf'''       )

						def mask_with_collator(A_							:					Optional[int]       ):
												# TF really needs an isin() function
												snake_case       :     Dict =    (
												    ~tf.cast(batch['''attention_mask''']    ,							tf.bool       )
												    | (batch['''input_ids'''] == tokenizer.cls_token_id)
												    | (batch['''input_ids'''] == tokenizer.sep_token_id)
												)
												snake_case       :     str =    data_collator.tf_mask_tokens(
												    batch['''input_ids''']    ,							vocab_size=len(A_       )    ,							mask_token_id=tokenizer.mask_token_id    ,							special_tokens_mask=A_    ,							)
												return batch

						snake_case       :     List[str] =    args.per_replica_batch_size * strategy.num_replicas_in_sync

						snake_case       :     Optional[int] =    prepare_dataset(
						    A_    ,							decode_fn=A_    ,							mask_fn=A_    ,							batch_size=A_    ,							shuffle=A_    ,							shuffle_buffer_size=args.shuffle_buffer_size    ,							)

						snake_case       :     Optional[Any] =    prepare_dataset(
						    A_    ,							decode_fn=A_    ,							mask_fn=A_    ,							batch_size=A_    ,							shuffle=A_    ,							)

						snake_case       :     Optional[Any] =    []
						if args.hub_model_id:
												callbacks.append(
												    PushToHubCallback(output_dir=args.output_dir    ,							hub_model_id=args.hub_model_id    ,							tokenizer=A_       )       )

						model.fit(
						    A_    ,							validation_data=A_    ,							epochs=args.num_epochs    ,							callbacks=A_    ,							)

						model.save_pretrained(args.output_dir       )


if __name__ == "__main__":
						UpperCAmelCase    					=   parse_args()
						main(args)



'''simple docstring'''





import inspect
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
UpperCAmelCase    					=   "src/transformers"


# This is to make sure the transformers module imported is the one in the repo.
UpperCAmelCase    					=   direct_transformers_import(PATH_TO_TRANSFORMERS)

UpperCAmelCase    					=   transformers.models.auto.configuration_auto.CONFIG_MAPPING

# Regex pattern used to find the checkpoint mentioned in the docstring of `config_class`.
# For example, `[bert-base-uncased](https://huggingface.co/bert-base-uncased)`
UpperCAmelCase    					=   re.compile(r"\[(.+?)\]\((https://huggingface\.co/.+?)\)")


UpperCAmelCase    					=   {
    "DecisionTransformerConfig",
    "EncoderDecoderConfig",
    "MusicgenConfig",
    "RagConfig",
    "SpeechEncoderDecoderConfig",
    "TimmBackboneConfig",
    "VisionEncoderDecoderConfig",
    "VisionTextDualEncoderConfig",
    "LlamaConfig",
}

def      A (       A_							:					Union[str, Any]       ):
						snake_case       :     Tuple =    None

						# source code of `config_class`
						snake_case       :     Union[str, Any] =    inspect.getsource(A_       )
						snake_case       :     List[Any] =    _re_checkpoint.findall(A_       )

						# Each `checkpoint` is a tuple of a checkpoint name and a checkpoint link.
						# For example, `('bert-base-uncased', 'https://huggingface.co/bert-base-uncased')`
						for ckpt_name, ckpt_link in checkpoints:
												# allow the link to end with `/`
												if ckpt_link.endswith('''/'''       ):
																		snake_case       :     Optional[int] =    ckpt_link[:-1]

												# verify the checkpoint name corresponds to the checkpoint link
												snake_case       :     Dict =    F"""https://huggingface.co/{ckpt_name}"""
												if ckpt_link == ckpt_link_from_name:
																		snake_case       :     Dict =    ckpt_name
																		break

						return checkpoint

def      A (       ):
						snake_case       :     List[str] =    []

						for config_class in list(CONFIG_MAPPING.values()       ):
												# Skip deprecated models
												if "models.deprecated" in config_class.__module__:
																		continue
												snake_case       :     Tuple =    get_checkpoint_from_config_class(A_       )

												snake_case       :     Optional[Any] =    config_class.__name__
												if checkpoint is None and name not in CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK:
																		configs_without_checkpoint.append(A_       )

						if len(A_       ) > 0:
												snake_case       :     Optional[int] =    '''\n'''.join(sorted(A_       )       )
												raise ValueError(F"""The following configurations don't contain any valid checkpoint:\n{message}"""       )


if __name__ == "__main__":
						check_config_docstrings_have_checkpoints()

