


"""simple docstring"""
import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


UpperCAmelCase				=						logging.getLogger(__name__)

UpperCAmelCase				=						tf.data.AUTOTUNE

def 				_snake_case      (   ):




   """simple docstring"""







   _lowerCamelCase	:					Tuple   =		argparse.ArgumentParser(description="""Train a masked language model on TPU."""   )
   parser.add_argument(
       """--pretrained_model_config"""						, type=__snake_case						, default="""roberta-base"""						, help="""The model config to use. Note that we don't copy the model's weights, only the config!"""						, )
   parser.add_argument(
       """--tokenizer"""						, type=__snake_case						, default="""unigram-tokenizer-wikitext"""						, help="""The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model's vocab size."""						, )

   parser.add_argument(
       """--per_replica_batch_size"""						, type=__snake_case						, default=8						, help="""Batch size per TPU core."""						, )

   parser.add_argument(
       """--no_tpu"""						, action="""store_true"""						, help="""If set, run on CPU and don't try to initialize a TPU. Useful for debugging on non-TPU instances."""						, )

   parser.add_argument(
       """--tpu_name"""						, type=__snake_case						, help="""Name of TPU resource to initialize. Should be blank on Colab, and 'local' on TPU VMs."""						, default="""local"""						, )

   parser.add_argument(
       """--tpu_zone"""						, type=__snake_case						, help="""Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes."""						, )

   parser.add_argument(
       """--gcp_project"""						, type=__snake_case						, help="""Google cloud project name. Only used for non-Colab TPU nodes."""   )

   parser.add_argument(
       """--bfloat16"""						, action="""store_true"""						, help="""Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU."""						, )

   parser.add_argument(
       """--train_dataset"""						, type=__snake_case						, help="""Path to training dataset to load. If the path begins with `gs://`"""
       """ then the dataset will be loaded from a Google Cloud Storage bucket."""						, )

   parser.add_argument(
       """--shuffle_buffer_size"""						, type=__snake_case						, default=2**18						, help="""Size of the shuffle buffer (in samples)"""						, )

   parser.add_argument(
       """--eval_dataset"""						, type=__snake_case						, help="""Path to evaluation dataset to load. If the path begins with `gs://`"""
       """ then the dataset will be loaded from a Google Cloud Storage bucket."""						, )

   parser.add_argument(
       """--num_epochs"""						, type=__snake_case						, default=1						, help="""Number of epochs to train for."""						, )

   parser.add_argument(
       """--learning_rate"""						, type=__snake_case						, default=1E-4						, help="""Learning rate to use for training."""						, )

   parser.add_argument(
       """--weight_decay_rate"""						, type=__snake_case						, default=1E-3						, help="""Weight decay rate to use for training."""						, )

   parser.add_argument(
       """--max_length"""						, type=__snake_case						, default=512						, help="""Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py"""						, )

   parser.add_argument(
       """--mlm_probability"""						, type=__snake_case						, default=0.15						, help="""Fraction of tokens to mask during training."""						, )

   parser.add_argument("""--output_dir"""						, type=__snake_case						, required=__snake_case						, help="""Path to save model checkpoints to."""   )
   parser.add_argument("""--hub_model_id"""						, type=__snake_case						, help="""Model ID to upload to on the Hugging Face Hub."""   )

   _lowerCamelCase	:					Any   =		parser.parse_args()
   return args

def 				_snake_case      (   __snake_case							:      str   ):




   """simple docstring"""







   try:
      if args.tpu_name:
         _lowerCamelCase	:					List[str]   =		tf.distribute.cluster_resolver.TPUClusterResolver(
             args.tpu_name						, zone=args.tpu_zone						, project=args.gcp_project   )
      else:
         _lowerCamelCase	:					Optional[Any]   =		tf.distribute.cluster_resolver.TPUClusterResolver()
   except ValueError:
      raise RuntimeError(
          """Couldn't connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or """
          """--gcp_project. When running on a TPU VM, use --tpu_name local."""   )

   tf.config.experimental_connect_to_cluster(__snake_case   )
   tf.tpu.experimental.initialize_tpu_system(__snake_case   )

   return tpu

def 				_snake_case      (   __snake_case							:      List[Any]   ):




   """simple docstring"""







   _lowerCamelCase	:					List[Any]   =		0
   for file in file_list:
      _lowerCamelCase	:					str   =		file.split("""/"""   )[-1]
      _lowerCamelCase	:					str   =		re.search(R"""-\d+-(\d+)\.tfrecord"""						, __snake_case   ).group(1   )
      _lowerCamelCase	:					Dict   =		int(__snake_case   )
      num_samples += sample_count

   return num_samples

def 				_snake_case      (   __snake_case							:      int						, __snake_case							:      List[str]						, __snake_case							:      Any						, __snake_case							:      Optional[Any]						, __snake_case							:      int						, __snake_case							:      List[Any]=None   ):




   """simple docstring"""







   _lowerCamelCase	:					Dict   =		count_samples(__snake_case   )
   _lowerCamelCase	:					Any   =		tf.data.Dataset.from_tensor_slices(__snake_case   )
   if shuffle:
      _lowerCamelCase	:					str   =		dataset.shuffle(len(__snake_case   )   )
   _lowerCamelCase	:					Tuple   =		tf.data.TFRecordDataset(__snake_case						, num_parallel_reads=__snake_case   )
   # TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
   _lowerCamelCase	:					List[Any]   =		dataset.apply(tf.data.experimental.assert_cardinality(__snake_case   )   )
   _lowerCamelCase	:					Union[str, Any]   =		dataset.map(__snake_case						, num_parallel_calls=__snake_case   )
   if shuffle:
      assert shuffle_buffer_size is not None
      _lowerCamelCase	:					Any   =		dataset.shuffle(args.shuffle_buffer_size   )
   _lowerCamelCase	:					Any   =		dataset.batch(__snake_case						, drop_remainder=__snake_case   )
   _lowerCamelCase	:					List[Any]   =		dataset.map(__snake_case						, num_parallel_calls=__snake_case   )
   _lowerCamelCase	:					Tuple   =		dataset.prefetch(__snake_case   )
   return dataset

def 				_snake_case      (   __snake_case							:      int   ):




   """simple docstring"""







   if not args.no_tpu:
      _lowerCamelCase	:					Union[str, Any]   =		initialize_tpu(__snake_case   )
      _lowerCamelCase	:					str   =		tf.distribute.TPUStrategy(__snake_case   )
   else:
      _lowerCamelCase	:					Tuple   =		tf.distribute.OneDeviceStrategy(device="""/gpu:0"""   )

   if args.bfloataa:
      tf.keras.mixed_precision.set_global_policy("""mixed_bfloat16"""   )

   _lowerCamelCase	:					Tuple   =		AutoTokenizer.from_pretrained(args.tokenizer   )
   _lowerCamelCase	:					Union[str, Any]   =		AutoConfig.from_pretrained(args.pretrained_model_config   )
   _lowerCamelCase	:					Any   =		tokenizer.vocab_size

   _lowerCamelCase	:					Optional[Any]   =		tf.io.gfile.glob(os.path.join(args.train_dataset						, """*.tfrecord"""   )   )
   if not training_records:
      raise ValueError(F'No .tfrecord files found in {args.train_dataset}.'   )
   _lowerCamelCase	:					Any   =		tf.io.gfile.glob(os.path.join(args.eval_dataset						, """*.tfrecord"""   )   )
   if not eval_records:
      raise ValueError(F'No .tfrecord files found in {args.eval_dataset}.'   )

   _lowerCamelCase	:					Dict   =		count_samples(__snake_case   )

   _lowerCamelCase	:					str   =		num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
   _lowerCamelCase	:					int   =		steps_per_epoch * args.num_epochs

   with strategy.scope():
      _lowerCamelCase	:					Any   =		TFAutoModelForMaskedLM.from_config(__snake_case   )
      model(model.dummy_inputs   )  # Pass some dummy inputs through the model to ensure all the weights are built
      _lowerCamelCase   ,    _lowerCamelCase	:					Dict   =		create_optimizer(
          num_train_steps=__snake_case						, num_warmup_steps=total_train_steps // 20						, init_lr=args.learning_rate						, weight_decay_rate=args.weight_decay_rate						, )

      # Transformers models compute the right loss for their task by default when labels are passed, and will
      # use this for training unless you specify your own loss function in compile().
      model.compile(optimizer=__snake_case						, metrics=["""accuracy"""]   )

   def decode_fn(__snake_case							:      Optional[int]   ):
      _lowerCamelCase	:					str   =		{
          """input_ids""": tf.io.FixedLenFeature(dtype=tf.intaa						, shape=(args.max_length,)   ),
          """attention_mask""": tf.io.FixedLenFeature(dtype=tf.intaa						, shape=(args.max_length,)   ),
      }
      return tf.io.parse_single_example(__snake_case						, __snake_case   )

   # Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
   # use their methods in our data pipeline.
   _lowerCamelCase	:					Tuple   =		DataCollatorForLanguageModeling(
       tokenizer=__snake_case						, mlm_probability=args.mlm_probability						, mlm=__snake_case						, return_tensors="""tf"""   )

   def mask_with_collator(__snake_case							:      int   ):
      # TF really needs an isin() function
      _lowerCamelCase	:					Optional[Any]   =		(
          ~tf.cast(batch["""attention_mask"""]						, tf.bool   )
          | (batch["""input_ids"""] == tokenizer.cls_token_id)
          | (batch["""input_ids"""] == tokenizer.sep_token_id)
      )
      _lowerCamelCase   ,    _lowerCamelCase	:					Optional[int]   =		data_collator.tf_mask_tokens(
          batch["""input_ids"""]						, vocab_size=len(__snake_case   )						, mask_token_id=tokenizer.mask_token_id						, special_tokens_mask=__snake_case						, )
      return batch

   _lowerCamelCase	:					Optional[int]   =		args.per_replica_batch_size * strategy.num_replicas_in_sync

   _lowerCamelCase	:					Tuple   =		prepare_dataset(
       __snake_case						, decode_fn=__snake_case						, mask_fn=__snake_case						, batch_size=__snake_case						, shuffle=__snake_case						, shuffle_buffer_size=args.shuffle_buffer_size						, )

   _lowerCamelCase	:					int   =		prepare_dataset(
       __snake_case						, decode_fn=__snake_case						, mask_fn=__snake_case						, batch_size=__snake_case						, shuffle=__snake_case						, )

   _lowerCamelCase	:					Dict   =		[]
   if args.hub_model_id:
      callbacks.append(
          PushToHubCallback(output_dir=args.output_dir						, hub_model_id=args.hub_model_id						, tokenizer=__snake_case   )   )

   model.fit(
       __snake_case						, validation_data=__snake_case						, epochs=args.num_epochs						, callbacks=__snake_case						, )

   model.save_pretrained(args.output_dir   )


if __name__ == "__main__":
 UpperCAmelCase				=						parse_args()
 main(args)







'''simple docstring'''


import inspect
import os
import unittest

import torch

import accelerate
from accelerate import debug_launcher
from accelerate.test_utils import (
    execute_subprocess_async,
    require_cpu,
    require_huggingface_suite,
    require_multi_gpu,
    require_single_gpu,
)
from accelerate.utils import patch_environment





@require_huggingface_suite
class  __lowercase    (	unittest.TestCase			):







   def 		UpperCamelCase__      (						self   )					->     Tuple:
     __a								=       inspect.getfile(accelerate.test_utils   )
     __a								=       os.path.sep.join(
         mod_file.split(os.path.sep   )[:-1] + ['scripts', 'external_deps', 'test_metrics.py']   )

     from accelerate.test_utils.scripts.external_deps import test_metrics  # noqa: F401

     __a								=       test_metrics







   @require_cpu
   def 		UpperCamelCase__      (						self   )					->     Tuple:
     debug_launcher(self.test_metrics.main   ,				num_processes=1   )







   @require_cpu
   def 		UpperCamelCase__      (						self   )					->     str:
     debug_launcher(self.test_metrics.main   )







   @require_single_gpu
   def 		UpperCamelCase__      (						self   )					->     Dict:
     self.test_metrics.main()







   @require_multi_gpu
   def 		UpperCamelCase__      (						self   )					->     int:
     print(f"Found {torch.cuda.device_count()} devices."   )
     __a								=       ['torchrun', f"--nproc_per_node={torch.cuda.device_count()}", self.test_file_path]
     with patch_environment(omp_num_threads=1   ):
       execute_subprocess_async(UpperCamelCase   ,				env=os.environ.copy()   )



