


'''simple docstring'''




import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


UpperCAmelCase_      :		Optional[Any]          =		logging.getLogger(__name__)

UpperCAmelCase_      :		Optional[Any]          =		tf.data.AUTOTUNE


def    UpperCAmelCase_  (       ):
   '''simple docstring'''





   _a					:  List[str]						=  argparse.ArgumentParser(description='Train a masked language model on TPU.'      )
   parser.add_argument(
       '--pretrained_model_config'		,	type=A		,	default='roberta-base'		,	help='The model config to use. Note that we don\'t copy the model\'s weights, only the config!'		,	)
   parser.add_argument(
       '--tokenizer'		,	type=A		,	default='unigram-tokenizer-wikitext'		,	help='The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model\'s vocab size.'		,	)

   parser.add_argument(
       '--per_replica_batch_size'		,	type=A		,	default=8		,	help='Batch size per TPU core.'		,	)

   parser.add_argument(
       '--no_tpu'		,	action='store_true'		,	help='If set, run on CPU and don\'t try to initialize a TPU. Useful for debugging on non-TPU instances.'		,	)

   parser.add_argument(
       '--tpu_name'		,	type=A		,	help='Name of TPU resource to initialize. Should be blank on Colab, and \'local\' on TPU VMs.'		,	default='local'		,	)

   parser.add_argument(
       '--tpu_zone'		,	type=A		,	help='Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes.'		,	)

   parser.add_argument(
       '--gcp_project'		,	type=A		,	help='Google cloud project name. Only used for non-Colab TPU nodes.'      )

   parser.add_argument(
       '--bfloat16'		,	action='store_true'		,	help='Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU.'		,	)

   parser.add_argument(
       '--train_dataset'		,	type=A		,	help='Path to training dataset to load. If the path begins with `gs://`'
       ' then the dataset will be loaded from a Google Cloud Storage bucket.'		,	)

   parser.add_argument(
       '--shuffle_buffer_size'		,	type=A		,	default=2**1_8		,	help='Size of the shuffle buffer (in samples)'		,	)

   parser.add_argument(
       '--eval_dataset'		,	type=A		,	help='Path to evaluation dataset to load. If the path begins with `gs://`'
       ' then the dataset will be loaded from a Google Cloud Storage bucket.'		,	)

   parser.add_argument(
       '--num_epochs'		,	type=A		,	default=1		,	help='Number of epochs to train for.'		,	)

   parser.add_argument(
       '--learning_rate'		,	type=A		,	default=1E-4		,	help='Learning rate to use for training.'		,	)

   parser.add_argument(
       '--weight_decay_rate'		,	type=A		,	default=1E-3		,	help='Weight decay rate to use for training.'		,	)

   parser.add_argument(
       '--max_length'		,	type=A		,	default=5_1_2		,	help='Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py'		,	)

   parser.add_argument(
       '--mlm_probability'		,	type=A		,	default=0.15		,	help='Fraction of tokens to mask during training.'		,	)

   parser.add_argument('--output_dir'		,	type=A		,	required=A		,	help='Path to save model checkpoints to.'      )
   parser.add_argument('--hub_model_id'		,	type=A		,	help='Model ID to upload to on the Hugging Face Hub.'      )

   _a					:  Optional[Any]						=  parser.parse_args()
   return args


def    UpperCAmelCase_  (       A      ):
   '''simple docstring'''





   try:
      if args.tpu_name:
         _a					:  Dict						=  tf.distribute.cluster_resolver.TPUClusterResolver(
             args.tpu_name		,	zone=args.tpu_zone		,	project=args.gcp_project      )
      else:
         _a					:  List[str]						=  tf.distribute.cluster_resolver.TPUClusterResolver()
   except ValueError:
      raise RuntimeError(
          'Couldn\'t connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or '
          '--gcp_project. When running on a TPU VM, use --tpu_name local.'      )

   tf.config.experimental_connect_to_cluster(A      )
   tf.tpu.experimental.initialize_tpu_system(A      )

   return tpu


def    UpperCAmelCase_  (       A      ):
   '''simple docstring'''





   _a					:  Tuple						=  0
   for file in file_list:
      _a					:  int						=  file.split('/'      )[-1]
      _a					:  List[Any]						=  re.search(r'-\d+-(\d+)\.tfrecord'		,	A      ).group(1      )
      _a					:  List[Any]						=  int(A      )
      num_samples += sample_count

   return num_samples


def    UpperCAmelCase_  (       A		,	A		,	A		,	A		,	A		,	A=None      ):
   '''simple docstring'''





   _a					:  List[str]						=  count_samples(A      )
   _a					:  Union[str, Any]						=  tf.data.Dataset.from_tensor_slices(A      )
   if shuffle:
      _a					:  Optional[Any]						=  dataset.shuffle(len(A      )      )
   _a					:  Dict						=  tf.data.TFRecordDataset(A		,	num_parallel_reads=A      )
   # TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
   _a					:  Optional[Any]						=  dataset.apply(tf.data.experimental.assert_cardinality(A      )      )
   _a					:  int						=  dataset.map(A		,	num_parallel_calls=A      )
   if shuffle:
      assert shuffle_buffer_size is not None
      _a					:  Tuple						=  dataset.shuffle(args.shuffle_buffer_size      )
   _a					:  List[str]						=  dataset.batch(A		,	drop_remainder=A      )
   _a					:  Any						=  dataset.map(A		,	num_parallel_calls=A      )
   _a					:  Union[str, Any]						=  dataset.prefetch(A      )
   return dataset


def    UpperCAmelCase_  (       A      ):
   '''simple docstring'''





   if not args.no_tpu:
      _a					:  Optional[Any]						=  initialize_tpu(A      )
      _a					:  Optional[int]						=  tf.distribute.TPUStrategy(A      )
   else:
      _a					:  List[Any]						=  tf.distribute.OneDeviceStrategy(device='/gpu:0'      )

   if args.bfloataa:
      tf.keras.mixed_precision.set_global_policy('mixed_bfloat16'      )

   _a					:  str						=  AutoTokenizer.from_pretrained(args.tokenizer      )
   _a					:  Tuple						=  AutoConfig.from_pretrained(args.pretrained_model_config      )
   _a					:  Optional[Any]						=  tokenizer.vocab_size

   _a					:  Optional[int]						=  tf.io.gfile.glob(os.path.join(args.train_dataset		,	'*.tfrecord'      )      )
   if not training_records:
      raise ValueError(f'''No .tfrecord files found in {args.train_dataset}.'''      )
   _a					:  List[str]						=  tf.io.gfile.glob(os.path.join(args.eval_dataset		,	'*.tfrecord'      )      )
   if not eval_records:
      raise ValueError(f'''No .tfrecord files found in {args.eval_dataset}.'''      )

   _a					:  Union[str, Any]						=  count_samples(A      )

   _a					:  List[str]						=  num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
   _a					:  Optional[int]						=  steps_per_epoch * args.num_epochs

   with strategy.scope():
      _a					:  Tuple						=  TFAutoModelForMaskedLM.from_config(A      )
      model(model.dummy_inputs      )  # Pass some dummy inputs through the model to ensure all the weights are built
      _a		,						_a					:  List[Any]						=  create_optimizer(
          num_train_steps=A		,	num_warmup_steps=total_train_steps // 2_0		,	init_lr=args.learning_rate		,	weight_decay_rate=args.weight_decay_rate		,	)

      # Transformers models compute the right loss for their task by default when labels are passed, and will
      # use this for training unless you specify your own loss function in compile().
      model.compile(optimizer=A		,	metrics=['accuracy']      )

   def decode_fn(A      ):
      _a					:  Dict						=  {
          'input_ids': tf.io.FixedLenFeature(dtype=tf.intaa		,	shape=(args.max_length,)      ),
          'attention_mask': tf.io.FixedLenFeature(dtype=tf.intaa		,	shape=(args.max_length,)      ),
      }
      return tf.io.parse_single_example(A		,	A      )

   # Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
   # use their methods in our data pipeline.
   _a					:  Optional[int]						=  DataCollatorForLanguageModeling(
       tokenizer=A		,	mlm_probability=args.mlm_probability		,	mlm=A		,	return_tensors='tf'      )

   def mask_with_collator(A      ):
      # TF really needs an isin() function
      _a					:  Tuple						=  (
          ~tf.cast(batch['attention_mask']		,	tf.bool      )
          | (batch['input_ids'] == tokenizer.cls_token_id)
          | (batch['input_ids'] == tokenizer.sep_token_id)
      )
      _a		,						_a					:  List[Any]						=  data_collator.tf_mask_tokens(
          batch['input_ids']		,	vocab_size=len(A      )		,	mask_token_id=tokenizer.mask_token_id		,	special_tokens_mask=A		,	)
      return batch

   _a					:  Tuple						=  args.per_replica_batch_size * strategy.num_replicas_in_sync

   _a					:  Union[str, Any]						=  prepare_dataset(
       A		,	decode_fn=A		,	mask_fn=A		,	batch_size=A		,	shuffle=A		,	shuffle_buffer_size=args.shuffle_buffer_size		,	)

   _a					:  Any						=  prepare_dataset(
       A		,	decode_fn=A		,	mask_fn=A		,	batch_size=A		,	shuffle=A		,	)

   _a					:  Dict						=  []
   if args.hub_model_id:
      callbacks.append(
          PushToHubCallback(output_dir=args.output_dir		,	hub_model_id=args.hub_model_id		,	tokenizer=A      )      )

   model.fit(
       A		,	validation_data=A		,	epochs=args.num_epochs		,	callbacks=A		,	)

   model.save_pretrained(args.output_dir      )


if __name__ == "__main__":
 UpperCAmelCase_      :		Any          =		parse_args()
 main(args)






'''simple docstring'''




import inspect
import os
import unittest
from dataclasses import dataclass

import torch

from accelerate import Accelerator, DistributedDataParallelKwargs, GradScalerKwargs
from accelerate.state import AcceleratorState
from accelerate.test_utils import execute_subprocess_async, require_cuda, require_multi_gpu
from accelerate.utils import KwargsHandler







@dataclass
class 					a			(				snake_case__  ):
 '''simple docstring'''




 __lowerCAmelCase						: int							  =		0
 __lowerCAmelCase						: bool							  =		False
 __lowerCAmelCase						: float							  =		3.0







class 					a			(				unittest.TestCase  ):
 '''simple docstring'''




 def   __UpperCamelCase   (    self      )      ->    Dict:
    # If no defaults are changed, `to_kwargs` returns an empty dict.
    self.assertDictEqual(MockClass().to_kwargs()		,      {}      )
    self.assertDictEqual(MockClass(a=2      ).to_kwargs()		,      {'a': 2}      )
    self.assertDictEqual(MockClass(a=2		,      b=lowerCamelCase_      ).to_kwargs()		,      {'a': 2, 'b': True}      )
    self.assertDictEqual(MockClass(a=2		,      c=2.25      ).to_kwargs()		,      {'a': 2, 'c': 2.25}      )




 @require_cuda
 def   __UpperCamelCase   (    self      )      ->    int:
    # If no defaults are changed, `to_kwargs` returns an empty dict.
    _a					:  List[str]						=  GradScalerKwargs(init_scale=1_0_2_4		,      growth_factor=2      )
    AcceleratorState._reset_state()
    _a					:  Optional[int]						=  Accelerator(mixed_precision='fp16'		,      kwargs_handlers=[scaler_handler]      )
    print(accelerator.use_fpaa      )
    _a					:  Tuple						=  accelerator.scaler

    # Check the kwargs have been applied
    self.assertEqual(scaler._init_scale		,      1024.0      )
    self.assertEqual(scaler._growth_factor		,      2.0      )

    # Check the other values are at the default
    self.assertEqual(scaler._backoff_factor		,      0.5      )
    self.assertEqual(scaler._growth_interval		,      2_0_0_0      )
    self.assertEqual(scaler._enabled		,      lowerCamelCase_      )




 @require_multi_gpu
 def   __UpperCamelCase   (    self      )      ->    Optional[Any]:
    _a					:  List[str]						=  ['torchrun', F'''--nproc_per_node={torch.cuda.device_count()}''', inspect.getfile(self.__class__      )]
    execute_subprocess_async(lowerCamelCase_		,      env=os.environ.copy()      )


if __name__ == "__main__":
 UpperCAmelCase_      :		Tuple          =		DistributedDataParallelKwargs(bucket_cap_mb=15, find_unused_parameters=True)
 UpperCAmelCase_      :		str          =		Accelerator(kwargs_handlers=[ddp_scaler])
 UpperCAmelCase_      :		List[Any]          =		torch.nn.Linear(100, 200)
 UpperCAmelCase_      :		Tuple          =		accelerator.prepare(model)

 # Check the values changed in kwargs
 UpperCAmelCase_      :		List[str]          =		""
 UpperCAmelCase_      :		Any          =		model.bucket_bytes_cap // (1024 * 1024)
 if observed_bucket_cap_map != 15:
  error_msg += f"Kwargs badly passed, should have `15` but found {observed_bucket_cap_map}.\n"
 if model.find_unused_parameters is not True:
  error_msg += f"Kwargs badly passed, should have `True` but found {model.find_unused_parameters}.\n"

 # Check the values of the defaults
 if model.dim != 0:
  error_msg += f"Default value not respected, should have `0` but found {model.dim}.\n"
 if model.broadcast_buffers is not True:
  error_msg += f"Default value not respected, should have `True` but found {model.broadcast_buffers}.\n"
 if model.gradient_as_bucket_view is not False:
  error_msg += f"Default value not respected, should have `False` but found {model.gradient_as_bucket_view}.\n"

 # Raise error at the end to make sure we don't stop at the first failure.
 if len(error_msg) > 0:
  raise ValueError(error_msg)


