
"""simple docstring"""


import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


SCREAMING_SNAKE_CASE      =     logging.getLogger(__name__)

SCREAMING_SNAKE_CASE      =     tf.data.AUTOTUNE



def 	lowerCamelCase__			(							)->							str:







       """simple docstring"""
       UpperCamelCase					 =		argparse.ArgumentParser(description="Train a masked language model on TPU."				)
       parser.add_argument(
           "--pretrained_model_config"		,      type=UpperCAmelCase_		,      default="roberta-base"		,      help="The model config to use. Note that we don't copy the model's weights, only the config!"		,      )
       parser.add_argument(
           "--tokenizer"		,      type=UpperCAmelCase_		,      default="unigram-tokenizer-wikitext"		,      help="The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model's vocab size."		,      )

       parser.add_argument(
           "--per_replica_batch_size"		,      type=UpperCAmelCase_		,      default=8		,      help="Batch size per TPU core."		,      )

       parser.add_argument(
           "--no_tpu"		,      action="store_true"		,      help="If set, run on CPU and don't try to initialize a TPU. Useful for debugging on non-TPU instances."		,      )

       parser.add_argument(
           "--tpu_name"		,      type=UpperCAmelCase_		,      help="Name of TPU resource to initialize. Should be blank on Colab, and 'local' on TPU VMs."		,      default="local"		,      )

       parser.add_argument(
           "--tpu_zone"		,      type=UpperCAmelCase_		,      help="Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes."		,      )

       parser.add_argument(
           "--gcp_project"		,      type=UpperCAmelCase_		,      help="Google cloud project name. Only used for non-Colab TPU nodes."				)

       parser.add_argument(
           "--bfloat16"		,      action="store_true"		,      help="Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU."		,      )

       parser.add_argument(
           "--train_dataset"		,      type=UpperCAmelCase_		,      help="Path to training dataset to load. If the path begins with `gs://`"
           " then the dataset will be loaded from a Google Cloud Storage bucket."		,      )

       parser.add_argument(
           "--shuffle_buffer_size"		,      type=UpperCAmelCase_		,      default=2**18		,      help="Size of the shuffle buffer (in samples)"		,      )

       parser.add_argument(
           "--eval_dataset"		,      type=UpperCAmelCase_		,      help="Path to evaluation dataset to load. If the path begins with `gs://`"
           " then the dataset will be loaded from a Google Cloud Storage bucket."		,      )

       parser.add_argument(
           "--num_epochs"		,      type=UpperCAmelCase_		,      default=1		,      help="Number of epochs to train for."		,      )

       parser.add_argument(
           "--learning_rate"		,      type=UpperCAmelCase_		,      default=1E-4		,      help="Learning rate to use for training."		,      )

       parser.add_argument(
           "--weight_decay_rate"		,      type=UpperCAmelCase_		,      default=1E-3		,      help="Weight decay rate to use for training."		,      )

       parser.add_argument(
           "--max_length"		,      type=UpperCAmelCase_		,      default=5_12		,      help="Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py"		,      )

       parser.add_argument(
           "--mlm_probability"		,      type=UpperCAmelCase_		,      default=0.15		,      help="Fraction of tokens to mask during training."		,      )

       parser.add_argument("--output_dir"		,      type=UpperCAmelCase_		,      required=UpperCAmelCase_		,      help="Path to save model checkpoints to."				)
       parser.add_argument("--hub_model_id"		,      type=UpperCAmelCase_		,      help="Model ID to upload to on the Hugging Face Hub."				)

       UpperCamelCase					 =		parser.parse_args()
       return args



def 	lowerCamelCase__			(							UpperCAmelCase_				)->							Optional[int]:







       """simple docstring"""
       try:
              if args.tpu_name:
                     UpperCamelCase					 =		tf.distribute.cluster_resolver.TPUClusterResolver(
                         args.tpu_name		,      zone=args.tpu_zone		,      project=args.gcp_project				)
              else:
                     UpperCamelCase					 =		tf.distribute.cluster_resolver.TPUClusterResolver()
       except ValueError:
              raise RuntimeError(
                  "Couldn't connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or "
                  "--gcp_project. When running on a TPU VM, use --tpu_name local."				)

       tf.config.experimental_connect_to_cluster(UpperCAmelCase_				)
       tf.tpu.experimental.initialize_tpu_system(UpperCAmelCase_				)

       return tpu



def 	lowerCamelCase__			(							UpperCAmelCase_				)->							Tuple:







       """simple docstring"""
       UpperCamelCase					 =		0
       for file in file_list:
              UpperCamelCase					 =		file.split("/"				)[-1]
              UpperCamelCase					 =		re.search(R"-\d+-(\d+)\.tfrecord"		,      UpperCAmelCase_				).group(1				)
              UpperCamelCase					 =		int(UpperCAmelCase_				)
              num_samples += sample_count

       return num_samples



def 	lowerCamelCase__			(							UpperCAmelCase_		,      UpperCAmelCase_		,      UpperCAmelCase_		,      UpperCAmelCase_		,      UpperCAmelCase_		,      UpperCAmelCase_=None				)->							int:







       """simple docstring"""
       UpperCamelCase					 =		count_samples(UpperCAmelCase_				)
       UpperCamelCase					 =		tf.data.Dataset.from_tensor_slices(UpperCAmelCase_				)
       if shuffle:
              UpperCamelCase					 =		dataset.shuffle(len(UpperCAmelCase_				)				)
       UpperCamelCase					 =		tf.data.TFRecordDataset(UpperCAmelCase_		,      num_parallel_reads=UpperCAmelCase_				)
       # TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
       UpperCamelCase					 =		dataset.apply(tf.data.experimental.assert_cardinality(UpperCAmelCase_				)				)
       UpperCamelCase					 =		dataset.map(UpperCAmelCase_		,      num_parallel_calls=UpperCAmelCase_				)
       if shuffle:
              assert shuffle_buffer_size is not None
              UpperCamelCase					 =		dataset.shuffle(args.shuffle_buffer_size				)
       UpperCamelCase					 =		dataset.batch(UpperCAmelCase_		,      drop_remainder=UpperCAmelCase_				)
       UpperCamelCase					 =		dataset.map(UpperCAmelCase_		,      num_parallel_calls=UpperCAmelCase_				)
       UpperCamelCase					 =		dataset.prefetch(UpperCAmelCase_				)
       return dataset



def 	lowerCamelCase__			(							UpperCAmelCase_				)->							Optional[Any]:







       """simple docstring"""
       if not args.no_tpu:
              UpperCamelCase					 =		initialize_tpu(UpperCAmelCase_				)
              UpperCamelCase					 =		tf.distribute.TPUStrategy(UpperCAmelCase_				)
       else:
              UpperCamelCase					 =		tf.distribute.OneDeviceStrategy(device="/gpu:0"				)

       if args.bfloataa:
              tf.keras.mixed_precision.set_global_policy("mixed_bfloat16"				)

       UpperCamelCase					 =		AutoTokenizer.from_pretrained(args.tokenizer				)
       UpperCamelCase					 =		AutoConfig.from_pretrained(args.pretrained_model_config				)
       UpperCamelCase					 =		tokenizer.vocab_size

       UpperCamelCase					 =		tf.io.gfile.glob(os.path.join(args.train_dataset		,      "*.tfrecord"				)				)
       if not training_records:
              raise ValueError(F"No .tfrecord files found in {args.train_dataset}."				)
       UpperCamelCase					 =		tf.io.gfile.glob(os.path.join(args.eval_dataset		,      "*.tfrecord"				)				)
       if not eval_records:
              raise ValueError(F"No .tfrecord files found in {args.eval_dataset}."				)

       UpperCamelCase					 =		count_samples(UpperCAmelCase_				)

       UpperCamelCase					 =		num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
       UpperCamelCase					 =		steps_per_epoch * args.num_epochs

       with strategy.scope():
              UpperCamelCase					 =		TFAutoModelForMaskedLM.from_config(UpperCAmelCase_				)
              model(model.dummy_inputs				)  # Pass some dummy inputs through the model to ensure all the weights are built
              UpperCamelCase ,       UpperCamelCase					 =		create_optimizer(
                  num_train_steps=UpperCAmelCase_		,      num_warmup_steps=total_train_steps // 20		,      init_lr=args.learning_rate		,      weight_decay_rate=args.weight_decay_rate		,      )

              # Transformers models compute the right loss for their task by default when labels are passed, and will
              # use this for training unless you specify your own loss function in compile().
              model.compile(optimizer=UpperCAmelCase_		,      metrics=["accuracy"]				)

       def decode_fn(UpperCAmelCase_				):
              UpperCamelCase					 =		{
                  "input_ids": tf.io.FixedLenFeature(dtype=tf.intaa		,      shape=(args.max_length,)				),
                  "attention_mask": tf.io.FixedLenFeature(dtype=tf.intaa		,      shape=(args.max_length,)				),
              }
              return tf.io.parse_single_example(UpperCAmelCase_		,      UpperCAmelCase_				)

       # Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
       # use their methods in our data pipeline.
       UpperCamelCase					 =		DataCollatorForLanguageModeling(
           tokenizer=UpperCAmelCase_		,      mlm_probability=args.mlm_probability		,      mlm=UpperCAmelCase_		,      return_tensors="tf"				)

       def mask_with_collator(UpperCAmelCase_				):
              # TF really needs an isin() function
              UpperCamelCase					 =		(
                  ~tf.cast(batch["attention_mask"]		,      tf.bool				)
                  | (batch["input_ids"] == tokenizer.cls_token_id)
                  | (batch["input_ids"] == tokenizer.sep_token_id)
              )
              UpperCamelCase ,       UpperCamelCase					 =		data_collator.tf_mask_tokens(
                  batch["input_ids"]		,      vocab_size=len(UpperCAmelCase_				)		,      mask_token_id=tokenizer.mask_token_id		,      special_tokens_mask=UpperCAmelCase_		,      )
              return batch

       UpperCamelCase					 =		args.per_replica_batch_size * strategy.num_replicas_in_sync

       UpperCamelCase					 =		prepare_dataset(
           UpperCAmelCase_		,      decode_fn=UpperCAmelCase_		,      mask_fn=UpperCAmelCase_		,      batch_size=UpperCAmelCase_		,      shuffle=UpperCAmelCase_		,      shuffle_buffer_size=args.shuffle_buffer_size		,      )

       UpperCamelCase					 =		prepare_dataset(
           UpperCAmelCase_		,      decode_fn=UpperCAmelCase_		,      mask_fn=UpperCAmelCase_		,      batch_size=UpperCAmelCase_		,      shuffle=UpperCAmelCase_		,      )

       UpperCamelCase					 =		[]
       if args.hub_model_id:
              callbacks.append(
                  PushToHubCallback(output_dir=args.output_dir		,      hub_model_id=args.hub_model_id		,      tokenizer=UpperCAmelCase_				)				)

       model.fit(
           UpperCAmelCase_		,      validation_data=UpperCAmelCase_		,      epochs=args.num_epochs		,      callbacks=UpperCAmelCase_		,      )

       model.save_pretrained(args.output_dir				)


if __name__ == "__main__":
      SCREAMING_SNAKE_CASE      =     parse_args()
      main(args)







"""simple docstring"""


from .integrations import (
    is_optuna_available,
    is_ray_available,
    is_sigopt_available,
    is_wandb_available,
    run_hp_search_optuna,
    run_hp_search_ray,
    run_hp_search_sigopt,
    run_hp_search_wandb,
)
from .trainer_utils import (
    HPSearchBackend,
    default_hp_space_optuna,
    default_hp_space_ray,
    default_hp_space_sigopt,
    default_hp_space_wandb,
)
from .utils import logging


SCREAMING_SNAKE_CASE      =     logging.get_logger(__name__)



class   __a  :
       UpperCamelCase_		:					str
       UpperCamelCase_		:					str   =			None






       @staticmethod
       def 	_SCREAMING_SNAKE_CASE       (  )->  Union[str, Any]:







              """simple docstring"""
              raise NotImplementedError






       def 	_SCREAMING_SNAKE_CASE       (  self :							List[str]       ,      UpperCAmelCase_ :							Optional[int]       ,      UpperCAmelCase_ :							int       ,      UpperCAmelCase_ :							str       ,      **UpperCAmelCase_ :							Dict			)->  Union[str, Any]:







              """simple docstring"""
              raise NotImplementedError






       def 	_SCREAMING_SNAKE_CASE       (  self :							Optional[int]       ,      UpperCAmelCase_ :							str			)->  Any:







              """simple docstring"""
              raise NotImplementedError






       def 	_SCREAMING_SNAKE_CASE       (  self :							Union[str, Any]			)->  List[str]:







              """simple docstring"""
              if not self.is_available():
                     raise RuntimeError(
                         f"You picked the {self.name} backend, but it is not installed. Run {self.pip_install()}."			)






       @classmethod
       def 	_SCREAMING_SNAKE_CASE       (  cls :							Union[str, Any]			)->  int:







              """simple docstring"""
              return f"`pip install {cls.pip_package or cls.name}`"





class   __a  (      _lowerCAmelCase	):
       UpperCamelCase_		:					List[str]   =			'''optuna'''






       @staticmethod
       def 	_SCREAMING_SNAKE_CASE       (  )->  Union[str, Any]:







              """simple docstring"""
              return is_optuna_available()






       def 	_SCREAMING_SNAKE_CASE       (  self :							Dict       ,      UpperCAmelCase_ :							List[str]       ,      UpperCAmelCase_ :							int       ,      UpperCAmelCase_ :							str       ,      **UpperCAmelCase_ :							Optional[Any]			)->  Union[str, Any]:







              """simple docstring"""
              return run_hp_search_optuna(UpperCAmelCase_       ,      UpperCAmelCase_       ,      UpperCAmelCase_       ,      **UpperCAmelCase_			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							Dict       ,      UpperCAmelCase_ :							Union[str, Any]			)->  Union[str, Any]:







              """simple docstring"""
              return default_hp_space_optuna(UpperCAmelCase_			)





class   __a  (      _lowerCAmelCase	):
       UpperCamelCase_		:					Optional[int]   =			'''ray'''
       UpperCamelCase_		:					Tuple   =			'''\'ray[tune]\''''






       @staticmethod
       def 	_SCREAMING_SNAKE_CASE       (  )->  Any:







              """simple docstring"""
              return is_ray_available()






       def 	_SCREAMING_SNAKE_CASE       (  self :							List[Any]       ,      UpperCAmelCase_ :							Any       ,      UpperCAmelCase_ :							int       ,      UpperCAmelCase_ :							str       ,      **UpperCAmelCase_ :							int			)->  int:







              """simple docstring"""
              return run_hp_search_ray(UpperCAmelCase_       ,      UpperCAmelCase_       ,      UpperCAmelCase_       ,      **UpperCAmelCase_			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							str       ,      UpperCAmelCase_ :							str			)->  Union[str, Any]:







              """simple docstring"""
              return default_hp_space_ray(UpperCAmelCase_			)





class   __a  (      _lowerCAmelCase	):
       UpperCamelCase_		:					Union[str, Any]   =			'''sigopt'''






       @staticmethod
       def 	_SCREAMING_SNAKE_CASE       (  )->  List[str]:







              """simple docstring"""
              return is_sigopt_available()






       def 	_SCREAMING_SNAKE_CASE       (  self :							Tuple       ,      UpperCAmelCase_ :							List[str]       ,      UpperCAmelCase_ :							int       ,      UpperCAmelCase_ :							str       ,      **UpperCAmelCase_ :							Tuple			)->  Tuple:







              """simple docstring"""
              return run_hp_search_sigopt(UpperCAmelCase_       ,      UpperCAmelCase_       ,      UpperCAmelCase_       ,      **UpperCAmelCase_			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							List[Any]       ,      UpperCAmelCase_ :							Tuple			)->  Optional[Any]:







              """simple docstring"""
              return default_hp_space_sigopt(UpperCAmelCase_			)





class   __a  (      _lowerCAmelCase	):
       UpperCamelCase_		:					Any   =			'''wandb'''






       @staticmethod
       def 	_SCREAMING_SNAKE_CASE       (  )->  List[Any]:







              """simple docstring"""
              return is_wandb_available()






       def 	_SCREAMING_SNAKE_CASE       (  self :							int       ,      UpperCAmelCase_ :							Optional[int]       ,      UpperCAmelCase_ :							int       ,      UpperCAmelCase_ :							str       ,      **UpperCAmelCase_ :							List[str]			)->  str:







              """simple docstring"""
              return run_hp_search_wandb(UpperCAmelCase_       ,      UpperCAmelCase_       ,      UpperCAmelCase_       ,      **UpperCAmelCase_			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							Tuple       ,      UpperCAmelCase_ :							List[Any]			)->  List[Any]:







              """simple docstring"""
              return default_hp_space_wandb(UpperCAmelCase_			)





SCREAMING_SNAKE_CASE      =     {
    HPSearchBackend(backend.name): backend for backend in [OptunaBackend, RayTuneBackend, SigOptBackend, WandbBackend]
}



def 	lowerCamelCase__			(							)->							str:







       """simple docstring"""
       UpperCamelCase					 =		[backend for backend in ALL_HYPERPARAMETER_SEARCH_BACKENDS.values() if backend.is_available()]
       if len(UpperCAmelCase_				) > 0:
              UpperCamelCase					 =		available_backends[0].name
              if len(UpperCAmelCase_				) > 1:
                     logger.info(
                         F"{len(UpperCAmelCase_				)} hyperparameter search backends available. Using {name} as the default."				)
              return name
       raise RuntimeError(
           "No hyperparameter search backend available.\n"
           + "\n".join(
               F" - To install {backend.name} run {backend.pip_install()}"
               for backend in ALL_HYPERPARAMETER_SEARCH_BACKENDS.values()				)				)





