



import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


_snake_case						=    logging.getLogger(__name__)

_snake_case						=    tf.data.AUTOTUNE






def 							A							(   ):





							'''simple docstring'''




							_lowerCAmelCase	:				int					  =       argparse.ArgumentParser(description="Train a masked language model on TPU."    )
							parser.add_argument(
							    "--pretrained_model_config"							,					type=_lowerCamelCase							,					default="roberta-base"							,					help="The model config to use. Note that we don't copy the model's weights, only the config!"							,					)
							parser.add_argument(
							    "--tokenizer"							,					type=_lowerCamelCase							,					default="unigram-tokenizer-wikitext"							,					help="The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model's vocab size."							,					)

							parser.add_argument(
							    "--per_replica_batch_size"							,					type=_lowerCamelCase							,					default=8							,					help="Batch size per TPU core."							,					)

							parser.add_argument(
							    "--no_tpu"							,					action="store_true"							,					help="If set, run on CPU and don't try to initialize a TPU. Useful for debugging on non-TPU instances."							,					)

							parser.add_argument(
							    "--tpu_name"							,					type=_lowerCamelCase							,					help="Name of TPU resource to initialize. Should be blank on Colab, and 'local' on TPU VMs."							,					default="local"							,					)

							parser.add_argument(
							    "--tpu_zone"							,					type=_lowerCamelCase							,					help="Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes."							,					)

							parser.add_argument(
							    "--gcp_project"							,					type=_lowerCamelCase							,					help="Google cloud project name. Only used for non-Colab TPU nodes."    )

							parser.add_argument(
							    "--bfloat16"							,					action="store_true"							,					help="Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU."							,					)

							parser.add_argument(
							    "--train_dataset"							,					type=_lowerCamelCase							,					help="Path to training dataset to load. If the path begins with `gs://`"
							    " then the dataset will be loaded from a Google Cloud Storage bucket."							,					)

							parser.add_argument(
							    "--shuffle_buffer_size"							,					type=_lowerCamelCase							,					default=2**18							,					help="Size of the shuffle buffer (in samples)"							,					)

							parser.add_argument(
							    "--eval_dataset"							,					type=_lowerCamelCase							,					help="Path to evaluation dataset to load. If the path begins with `gs://`"
							    " then the dataset will be loaded from a Google Cloud Storage bucket."							,					)

							parser.add_argument(
							    "--num_epochs"							,					type=_lowerCamelCase							,					default=1							,					help="Number of epochs to train for."							,					)

							parser.add_argument(
							    "--learning_rate"							,					type=_lowerCamelCase							,					default=1e-4							,					help="Learning rate to use for training."							,					)

							parser.add_argument(
							    "--weight_decay_rate"							,					type=_lowerCamelCase							,					default=1e-3							,					help="Weight decay rate to use for training."							,					)

							parser.add_argument(
							    "--max_length"							,					type=_lowerCamelCase							,					default=512							,					help="Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py"							,					)

							parser.add_argument(
							    "--mlm_probability"							,					type=_lowerCamelCase							,					default=0.15							,					help="Fraction of tokens to mask during training."							,					)

							parser.add_argument("--output_dir"							,					type=_lowerCamelCase							,					required=_lowerCamelCase							,					help="Path to save model checkpoints to."    )
							parser.add_argument("--hub_model_id"							,					type=_lowerCamelCase							,					help="Model ID to upload to on the Hugging Face Hub."    )

							_lowerCAmelCase	:				Optional[Any]					  =       parser.parse_args()
							return args






def 							A							(   _lowerCamelCase    ):





							'''simple docstring'''




							try:
														if args.tpu_name:
																					_lowerCAmelCase	:				str					  =       tf.distribute.cluster_resolver.TPUClusterResolver(
																					    args.tpu_name							,					zone=args.tpu_zone							,					project=args.gcp_project    )
														else:
																					_lowerCAmelCase	:				List[str]					  =       tf.distribute.cluster_resolver.TPUClusterResolver()
							except ValueError:
														raise RuntimeError(
														    "Couldn't connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or "
														    "--gcp_project. When running on a TPU VM, use --tpu_name local."    )

							tf.config.experimental_connect_to_cluster(_lowerCamelCase    )
							tf.tpu.experimental.initialize_tpu_system(_lowerCamelCase    )

							return tpu






def 							A							(   _lowerCamelCase    ):





							'''simple docstring'''




							_lowerCAmelCase	:				str					  =       0
							for file in file_list:
														_lowerCAmelCase	:				Optional[Any]					  =       file.split("/"    )[-1]
														_lowerCAmelCase	:				Optional[Any]					  =       re.search(r"-\d+-(\d+)\.tfrecord"							,					_lowerCamelCase    ).group(1    )
														_lowerCAmelCase	:				int					  =       int(_lowerCamelCase    )
														num_samples += sample_count

							return num_samples






def 							A							(   _lowerCamelCase							,					_lowerCamelCase							,					_lowerCamelCase							,					_lowerCamelCase							,					_lowerCamelCase							,					_lowerCamelCase=None    ):





							'''simple docstring'''




							_lowerCAmelCase	:				Optional[int]					  =       count_samples(_lowerCamelCase    )
							_lowerCAmelCase	:				Tuple					  =       tf.data.Dataset.from_tensor_slices(_lowerCamelCase    )
							if shuffle:
														_lowerCAmelCase	:				Union[str, Any]					  =       dataset.shuffle(len(_lowerCamelCase    )    )
							_lowerCAmelCase	:				int					  =       tf.data.TFRecordDataset(_lowerCamelCase							,					num_parallel_reads=_lowerCamelCase    )
							# TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
							_lowerCAmelCase	:				List[Any]					  =       dataset.apply(tf.data.experimental.assert_cardinality(_lowerCamelCase    )    )
							_lowerCAmelCase	:				List[str]					  =       dataset.map(_lowerCamelCase							,					num_parallel_calls=_lowerCamelCase    )
							if shuffle:
														assert shuffle_buffer_size is not None
														_lowerCAmelCase	:				str					  =       dataset.shuffle(args.shuffle_buffer_size    )
							_lowerCAmelCase	:				Dict					  =       dataset.batch(_lowerCamelCase							,					drop_remainder=_lowerCamelCase    )
							_lowerCAmelCase	:				Any					  =       dataset.map(_lowerCamelCase							,					num_parallel_calls=_lowerCamelCase    )
							_lowerCAmelCase	:				Dict					  =       dataset.prefetch(_lowerCamelCase    )
							return dataset






def 							A							(   _lowerCamelCase    ):





							'''simple docstring'''




							if not args.no_tpu:
														_lowerCAmelCase	:				Any					  =       initialize_tpu(_lowerCamelCase    )
														_lowerCAmelCase	:				Any					  =       tf.distribute.TPUStrategy(_lowerCamelCase    )
							else:
														_lowerCAmelCase	:				Any					  =       tf.distribute.OneDeviceStrategy(device="/gpu:0"    )

							if args.bfloataa:
														tf.keras.mixed_precision.set_global_policy("mixed_bfloat16"    )

							_lowerCAmelCase	:				Optional[int]					  =       AutoTokenizer.from_pretrained(args.tokenizer    )
							_lowerCAmelCase	:				List[str]					  =       AutoConfig.from_pretrained(args.pretrained_model_config    )
							_lowerCAmelCase	:				Optional[Any]					  =       tokenizer.vocab_size

							_lowerCAmelCase	:				Optional[int]					  =       tf.io.gfile.glob(os.path.join(args.train_dataset							,					"*.tfrecord"    )    )
							if not training_records:
														raise ValueError(F"No .tfrecord files found in {args.train_dataset}."    )
							_lowerCAmelCase	:				Tuple					  =       tf.io.gfile.glob(os.path.join(args.eval_dataset							,					"*.tfrecord"    )    )
							if not eval_records:
														raise ValueError(F"No .tfrecord files found in {args.eval_dataset}."    )

							_lowerCAmelCase	:				int					  =       count_samples(_lowerCamelCase    )

							_lowerCAmelCase	:				Any					  =       num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
							_lowerCAmelCase	:				List[str]					  =       steps_per_epoch * args.num_epochs

							with strategy.scope():
														_lowerCAmelCase	:				Union[str, Any]					  =       TFAutoModelForMaskedLM.from_config(_lowerCamelCase    )
														model(model.dummy_inputs    )  # Pass some dummy inputs through the model to ensure all the weights are built
														_lowerCAmelCase	:				Any					  =       create_optimizer(
														    num_train_steps=_lowerCamelCase							,					num_warmup_steps=total_train_steps // 20							,					init_lr=args.learning_rate							,					weight_decay_rate=args.weight_decay_rate							,					)

														# Transformers models compute the right loss for their task by default when labels are passed, and will
														# use this for training unless you specify your own loss function in compile().
														model.compile(optimizer=_lowerCamelCase							,					metrics=["accuracy"]    )

							def decode_fn(_lowerCamelCase    ):
														_lowerCAmelCase	:				List[Any]					  =       {
														    "input_ids": tf.io.FixedLenFeature(dtype=tf.intaa							,					shape=(args.max_length,)    ),
														    "attention_mask": tf.io.FixedLenFeature(dtype=tf.intaa							,					shape=(args.max_length,)    ),
														}
														return tf.io.parse_single_example(_lowerCamelCase							,					_lowerCamelCase    )

							# Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
							# use their methods in our data pipeline.
							_lowerCAmelCase	:				Optional[Any]					  =       DataCollatorForLanguageModeling(
							    tokenizer=_lowerCamelCase							,					mlm_probability=args.mlm_probability							,					mlm=_lowerCamelCase							,					return_tensors="tf"    )

							def mask_with_collator(_lowerCamelCase    ):
														# TF really needs an isin() function
														_lowerCAmelCase	:				str					  =       (
														    ~tf.cast(batch["attention_mask"]							,					tf.bool    )
														    | (batch["input_ids"] == tokenizer.cls_token_id)
														    | (batch["input_ids"] == tokenizer.sep_token_id)
														)
														_lowerCAmelCase	:				List[str]					  =       data_collator.tf_mask_tokens(
														    batch["input_ids"]							,					vocab_size=len(_lowerCamelCase    )							,					mask_token_id=tokenizer.mask_token_id							,					special_tokens_mask=_lowerCamelCase							,					)
														return batch

							_lowerCAmelCase	:				Dict					  =       args.per_replica_batch_size * strategy.num_replicas_in_sync

							_lowerCAmelCase	:				Dict					  =       prepare_dataset(
							    _lowerCamelCase							,					decode_fn=_lowerCamelCase							,					mask_fn=_lowerCamelCase							,					batch_size=_lowerCamelCase							,					shuffle=_lowerCamelCase							,					shuffle_buffer_size=args.shuffle_buffer_size							,					)

							_lowerCAmelCase	:				List[Any]					  =       prepare_dataset(
							    _lowerCamelCase							,					decode_fn=_lowerCamelCase							,					mask_fn=_lowerCamelCase							,					batch_size=_lowerCamelCase							,					shuffle=_lowerCamelCase							,					)

							_lowerCAmelCase	:				List[Any]					  =       []
							if args.hub_model_id:
														callbacks.append(
														    PushToHubCallback(output_dir=args.output_dir							,					hub_model_id=args.hub_model_id							,					tokenizer=_lowerCamelCase    )    )

							model.fit(
							    _lowerCamelCase							,					validation_data=_lowerCamelCase							,					epochs=args.num_epochs							,					callbacks=_lowerCamelCase							,					)

							model.save_pretrained(args.output_dir    )


if __name__ == "__main__":
						_snake_case						=    parse_args()
						main(args)


from queue import Queue
from typing import TYPE_CHECKING, Optional


if TYPE_CHECKING:
		from ..models.auto import AutoTokenizer




class 					lowerCamelCase__				:




	'''simple docstring'''




	def   _lowerCamelCase							(   self						:Optional[Any]	,  a						:int		)						->					Any:
								raise NotImplementedError()







	def   _lowerCamelCase							(   self						:Any		)						->					Optional[int]:
								raise NotImplementedError()




class 					lowerCamelCase__				(							__lowercase):




	'''simple docstring'''




	def __init__(   self						:List[Any]	,  a						:"AutoTokenizer"	,  a						:bool = False	,  **a						:Tuple		)						->					Union[str, Any]:
								__UpperCamelCase      :  int       =   tokenizer
								__UpperCamelCase      :  Any       =   skip_prompt
								__UpperCamelCase      :  Union[str, Any]       =   decode_kwargs

								# variables used in the streaming process
								__UpperCamelCase      :  Union[str, Any]       =   []
								__UpperCamelCase      :  str       =   0
								__UpperCamelCase      :  str       =   True







	def   _lowerCamelCase							(   self						:List[str]	,  a						:Union[str, Any]		)						->					Any:
								if len(value.shape		) > 1 and value.shape[0] > 1:
															raise ValueError("TextStreamer only supports batch size 1"		)
								elif len(value.shape		) > 1:
															__UpperCamelCase      :  Optional[int]       =   value[0]

								if self.skip_prompt and self.next_tokens_are_prompt:
															__UpperCamelCase      :  Optional[Any]       =   False
															return

								# Add the new token to the cache and decodes the entire thing.
								self.token_cache.extend(value.tolist()		)
								__UpperCamelCase      :  Optional[int]       =   self.tokenizer.decode(self.token_cache	,  **self.decode_kwargs		)

								# After the symbol for a new line, we flush the cache.
								if text.endswith("\n"		):
															__UpperCamelCase      :  Optional[int]       =   text[self.print_len :]
															__UpperCamelCase      :  Dict       =   []
															__UpperCamelCase      :  int       =   0
								# If the last token is a CJK character, we print the characters.
								elif len(a		) > 0 and self._is_chinese_char(ord(text[-1]		)		):
															__UpperCamelCase      :  Optional[int]       =   text[self.print_len :]
															self.print_len += len(a		)
								# Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words,
								# which may change with the subsequent token -- there are probably smarter ways to do this!)
								else:
															__UpperCamelCase      :  Any       =   text[self.print_len : text.rfind(" "		) + 1]
															self.print_len += len(a		)

								self.on_finalized_text(a		)







	def   _lowerCamelCase							(   self						:List[str]		)						->					List[str]:
								# Flush the cache, if it exists
								if len(self.token_cache		) > 0:
															__UpperCamelCase      :  str       =   self.tokenizer.decode(self.token_cache	,  **self.decode_kwargs		)
															__UpperCamelCase      :  str       =   text[self.print_len :]
															__UpperCamelCase      :  str       =   []
															__UpperCamelCase      :  int       =   0
								else:
															__UpperCamelCase      :  int       =   ""

								__UpperCamelCase      :  Optional[Any]       =   True
								self.on_finalized_text(a	,  stream_end=a		)







	def   _lowerCamelCase							(   self						:Union[str, Any]	,  a						:str	,  a						:bool = False		)						->					str:
								print(a	,  flush=a	,  end="" if not stream_end else None		)







	def   _lowerCamelCase							(   self						:Optional[int]	,  a						:Optional[Any]		)						->					str:
								# This defines a "chinese character" as anything in the CJK Unicode block:
								#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
								#
								# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
								# despite its name. The modern Korean Hangul alphabet is a different block,
								# as is Japanese Hiragana and Katakana. Those alphabets are used to write
								# space-separated words, so they are not treated specially and handled
								# like the all of the other languages.
								if (
								    (cp >= 0x4e_00 and cp <= 0x9f_ff)
								    or (cp >= 0x34_00 and cp <= 0x4d_bf)  #
								    or (cp >= 0x2_00_00 and cp <= 0x2_a6_df)  #
								    or (cp >= 0x2_a7_00 and cp <= 0x2_b7_3f)  #
								    or (cp >= 0x2_b7_40 and cp <= 0x2_b8_1f)  #
								    or (cp >= 0x2_b8_20 and cp <= 0x2_ce_af)  #
								    or (cp >= 0xf9_00 and cp <= 0xfa_ff)
								    or (cp >= 0x2_f8_00 and cp <= 0x2_fa_1f)  #
								):  #
															return True

								return False




class 					lowerCamelCase__				(							__lowercase):




	'''simple docstring'''




	def __init__(   self						:Any	,  a						:"AutoTokenizer"	,  a						:bool = False	,  a						:Optional[float] = None	,  **a						:Dict		)						->					str:
								super().__init__(a	,  a	,  **a		)
								__UpperCamelCase      :  str       =   Queue()
								__UpperCamelCase      :  Dict       =   None
								__UpperCamelCase      :  Optional[int]       =   timeout







	def   _lowerCamelCase							(   self						:List[Any]	,  a						:str	,  a						:bool = False		)						->					Dict:
								self.text_queue.put(a	,  timeout=self.timeout		)
								if stream_end:
															self.text_queue.put(self.stop_signal	,  timeout=self.timeout		)







	def __iter__(   self						:Tuple		)						->					Tuple:
								return self







	def   _lowerCamelCase							(   self						:List[Any]		)						->					Dict:
								__UpperCamelCase      :  List[Any]       =   self.text_queue.get(timeout=self.timeout		)
								if value == self.stop_signal:
															raise StopIteration()
								else:
															return value