


import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


lowerCAmelCase			=				logging.getLogger(__name__)

lowerCAmelCase			=				tf.data.AUTOTUNE
def      _a					( ):


		"""simple docstring"""




		lowercase__				= argparse.ArgumentParser(description='''Train a masked language model on TPU.'''	)
		parser.add_argument(
		    '''--pretrained_model_config'''		,							type=SCREAMING_SNAKE_CASE		,							default='''roberta-base'''		,							help='''The model config to use. Note that we don\'t copy the model\'s weights, only the config!'''		,							)
		parser.add_argument(
		    '''--tokenizer'''		,							type=SCREAMING_SNAKE_CASE		,							default='''unigram-tokenizer-wikitext'''		,							help='''The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model\'s vocab size.'''		,							)

		parser.add_argument(
		    '''--per_replica_batch_size'''		,							type=SCREAMING_SNAKE_CASE		,							default=8		,							help='''Batch size per TPU core.'''		,							)

		parser.add_argument(
		    '''--no_tpu'''		,							action='''store_true'''		,							help='''If set, run on CPU and don\'t try to initialize a TPU. Useful for debugging on non-TPU instances.'''		,							)

		parser.add_argument(
		    '''--tpu_name'''		,							type=SCREAMING_SNAKE_CASE		,							help='''Name of TPU resource to initialize. Should be blank on Colab, and \'local\' on TPU VMs.'''		,							default='''local'''		,							)

		parser.add_argument(
		    '''--tpu_zone'''		,							type=SCREAMING_SNAKE_CASE		,							help='''Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes.'''		,							)

		parser.add_argument(
		    '''--gcp_project'''		,							type=SCREAMING_SNAKE_CASE		,							help='''Google cloud project name. Only used for non-Colab TPU nodes.'''	)

		parser.add_argument(
		    '''--bfloat16'''		,							action='''store_true'''		,							help='''Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU.'''		,							)

		parser.add_argument(
		    '''--train_dataset'''		,							type=SCREAMING_SNAKE_CASE		,							help='''Path to training dataset to load. If the path begins with `gs://`'''
		    ''' then the dataset will be loaded from a Google Cloud Storage bucket.'''		,							)

		parser.add_argument(
		    '''--shuffle_buffer_size'''		,							type=SCREAMING_SNAKE_CASE		,							default=2**18		,							help='''Size of the shuffle buffer (in samples)'''		,							)

		parser.add_argument(
		    '''--eval_dataset'''		,							type=SCREAMING_SNAKE_CASE		,							help='''Path to evaluation dataset to load. If the path begins with `gs://`'''
		    ''' then the dataset will be loaded from a Google Cloud Storage bucket.'''		,							)

		parser.add_argument(
		    '''--num_epochs'''		,							type=SCREAMING_SNAKE_CASE		,							default=1		,							help='''Number of epochs to train for.'''		,							)

		parser.add_argument(
		    '''--learning_rate'''		,							type=SCREAMING_SNAKE_CASE		,							default=1E-4		,							help='''Learning rate to use for training.'''		,							)

		parser.add_argument(
		    '''--weight_decay_rate'''		,							type=SCREAMING_SNAKE_CASE		,							default=1E-3		,							help='''Weight decay rate to use for training.'''		,							)

		parser.add_argument(
		    '''--max_length'''		,							type=SCREAMING_SNAKE_CASE		,							default=5_12		,							help='''Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py'''		,							)

		parser.add_argument(
		    '''--mlm_probability'''		,							type=SCREAMING_SNAKE_CASE		,							default=0.15		,							help='''Fraction of tokens to mask during training.'''		,							)

		parser.add_argument('''--output_dir'''		,							type=SCREAMING_SNAKE_CASE		,							required=SCREAMING_SNAKE_CASE		,							help='''Path to save model checkpoints to.'''	)
		parser.add_argument('''--hub_model_id'''		,							type=SCREAMING_SNAKE_CASE		,							help='''Model ID to upload to on the Hugging Face Hub.'''	)

		lowercase__				= parser.parse_args()
		return args
def      _a					( SCREAMING_SNAKE_CASE	):


		"""simple docstring"""




		try:
				if args.tpu_name:
						lowercase__				= tf.distribute.cluster_resolver.TPUClusterResolver(
						    args.tpu_name		,							zone=args.tpu_zone		,							project=args.gcp_project	)
				else:
						lowercase__				= tf.distribute.cluster_resolver.TPUClusterResolver()
		except ValueError:
				raise RuntimeError(
				    '''Couldn\'t connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or '''
				    '''--gcp_project. When running on a TPU VM, use --tpu_name local.'''	)

		tf.config.experimental_connect_to_cluster(SCREAMING_SNAKE_CASE	)
		tf.tpu.experimental.initialize_tpu_system(SCREAMING_SNAKE_CASE	)

		return tpu
def      _a					( SCREAMING_SNAKE_CASE	):


		"""simple docstring"""




		lowercase__				= 0
		for file in file_list:
				lowercase__				= file.split('''/'''	)[-1]
				lowercase__				= re.search(R'''-\d+-(\d+)\.tfrecord'''		,							SCREAMING_SNAKE_CASE	).group(1	)
				lowercase__				= int(SCREAMING_SNAKE_CASE	)
				num_samples += sample_count

		return num_samples
def      _a					( SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE=None	):


		"""simple docstring"""




		lowercase__				= count_samples(SCREAMING_SNAKE_CASE	)
		lowercase__				= tf.data.Dataset.from_tensor_slices(SCREAMING_SNAKE_CASE	)
		if shuffle:
				lowercase__				= dataset.shuffle(len(SCREAMING_SNAKE_CASE	)	)
		lowercase__				= tf.data.TFRecordDataset(SCREAMING_SNAKE_CASE		,							num_parallel_reads=SCREAMING_SNAKE_CASE	)
		# TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
		lowercase__				= dataset.apply(tf.data.experimental.assert_cardinality(SCREAMING_SNAKE_CASE	)	)
		lowercase__				= dataset.map(SCREAMING_SNAKE_CASE		,							num_parallel_calls=SCREAMING_SNAKE_CASE	)
		if shuffle:
				assert shuffle_buffer_size is not None
				lowercase__				= dataset.shuffle(args.shuffle_buffer_size	)
		lowercase__				= dataset.batch(SCREAMING_SNAKE_CASE		,							drop_remainder=SCREAMING_SNAKE_CASE	)
		lowercase__				= dataset.map(SCREAMING_SNAKE_CASE		,							num_parallel_calls=SCREAMING_SNAKE_CASE	)
		lowercase__				= dataset.prefetch(SCREAMING_SNAKE_CASE	)
		return dataset
def      _a					( SCREAMING_SNAKE_CASE	):


		"""simple docstring"""




		if not args.no_tpu:
				lowercase__				= initialize_tpu(SCREAMING_SNAKE_CASE	)
				lowercase__				= tf.distribute.TPUStrategy(SCREAMING_SNAKE_CASE	)
		else:
				lowercase__				= tf.distribute.OneDeviceStrategy(device='''/gpu:0'''	)

		if args.bfloataa:
				tf.keras.mixed_precision.set_global_policy('''mixed_bfloat16'''	)

		lowercase__				= AutoTokenizer.from_pretrained(args.tokenizer	)
		lowercase__				= AutoConfig.from_pretrained(args.pretrained_model_config	)
		lowercase__				= tokenizer.vocab_size

		lowercase__				= tf.io.gfile.glob(os.path.join(args.train_dataset		,							'''*.tfrecord'''	)	)
		if not training_records:
				raise ValueError(f'No .tfrecord files found in {args.train_dataset}.'	)
		lowercase__				= tf.io.gfile.glob(os.path.join(args.eval_dataset		,							'''*.tfrecord'''	)	)
		if not eval_records:
				raise ValueError(f'No .tfrecord files found in {args.eval_dataset}.'	)

		lowercase__				= count_samples(SCREAMING_SNAKE_CASE	)

		lowercase__				= num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
		lowercase__				= steps_per_epoch * args.num_epochs

		with strategy.scope():
				lowercase__				= TFAutoModelForMaskedLM.from_config(SCREAMING_SNAKE_CASE	)
				model(model.dummy_inputs	)  # Pass some dummy inputs through the model to ensure all the weights are built
				lowercase__	,						lowercase__				= create_optimizer(
				    num_train_steps=SCREAMING_SNAKE_CASE		,							num_warmup_steps=total_train_steps // 20		,							init_lr=args.learning_rate		,							weight_decay_rate=args.weight_decay_rate		,							)

				# Transformers models compute the right loss for their task by default when labels are passed, and will
				# use this for training unless you specify your own loss function in compile().
				model.compile(optimizer=SCREAMING_SNAKE_CASE		,							metrics=['''accuracy''']	)

		def decode_fn(SCREAMING_SNAKE_CASE	):
				lowercase__				= {
				    '''input_ids''': tf.io.FixedLenFeature(dtype=tf.intaa		,							shape=(args.max_length,)	),
				    '''attention_mask''': tf.io.FixedLenFeature(dtype=tf.intaa		,							shape=(args.max_length,)	),
				}
				return tf.io.parse_single_example(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE	)

		# Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
		# use their methods in our data pipeline.
		lowercase__				= DataCollatorForLanguageModeling(
		    tokenizer=SCREAMING_SNAKE_CASE		,							mlm_probability=args.mlm_probability		,							mlm=SCREAMING_SNAKE_CASE		,							return_tensors='''tf'''	)

		def mask_with_collator(SCREAMING_SNAKE_CASE	):
				# TF really needs an isin() function
				lowercase__				= (
				    ~tf.cast(batch['''attention_mask''']		,							tf.bool	)
				    | (batch['''input_ids'''] == tokenizer.cls_token_id)
				    | (batch['''input_ids'''] == tokenizer.sep_token_id)
				)
				lowercase__	,						lowercase__				= data_collator.tf_mask_tokens(
				    batch['''input_ids''']		,							vocab_size=len(SCREAMING_SNAKE_CASE	)		,							mask_token_id=tokenizer.mask_token_id		,							special_tokens_mask=SCREAMING_SNAKE_CASE		,							)
				return batch

		lowercase__				= args.per_replica_batch_size * strategy.num_replicas_in_sync

		lowercase__				= prepare_dataset(
		    SCREAMING_SNAKE_CASE		,							decode_fn=SCREAMING_SNAKE_CASE		,							mask_fn=SCREAMING_SNAKE_CASE		,							batch_size=SCREAMING_SNAKE_CASE		,							shuffle=SCREAMING_SNAKE_CASE		,							shuffle_buffer_size=args.shuffle_buffer_size		,							)

		lowercase__				= prepare_dataset(
		    SCREAMING_SNAKE_CASE		,							decode_fn=SCREAMING_SNAKE_CASE		,							mask_fn=SCREAMING_SNAKE_CASE		,							batch_size=SCREAMING_SNAKE_CASE		,							shuffle=SCREAMING_SNAKE_CASE		,							)

		lowercase__				= []
		if args.hub_model_id:
				callbacks.append(
				    PushToHubCallback(output_dir=args.output_dir		,							hub_model_id=args.hub_model_id		,							tokenizer=SCREAMING_SNAKE_CASE	)	)

		model.fit(
		    SCREAMING_SNAKE_CASE		,							validation_data=SCREAMING_SNAKE_CASE		,							epochs=args.num_epochs		,							callbacks=SCREAMING_SNAKE_CASE		,							)

		model.save_pretrained(args.output_dir	)


if __name__ == "__main__":
			lowerCAmelCase			=				parse_args()
			main(args)









import argparse

import torch

from transformers import GPTaLMHeadModel, RobertaForMaskedLM


if __name__ == "__main__":
       _lowerCamelCase      							=   argparse.ArgumentParser(
           description=(
               'Extraction some layers of the full RobertaForMaskedLM or GPT2LMHeadModel for Transfer Learned'
               ' Distillation'
           )
       )
       parser.add_argument('--model_type', default='roberta', choices=['roberta', 'gpt2'])
       parser.add_argument('--model_name', default='roberta-large', type=str)
       parser.add_argument('--dump_checkpoint', default='serialization_dir/tf_roberta_048131723.pth', type=str)
       parser.add_argument('--vocab_transform', action='store_true')
       _lowerCamelCase      							=   parser.parse_args()

       if args.model_type == "roberta":
              _lowerCamelCase      							=   RobertaForMaskedLM.from_pretrained(args.model_name)
              _lowerCamelCase      							=   'roberta'
       elif args.model_type == "gpt2":
              _lowerCamelCase      							=   GPTaLMHeadModel.from_pretrained(args.model_name)
              _lowerCamelCase      							=   'transformer'

       _lowerCamelCase      							=   model.state_dict()
       _lowerCamelCase      							=   {}

       # Embeddings #
       if args.model_type == "gpt2":
              for param_name in ["wte.weight", "wpe.weight"]:
                     _lowerCamelCase      							=   state_dict[F'''{prefix}.{param_name}''']
       else:
              for w in ["word_embeddings", "position_embeddings", "token_type_embeddings"]:
                     _lowerCamelCase      							=   F'''{prefix}.embeddings.{w}.weight'''
                     _lowerCamelCase      							=   state_dict[param_name]
              for w in ["weight", "bias"]:
                     _lowerCamelCase      							=   F'''{prefix}.embeddings.LayerNorm.{w}'''
                     _lowerCamelCase      							=   state_dict[param_name]

    # Transformer Blocks #
       _lowerCamelCase      							=   0
       for teacher_idx in [0, 2, 4, 7, 9, 11]:
              if args.model_type == "gpt2":
                     for layer in ["ln_1", "attn.c_attn", "attn.c_proj", "ln_2", "mlp.c_fc", "mlp.c_proj"]:
                            for w in ["weight", "bias"]:
                                   _lowerCamelCase      							=   state_dict[
                                       F'''{prefix}.h.{teacher_idx}.{layer}.{w}'''
                                   ]
                     _lowerCamelCase      							=   state_dict[F'''{prefix}.h.{teacher_idx}.attn.bias''']
              else:
                     for layer in [
                         "attention.self.query",
                         "attention.self.key",
                         "attention.self.value",
                         "attention.output.dense",
                         "attention.output.LayerNorm",
                         "intermediate.dense",
                         "output.dense",
                         "output.LayerNorm",
                     ]:
                            for w in ["weight", "bias"]:
                                   _lowerCamelCase      							=   state_dict[
                                       F'''{prefix}.encoder.layer.{teacher_idx}.{layer}.{w}'''
                                   ]
              std_idx += 1

       # Language Modeling Head ###s
       if args.model_type == "roberta":
              for layer in ["lm_head.decoder.weight", "lm_head.bias"]:
                     _lowerCamelCase      							=   state_dict[F'''{layer}''']
              if args.vocab_transform:
                     for w in ["weight", "bias"]:
                            _lowerCamelCase      							=   state_dict[F'''lm_head.dense.{w}''']
                            _lowerCamelCase      							=   state_dict[F'''lm_head.layer_norm.{w}''']
       elif args.model_type == "gpt2":
              for w in ["weight", "bias"]:
                     _lowerCamelCase      							=   state_dict[F'''{prefix}.ln_f.{w}''']
              _lowerCamelCase      							=   state_dict['lm_head.weight']

       print(F'''N layers selected for distillation: {std_idx}''')
       print(F'''Number of params transferred for distillation: {len(compressed_sd.keys())}''')

       print(F'''Save transferred checkpoint to {args.dump_checkpoint}.''')
       torch.save(compressed_sd, args.dump_checkpoint)

