





import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


lowercase_          =		logging.getLogger(__name__)

lowercase_          =		tf.data.AUTOTUNE




def       __UpperCamelCase  ()    ->       int:
  lowercase__           =  argparse.ArgumentParser(description='Train a masked language model on TPU.'		)
  parser.add_argument(
      '--pretrained_model_config'		,						type=_UpperCAmelCase		,						default='roberta-base'		,						help='The model config to use. Note that we don\'t copy the model\'s weights, only the config!'		,						)
  parser.add_argument(
      '--tokenizer'		,						type=_UpperCAmelCase		,						default='unigram-tokenizer-wikitext'		,						help='The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model\'s vocab size.'		,						)

  parser.add_argument(
      '--per_replica_batch_size'		,						type=_UpperCAmelCase		,						default=8		,						help='Batch size per TPU core.'		,						)

  parser.add_argument(
      '--no_tpu'		,						action='store_true'		,						help='If set, run on CPU and don\'t try to initialize a TPU. Useful for debugging on non-TPU instances.'		,						)

  parser.add_argument(
      '--tpu_name'		,						type=_UpperCAmelCase		,						help='Name of TPU resource to initialize. Should be blank on Colab, and \'local\' on TPU VMs.'		,						default='local'		,						)

  parser.add_argument(
      '--tpu_zone'		,						type=_UpperCAmelCase		,						help='Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes.'		,						)

  parser.add_argument(
      '--gcp_project'		,						type=_UpperCAmelCase		,						help='Google cloud project name. Only used for non-Colab TPU nodes.'		)

  parser.add_argument(
      '--bfloat16'		,						action='store_true'		,						help='Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU.'		,						)

  parser.add_argument(
      '--train_dataset'		,						type=_UpperCAmelCase		,						help='Path to training dataset to load. If the path begins with `gs://`'
      ' then the dataset will be loaded from a Google Cloud Storage bucket.'		,						)

  parser.add_argument(
      '--shuffle_buffer_size'		,						type=_UpperCAmelCase		,						default=2**18		,						help='Size of the shuffle buffer (in samples)'		,						)

  parser.add_argument(
      '--eval_dataset'		,						type=_UpperCAmelCase		,						help='Path to evaluation dataset to load. If the path begins with `gs://`'
      ' then the dataset will be loaded from a Google Cloud Storage bucket.'		,						)

  parser.add_argument(
      '--num_epochs'		,						type=_UpperCAmelCase		,						default=1		,						help='Number of epochs to train for.'		,						)

  parser.add_argument(
      '--learning_rate'		,						type=_UpperCAmelCase		,						default=1E-4		,						help='Learning rate to use for training.'		,						)

  parser.add_argument(
      '--weight_decay_rate'		,						type=_UpperCAmelCase		,						default=1E-3		,						help='Weight decay rate to use for training.'		,						)

  parser.add_argument(
      '--max_length'		,						type=_UpperCAmelCase		,						default=512		,						help='Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py'		,						)

  parser.add_argument(
      '--mlm_probability'		,						type=_UpperCAmelCase		,						default=0.1_5		,						help='Fraction of tokens to mask during training.'		,						)

  parser.add_argument('--output_dir'		,						type=_UpperCAmelCase		,						required=_UpperCAmelCase		,						help='Path to save model checkpoints to.'		)
  parser.add_argument('--hub_model_id'		,						type=_UpperCAmelCase		,						help='Model ID to upload to on the Hugging Face Hub.'		)

  lowercase__           =  parser.parse_args()
  return args




def       __UpperCamelCase  (_SCREAMING_SNAKE_CASE		)    ->       Any:
  try:
    if args.tpu_name:
      lowercase__           =  tf.distribute.cluster_resolver.TPUClusterResolver(
          args.tpu_name		,						zone=args.tpu_zone		,						project=args.gcp_project		)
    else:
      lowercase__           =  tf.distribute.cluster_resolver.TPUClusterResolver()
  except ValueError:
    raise RuntimeError(
        'Couldn\'t connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or '
        '--gcp_project. When running on a TPU VM, use --tpu_name local.'		)

  tf.config.experimental_connect_to_cluster(_UpperCAmelCase		)
  tf.tpu.experimental.initialize_tpu_system(_UpperCAmelCase		)

  return tpu




def       __UpperCamelCase  (_SCREAMING_SNAKE_CASE		)    ->       Any:
  lowercase__           =  0
  for file in file_list:
    lowercase__           =  file.split('/'		)[-1]
    lowercase__           =  re.search(R'-\d+-(\d+)\.tfrecord'		,						_UpperCAmelCase		).group(1		)
    lowercase__           =  int(_UpperCAmelCase		)
    num_samples += sample_count

  return num_samples




def       __UpperCamelCase  (_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE=None		)    ->       List[str]:
  lowercase__           =  count_samples(_UpperCAmelCase		)
  lowercase__           =  tf.data.Dataset.from_tensor_slices(_UpperCAmelCase		)
  if shuffle:
    lowercase__           =  dataset.shuffle(len(_UpperCAmelCase		)		)
  lowercase__           =  tf.data.TFRecordDataset(_UpperCAmelCase		,						num_parallel_reads=_UpperCAmelCase		)
  # TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
  lowercase__           =  dataset.apply(tf.data.experimental.assert_cardinality(_UpperCAmelCase		)		)
  lowercase__           =  dataset.map(_UpperCAmelCase		,						num_parallel_calls=_UpperCAmelCase		)
  if shuffle:
    assert shuffle_buffer_size is not None
    lowercase__           =  dataset.shuffle(args.shuffle_buffer_size		)
  lowercase__           =  dataset.batch(_UpperCAmelCase		,						drop_remainder=_UpperCAmelCase		)
  lowercase__           =  dataset.map(_UpperCAmelCase		,						num_parallel_calls=_UpperCAmelCase		)
  lowercase__           =  dataset.prefetch(_UpperCAmelCase		)
  return dataset




def       __UpperCamelCase  (_SCREAMING_SNAKE_CASE		)    ->       str:
  if not args.no_tpu:
    lowercase__           =  initialize_tpu(_UpperCAmelCase		)
    lowercase__           =  tf.distribute.TPUStrategy(_UpperCAmelCase		)
  else:
    lowercase__           =  tf.distribute.OneDeviceStrategy(device='/gpu:0'		)

  if args.bfloataa:
    tf.keras.mixed_precision.set_global_policy('mixed_bfloat16'		)

  lowercase__           =  AutoTokenizer.from_pretrained(args.tokenizer		)
  lowercase__           =  AutoConfig.from_pretrained(args.pretrained_model_config		)
  lowercase__           =  tokenizer.vocab_size

  lowercase__           =  tf.io.gfile.glob(os.path.join(args.train_dataset		,						'*.tfrecord'		)		)
  if not training_records:
    raise ValueError(F"""No .tfrecord files found in {args.train_dataset}."""		)
  lowercase__           =  tf.io.gfile.glob(os.path.join(args.eval_dataset		,						'*.tfrecord'		)		)
  if not eval_records:
    raise ValueError(F"""No .tfrecord files found in {args.eval_dataset}."""		)

  lowercase__           =  count_samples(_UpperCAmelCase		)

  lowercase__           =  num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
  lowercase__           =  steps_per_epoch * args.num_epochs

  with strategy.scope():
    lowercase__           =  TFAutoModelForMaskedLM.from_config(_UpperCAmelCase		)
    model(model.dummy_inputs		)  # Pass some dummy inputs through the model to ensure all the weights are built
    lowercase__           =  create_optimizer(
        num_train_steps=_UpperCAmelCase		,						num_warmup_steps=total_train_steps // 20		,						init_lr=args.learning_rate		,						weight_decay_rate=args.weight_decay_rate		,						)

    # Transformers models compute the right loss for their task by default when labels are passed, and will
    # use this for training unless you specify your own loss function in compile().
    model.compile(optimizer=_UpperCAmelCase		,						metrics=['accuracy']		)

  def decode_fn(_SCREAMING_SNAKE_CASE		):
    lowercase__           =  {
        'input_ids': tf.io.FixedLenFeature(dtype=tf.intaa		,						shape=(args.max_length,)		),
        'attention_mask': tf.io.FixedLenFeature(dtype=tf.intaa		,						shape=(args.max_length,)		),
    }
    return tf.io.parse_single_example(_UpperCAmelCase		,						_UpperCAmelCase		)

  # Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
  # use their methods in our data pipeline.
  lowercase__           =  DataCollatorForLanguageModeling(
      tokenizer=_UpperCAmelCase		,						mlm_probability=args.mlm_probability		,						mlm=_UpperCAmelCase		,						return_tensors='tf'		)

  def mask_with_collator(_SCREAMING_SNAKE_CASE		):
    # TF really needs an isin() function
    lowercase__           =  (
        ~tf.cast(batch['attention_mask']		,						tf.bool		)
        | (batch['input_ids'] == tokenizer.cls_token_id)
        | (batch['input_ids'] == tokenizer.sep_token_id)
    )
    lowercase__           =  data_collator.tf_mask_tokens(
        batch['input_ids']		,						vocab_size=len(_UpperCAmelCase		)		,						mask_token_id=tokenizer.mask_token_id		,						special_tokens_mask=_UpperCAmelCase		,						)
    return batch

  lowercase__           =  args.per_replica_batch_size * strategy.num_replicas_in_sync

  lowercase__           =  prepare_dataset(
      _UpperCAmelCase		,						decode_fn=_UpperCAmelCase		,						mask_fn=_UpperCAmelCase		,						batch_size=_UpperCAmelCase		,						shuffle=_UpperCAmelCase		,						shuffle_buffer_size=args.shuffle_buffer_size		,						)

  lowercase__           =  prepare_dataset(
      _UpperCAmelCase		,						decode_fn=_UpperCAmelCase		,						mask_fn=_UpperCAmelCase		,						batch_size=_UpperCAmelCase		,						shuffle=_UpperCAmelCase		,						)

  lowercase__           =  []
  if args.hub_model_id:
    callbacks.append(
        PushToHubCallback(output_dir=args.output_dir		,						hub_model_id=args.hub_model_id		,						tokenizer=_UpperCAmelCase		)		)

  model.fit(
      _UpperCAmelCase		,						validation_data=_UpperCAmelCase		,						epochs=args.num_epochs		,						callbacks=_UpperCAmelCase		,						)

  model.save_pretrained(args.output_dir		)


if __name__ == "__main__":
      lowercase_          =		parse_args()
      main(args)












import argparse

import torch
from torch import nn

from transformers import MBartConfig, MBartForConditionalGeneration




def       __UpperCamelCase  (_SCREAMING_SNAKE_CASE		)    ->       Union[str, Any]:
  lowercase__           =  [
      'encoder.version',
      'decoder.version',
      'model.encoder.version',
      'model.decoder.version',
      '_float_tensor',
      'decoder.output_projection.weight',
  ]
  for k in ignore_keys:
    state_dict.pop(_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE		)




def       __UpperCamelCase  (_SCREAMING_SNAKE_CASE		)    ->       Dict:
  lowercase__      ,   lowercase__           =  emb.weight.shape
  lowercase__           =  nn.Linear(_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE		,						bias=_SCREAMING_SNAKE_CASE		)
  lowercase__           =  emb.weight.data
  return lin_layer




def       __UpperCamelCase  (_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE="facebook/mbart-large-en-ro"		,						_SCREAMING_SNAKE_CASE=False		,						_SCREAMING_SNAKE_CASE=False		)    ->       str:
  lowercase__           =  torch.load(_SCREAMING_SNAKE_CASE		,						map_location='cpu'		)['model']
  remove_ignore_keys_(_SCREAMING_SNAKE_CASE		)
  lowercase__           =  state_dict['encoder.embed_tokens.weight'].shape[0]

  lowercase__           =  MBartConfig.from_pretrained(_SCREAMING_SNAKE_CASE		,						vocab_size=_SCREAMING_SNAKE_CASE		)
  if mbart_aa and finetuned:
    lowercase__           =  'relu'

  lowercase__           =  state_dict['decoder.embed_tokens.weight']
  lowercase__           =  MBartForConditionalGeneration(_SCREAMING_SNAKE_CASE		)
  model.model.load_state_dict(_SCREAMING_SNAKE_CASE		)

  if finetuned:
    lowercase__           =  make_linear_from_emb(model.model.shared		)

  return model


if __name__ == "__main__":
      lowercase_          =		argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          """fairseq_path""", type=str, help="""bart.large, bart.large.cnn or a path to a model.pt on local filesystem."""
      )
      parser.add_argument("""pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
      parser.add_argument(
          """--hf_config""",
          default="""facebook/mbart-large-cc25""",
          type=str,
          help="""Which huggingface architecture to use: mbart-large""",
      )
      parser.add_argument("""--mbart_50""", action="""store_true""", help="""whether the model is mMART-50 checkpoint""")
      parser.add_argument("""--finetuned""", action="""store_true""", help="""whether the model is a fine-tuned checkpoint""")
      lowercase_          =		parser.parse_args()
      lowercase_          =		convert_fairseq_mbart_checkpoint_from_disk(
          args.fairseq_path, hf_config_path=args.hf_config, finetuned=args.finetuned, mbart_aa=args.mbart_aa
      )
      model.save_pretrained(args.pytorch_dump_folder_path)


