def transformer_wikitext103_l4k_memory_v0():
  """HParams for training languagemodel_wikitext103_l4k with memory."""
  hparams = transformer_wikitext103_l4k_v0()

  hparams.split_targets_chunk_length = 64
  hparams.split_targets_max_chunks = 64
  hparams.split_targets_strided_training = True
  hparams.add_hparam("memory_type", "transformer_xl")

  # The hparams specify batch size *before* chunking, but we want to have a
  # consistent 4K batch size *after* chunking to fully utilize the hardware.
  target_tokens_per_batch = 4096
  hparams.batch_size = int(target_tokens_per_batch * (
      hparams.max_length / hparams.split_targets_chunk_length))  # 262144

  hparams.pos = None
  hparams.self_attention_type = "dot_product_relative"
  hparams.max_relative_position = 2 * hparams.split_targets_chunk_length

  hparams.add_hparam("unconditional", True)
  hparams.add_hparam("recurrent_memory_batch_size", 0)  # 0 = try to guess
  # By default, cache one chunk only (like Transformer-XL)
  hparams.add_hparam("num_memory_items", hparams.split_targets_chunk_length)

  return hparams
HParams for training languagemodel_wikitext103_l4k with memory.