diff --git a/tensor2tensor/utils/learning_rate.py b/tensor2tensor/utils/learning_rate.py
index <HASH>..<HASH> 100644
--- a/tensor2tensor/utils/learning_rate.py
+++ b/tensor2tensor/utils/learning_rate.py
@@ -52,6 +52,14 @@ def learning_rate_factor(name, step_num, hparams):
     return tf.math.cos(
         step * np.pi /
         (hparams.train_steps - hparams.learning_rate_warmup_steps)) / 2.0 + 0.5
+  elif name == "multi_cycle_cos_decay":
+    # Cosine decay with a variable number of cycles. This is different from
+    # "cosdecay" because it starts at 1 when the warmup steps end. Use
+    # hparams.learning_rate_decay_steps to determine the number of cycles.
+    x = tf.maximum(step_num, hparams.learning_rate_warmup_steps)
+    step = x - hparams.learning_rate_warmup_steps
+    return tf.math.cos(
+        step * np.pi / hparams.learning_rate_decay_steps) / 2.0 + 0.5
   elif name == "rsqrt_decay":
     return tf.rsqrt(tf.maximum(step_num, hparams.learning_rate_warmup_steps))
   elif name == "rsqrt_normalized_decay":
Cosine learning rate decay with multiple cycles.

PiperOrigin-RevId: <I>