message: Added option (off by default) to enable a higher-performance variant of the
  Adam optimizer's variable update formula.
diff: @@ -62,7 +62,10 @@ message FtrlParameters {
  // (https://www.tensorflow.org/api_docs/python/tf/contrib/opt/AdamOptimizer). If
  // use_non_lazy_adam is enabled, use_gradient_accumulation is also required in
  // order to get correct results; a warning will be printed otherwise (which may
  -// change to an error in the future).
  +// change to an error in the future). If use_max_with_epsilon is set, the Adam
  +// variable update formula will be changed from m / (sqrt(v) + epsilon) to
  +// m / max(sqrt(v), abs(epsilon)); this option improves the performance of TPU
  +// training and is not expected to harm model quality.
  message AdamParameters {
  float beta1 = 3;
  float beta2 = 4;
  @@ -70,6 +73,7 @@ message AdamParameters {
  float initial_m = 6;
  float initial_v = 7;
  bool use_non_lazy_adam = 8;
  + bool use_max_with_epsilon = 9;
  }
  // https://www.tensorflow.org/api_docs/python/tf/train/MomentumOptimizer
  