     /// The current step.
     public var step: Int = 0
     /// The accumulated, exponentially decaying average of squared gradients.
-    public var averageSquared: Model.TangentVector
+    public var averageSquared: Model.TangentVector = .zero
     /// The accumulated parameter updates.
-    public var accumulatedDelta: Model.TangentVector
+    public var accumulatedDelta: Model.TangentVector = .zero
 
     public init(
         for model: __shared Model,

/// ADADELTA optimizer.
///
/// ADADELTA is a more robust extension of AdaGrad. ADADELTA adapts learning rates based on a moving
/// window of gradient updates rather accumulating all past gradients. ADADELTA can continue to
/// learn even after many update steps.
/// 
/// Reference: ["ADADELTA: An Adaptive Learning Rate Method"](https://arxiv.org/abs/1212.5701)
public class AdaDelta<Model: Layer>: Optimizer
    where Model.AllDifferentiableVariables == Model.TangentVector {
    public typealias Model = Model
    /// The learning rate.
    public var learningRate: Float

        self.rho = rho
        self.epsilon = epsilon
        self.decay = decay
    }

    public func update(_ model: inout Model, along direction: Model.TangentVector) {
        update(&model.allDifferentiableVariables, along: direction)
    }

    // TODO: Deprecate this when `Differentiable.AllDifferentiableVariables` is removed.
    public func update(
        _ model: inout Model.AllDifferentiableVariables,
        along direction: Model.TangentVector
    ) {
        step += 1
        let learningRate = self.learningRate / (1 + decay * Float(step))
        averageSquared = rho * averageSquared + (1 - rho) * direction .* direction
        var stepSize = direction .* Model.TangentVector.sqrt(accumulatedDelta + epsilon)
        stepSize ./= Model.TangentVector.sqrt(averageSquared + epsilon)
        model.move(along: -learningRate * stepSize)
        accumulatedDelta = rho * accumulatedDelta + (1 - rho) * stepSize .* stepSize
    }
}

