hmm, not allowed
MODIFY
tinygrad/optim.py
tinygrad/optim.py
@@ -28,7 +28,7 @@ class RMSprop(Optimizer):
def step(self):
for i, t in enumerate(self.params):
self.v[i] = self.decay * self.v[i] + (1.0 - self.decay) * (t.grad * t.grad)
- t -= (t.grad * self.lr).mul(self.v[i].add(self.eps).pow(-0.5))
+ t -= (t.grad * self.lr).div(self.v[i].sqrt() + self.eps)
class Adam(Optimizer):
def __init__(self, params, lr=0.001, b1=0.9, b2=0.999, eps=1e-8):
@@ -44,4 +44,4 @@ class Adam(Optimizer):
for i, t in enumerate(self.params):
self.m[i] = self.b1 * self.m[i] + (1.0 - self.b1) * t.grad
self.v[i] = self.b2 * self.v[i] + (1.0 - self.b2) * (t.grad * t.grad)
- t -= a * self.m[i].mul(self.v[i].add(self.eps).pow(-0.5))
+ t -= a * self.m[i].div(self.v[i].sqrt() + self.eps)

hmm, not allowed