no pow if mul will do
MODIFY
tinygrad/optim.py
tinygrad/optim.py
@@ -29,7 +29,7 @@ class RMSprop(Optimizer):
def step(self):
for i, t in enumerate(self.params):
- self.v[i] = self.decay * self.v[i] + (1.0 - self.decay) * t.grad.pow(2.0)
+ self.v[i] = self.decay * self.v[i] + (1.0 - self.decay) * t.grad * t.grad
t -= (t.grad * self.lr).div(self.v[i].sqrt() + self.eps)
class Adam(Optimizer):
@@ -45,5 +45,5 @@ class Adam(Optimizer):
a = self.lr * ((1.0 - self.b2**self.t)**0.5) / (1.0 - self.b1**self.t)
for i,t in enumerate(self.params):
self.m[i] = self.b1 * self.m[i] + (1.0 - self.b1) * t.grad
- self.v[i] = self.b2 * self.v[i] + (1.0 - self.b2) * t.grad.pow(2.0)
+ self.v[i] = self.b2 * self.v[i] + (1.0 - self.b2) * t.grad * t.grad
t -= a * self.m[i].div(self.v[i].sqrt() + self.eps)

no pow if mul will do