add ELU support
MODIFY
test/test_ops.py
test/test_ops.py
@@ -73,6 +73,8 @@ class TestOps(unittest.TestCase):
pass
# fails?
#helper_test_op([(45,65)], lambda x: 0.5 * x * (1.0 + torch.tanh(x * 0.7978845608 * (1.0 + 0.044715 * x * x))), Tensor.gelu)
+ def test_elu(self):
+ helper_test_op([(45,65)], lambda x: torch.nn.functional.elu(x), Tensor.elu)
def test_relu6(self):
helper_test_op([(45,65)], lambda x: torch.nn.functional.relu6(x), Tensor.relu6)
def test_hardswish(self):

MODIFY
tinygrad/tensor.py
tinygrad/tensor.py
@@ -293,6 +293,9 @@ class Tensor:
#e = self.exp(); return e.div(1 + e)
return (1.0 + (0.0-self).exp()) ** -1.0
+ def elu(self, alpha=1.0):
+ return self.relu() - (-alpha*(self.exp() - 1)).relu()
+
def swish(self):
return self * self.sigmoid()

add ELU support