fix test failure on MATMUL=1 backward pass
MODIFY
accel/opencl/ops_opencl.py
accel/opencl/ops_opencl.py
@@ -232,7 +232,7 @@ class OpenCLBuffer(GPUBuffer):
options.append("-DDEPTHWISE_UNSTRIDED")
elif C.cin == 1:
options.append("-DDEPTHWISE")
- if int(os.getenv("MATMUL", 0)) and C.groups == 1 and C.H == 1 and C.W == 1 and C.iy == 1 and C.ix == 1 and C.oy == 1 and C.ox == 1 and C.sx == 1 and C.sy == 1 and C.dx == 1 and C.dy == 1:
+ if int(os.getenv("MATMUL", 0)) and C.groups == 1 and C.H == 1 and C.W == 1 and C.iy == 1 and C.ix == 1 and C.oy == 1 and C.ox == 1 and C.sx == 1 and C.sy == 1 and C.dx == 1 and C.dy == 1 and C.bs == 1:
options.append("-DMATMUL")
# NOTE: this is not actually a matmul, it's a vector * matrix

MODIFY
test/test_ops.py
test/test_ops.py
@@ -17,14 +17,21 @@ def helper_test_op(shps, torch_fxn, tinygrad_fxn, atol=1e-6, rtol=1e-3, grad_ato
out = torch_fxn(*ts)
ret = tinygrad_fxn(*tst)
- np.testing.assert_allclose(ret.cpu().data, out.detach().numpy(), atol=atol, rtol=rtol)
+ def compare(s, x,y,atol,rtol):
+ assert x.shape == y.shape
+ try:
+ np.testing.assert_allclose(x,y, atol=atol, rtol=rtol)
+ except Exception:
+ raise Exception(f"{s} failed shape {x.shape}")
+
+ compare("forward pass", ret.cpu().data, out.detach().numpy(), atol=atol, rtol=rtol)
if not forward_only:
out.mean().backward()
ret.mean().backward()
- for t, tt in zip(ts, tst):
- np.testing.assert_allclose(t.grad, tt.cpu().grad.data, atol=grad_atol, rtol=grad_rtol)
+ for i, (t, tt) in enumerate(zip(ts, tst)):
+ compare(f"backward pass tensor {i}", t.grad, tt.cpu().grad.data, atol=grad_atol, rtol=grad_rtol)
# speed
torch_fp = timeit.Timer(functools.partial(torch_fxn, *ts)).timeit(5) * 1000/5
@@ -92,6 +99,8 @@ class TestOps(unittest.TestCase):
helper_test_op([(45,65)], _mish_pytorch, Tensor.mish, atol=1e-4)
def test_dot(self):
helper_test_op([(45,65), (65,100)], lambda x,y: x.matmul(y), Tensor.dot, atol=1e-4)
+ def test_matmul_simple(self):
+ helper_test_op([(2), (2,2)], lambda x,y: x.matmul(y), Tensor.dot, atol=1e-4)
def test_matmul(self):
helper_test_op([(65), (65,99)], lambda x,y: x.matmul(y), Tensor.dot, atol=1e-4)
def test_broadcastdot(self):

fix test failure on MATMUL=1 backward pass