debug in backward pass too
MODIFY
examples/train_efficientnet.py
examples/train_efficientnet.py
+import os
import time
import numpy as np
from efficientnet import EfficientNet
from tinygrad.tensor import Tensor
if __name__ == "__main__":
- Tensor.default_gpu = True
+ Tensor.default_gpu = os.getenv("GPU") is not None
model = EfficientNet()
BS = 4
@@ -16,6 +17,8 @@ if __name__ == "__main__":
et = time.time()
print("forward %.2f s" % (et-st))
+ Y = [0]*BS
+
y = np.zeros((BS,1000), np.float32)
y[range(y.shape[0]),Y] = -1000.0
y = Tensor(y)

MODIFY
tinygrad/tensor.py
tinygrad/tensor.py
@@ -16,7 +16,7 @@ if DEBUG:
debug_times = collections.defaultdict(float)
def print_debug_exit():
for name, _ in sorted(debug_times.items(), key=lambda x: -x[1]):
- print("%20s : %3d %5.2f ms" % (name, debug_counts[name], debug_times[name]))
+ print("%20s : %3d %10.2f ms" % (name, debug_counts[name], debug_times[name]))
atexit.register(print_debug_exit)
cl_ctx, cl_queue = None, None
@@ -101,7 +101,17 @@ class Tensor:
assert(self.grad is not None)
+ if DEBUG:
+ st = time.time()
grads = self._ctx.backward(self._ctx, self.grad.data)
+ if DEBUG:
+ global debug_counts, debug_times
+ name = "back_"+self._ctx.__class__.__name__
+ et = (time.time()-st)*1000.
+ debug_counts[name] += 1
+ debug_times[name] += et
+ print("%20s : %7.2f ms" % (name, et))
+
if len(self._ctx.parents) == 1:
grads = [grads]
for t,g in zip(self._ctx.parents, grads):
@@ -206,7 +216,7 @@ def register(name, fxn, gpu=False):
et = (time.time()-st)*1000.
debug_counts[name] += 1
debug_times[name] += et
- print("%20s : %5.2f ms %s" % (name, et, [y.shape for y in x]))
+ print("%20s : %7.2f ms %s" % (name, et, [y.shape for y in x]))
return ret
setattr(Tensor, name, dispatch)
if name in ['add', 'sub', 'mul', 'div']:

debug in backward pass too