dashed loadops
MODIFY
examples/benchmark_train_efficientnet.py
examples/benchmark_train_efficientnet.py
@@ -33,8 +33,8 @@ if __name__ == "__main__":
Tensor.training = True
for i in trange(CNT):
- x_train = Tensor.randn(BS, 3, 224, 224, requires_grad=False)
- y_train = Tensor.randn(BS, 1000, requires_grad=False)
+ x_train = Tensor.randn(BS, 3, 224, 224, requires_grad=False).realize()
+ y_train = Tensor.randn(BS, 1000, requires_grad=False).realize()
st = time.monotonic()
out = model.forward(x_train)
@@ -50,7 +50,9 @@ if __name__ == "__main__":
et = time.monotonic()
loss = loss.detach().cpu().data[0]
- print(f"{(et-st)*1000.0:7.2f} ms run, {(mt-st)*1000.0:7.2f} ms build, {loss:7.2f} loss, {tensors_allocated():4d} tensors, {get_memory_used()/1e9:.2f} GB used")
+ cl = time.monotonic()
+
+ print(f"{(cl-st)*1000.0:7.2f} ms run, {(mt-st)*1000.0:7.2f} ms build, {(et-mt)*1000.0:7.2f} ms realize, {(cl-et)*1000.0:7.2f} ms CL, {loss:7.2f} loss, {tensors_allocated():4d} tensors, {get_memory_used()/1e9:.2f} GB used")

MODIFY
tinygrad/llops/ops_gpu.py
tinygrad/llops/ops_gpu.py
@@ -38,7 +38,7 @@ class CLProgram:
if argdtypes is not None: self.clprg.set_scalar_arg_dtypes(argdtypes)
def __call__(self, *args):
CL.kernel_count += 1
- if DEBUG >= 1: print(f"**CL** {CL.kernel_count:4d} {self.name:20s} {len(args[2:]):3d} {args[0]} {args[1]}")
+ if DEBUG >= 1: print(f"**CL** {CL.kernel_count:4d} {self.name:20s} args {len(args[2:]):5d} size {prod(args[0]):7d} kernels {args[0]} {args[1]}")
if DEBUG >= 3: print(self.prg)
if CL.CACHE is not None: CL.CACHE.append((self, args))
else: self.clprg(CL().cl_queue, *args)

MODIFY
tinygrad/ops.py
tinygrad/ops.py
@@ -68,6 +68,8 @@ def log_op(optype : OpType, op : List[Op], ret : DeviceBuffer, inp : List[Device
top_colors = {LoadOps: '#FFFF80', UnaryOps: "#c0c0c0", ReduceOps: "#8080ff", BinaryOps: "#c0c0c0", MovementOps: "#80ff80", ProcessingOps: "#ff8080"}
+ dashed = optype == LoadOps and getattr(ret, "_backing", None) is not None
+
for x in inp:
if len(op) <= 2: sop = '.'.join([str(y).split(".")[1] for y in op][::-1])
elif len(op) <= 4: sop = '.'.join([str(y).split(".")[1][0:2] for y in op][::-1])
@@ -75,14 +77,13 @@ def log_op(optype : OpType, op : List[Op], ret : DeviceBuffer, inp : List[Device
G.add_edge(nm(x), nm(ret), label=sop)
if 'label' not in G.nodes[nm(x)]: G.nodes[nm(x)]['label'] = str(x.shape)
if nm(ret) not in G.nodes: G.add_node(nm(ret))
- st = getattr(ret, "st", None)
- non_contiguous = st is not None and not st.contiguous
- if non_contiguous and st is not None: # checked twice to make type checker happy
- G.nodes[nm(ret)]['label'] = str(tuple(x[0] if x[1]!=0 else 0 for x in st.views[-1].shape_strides))
+ if getattr(ret, "st", None) is not None and not ret.st.contiguous: # checked twice to make type checker happy
+ G.nodes[nm(ret)]['label'] = str(tuple(x[0] if x[1]!=0 else 0 for x in ret.st.views[-1].shape_strides))
+ dashed = True
else:
G.nodes[nm(ret)]['label'] = str(ret.shape)
- G.nodes[nm(ret)]['fillcolor'] = (top_colors[optype] + ('80' if non_contiguous else '')) if optype in top_colors else "#ffffff"
- G.nodes[nm(ret)]['style'] = 'filled, dashed' if non_contiguous else 'filled'
+ G.nodes[nm(ret)]['fillcolor'] = (top_colors[optype] + ('80' if dashed else '')) if optype in top_colors else "#ffffff"
+ G.nodes[nm(ret)]['style'] = 'filled, dashed' if dashed else 'filled'
# **** realize functions ****

MODIFY
tinygrad/tensor.py
tinygrad/tensor.py
@@ -52,6 +52,7 @@ class Tensor:
def realize(self):
self.lazydata.realize()
+ return self
def assign(self, x):
if not isinstance(x, Tensor):
@@ -309,8 +310,7 @@ class Tensor:
def pow(self, x): return Tensor.broadcasted(Tensor._pow, self, x)
# TODO: should be broadcasted binary op
- def div(self, y):
- return self * (y ** -1.0)
+ def div(self, y): return self * (y ** -1.0)
__truediv__ = div
# ***** functional nn ops *****

dashed loadops