remove useless init, add ops counter
ADD
accel/MAPPING
+We have to figure out how to make the tinygrad ops match to hw.
+Generic folded reduce may not work.
+
+
+GPUs:
+
+RDNA2: https://developer.amd.com/wp-content/resources/RDNA2_Shader_ISA_November2020.pdf
+
+We have RX6900XT with 80 CU, 40 WGP, and 20 "processors"
+@ 1.825 GHz, there's 18,688 FP32 GFLOPS of compute. 10240 FLOPS/cycle, 128 per CU
+
+286 GFLOP for ENET=2 BS=64. At theoretical max, (286/18688)*1000 = 15.3 ms.
+We observe about 10x factor off with pytorch.
+
+
+On M1 GPU, theoretical is 2.275 TFLOPS. https://www.notebookcheck.net/Apple-M1-GPU-Benchmarks-and-Specs.503610.0.html
+
+We observe 2000ms for BS=8 (37 GFLOP). 37/2275 = 11.9 ms. tinygrad is over a factor of 100x off (similar on AMD GPU)
+
+
+Making matters worse, flop estimates may be off by a factor of 10.
+
+
+TPUs:
+
+
+
+
+
+
+

MODIFY
tinygrad/llops/ops_gpu.py
tinygrad/llops/ops_gpu.py
@@ -22,7 +22,7 @@ class CLBuffer:
else: CL.mem_used -= self.cl.size
class CL:
- CACHE, kernel_count, mem_used = None, -1, 0
+ CACHE, kernel_count, mem_used, ops_sum = None, -1, 0, 0.0
BUFFER_CACHE : Dict[int, List[cl.Buffer]] = defaultdict(list)
cl_ctx : Optional[cl.Context] = None
cl_queue : Optional[cl.CommandQueue] = None
@@ -56,7 +56,8 @@ class CLProgram:
else: e = self.clprg(CL().cl_queue, *args)
if DEBUG >= 2: CL.cl_queue.finish()
if DEBUG >= 1:
- print(f"**CL** {CL.kernel_count:6d} {self.name:20s} args {len(args[2:]):5d} size {prod(args[0]):8d} kernels {str(args[0]):20s} {str(args[1]):20s}" + \
+ CL.ops_sum += max([x.size//4 for x in args[2:]])*(len(args)-2)
+ print(f"**CL** {CL.kernel_count:6d} {self.name:20s} args {len(args[2:]):5d} size {prod(args[0]):8d} kernels {str(args[0]):20s} {str(args[1]):14s} GOPs {CL.ops_sum/1e9:7.2f} " + \
("" if DEBUG <= 1 or CL.CACHE is not None else f"runtime {(e.profile.end - e.profile.start)/1e3:9.2f} us"))
if DEBUG >= 4: print(self.prg)

MODIFY
tinygrad/nn.py
tinygrad/nn.py
@@ -28,7 +28,6 @@ class BatchNorm2D:
if self.track_running_stats:
self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * batch_mean
self.running_var = (1 - self.momentum) * self.running_var + self.momentum * batch_var
- if self.num_batches_tracked is None: self.num_batches_tracked = Tensor.zeros(1, requires_grad=False)
self.num_batches_tracked += 1
return batch_normalize(x, self.weight, self.bias, batch_mean, batch_var, self.eps)

remove useless init, add ops counter