Refactor CL.time_sum into GlobalCounters
MODIFY
examples/benchmark_train_efficientnet.py
examples/benchmark_train_efficientnet.py
@@ -31,7 +31,7 @@ if __name__ == "__main__":
Tensor.training = TRAINING
Tensor.no_grad = not BACKWARD
for i in trange(CNT):
- CL.time_sum = 0
+ GlobalCounters.time_sum = 0
GlobalCounters.global_ops = 0
cpy = time.monotonic()
x_train = Tensor.randn(BS, 3, 224, 224, requires_grad=False).realize()

MODIFY
examples/hlb_cifar10.py
examples/hlb_cifar10.py
@@ -107,7 +107,8 @@ def train_cifar():
for i in range(1 if getenv("GRAPH") else 10):
# TODO: the real batch size is 512
X, Y = fetch_batch(X_train, Y_train, BS=getenv("BS", 512))
- CL.time_sum, CL.kernel_count = 0, -1
+ GlobalCounters.time_sum = 0
+ CL.kernel_count = -1
CL.ops_sum = 0 # TODO: this should be GlobalCounters.global_ops
st = time.monotonic()
loss = train_step_jitted(model, optimizer, X, Y, enable_jit=getenv("CLCACHE"))

MODIFY
tinygrad/ops.py
tinygrad/ops.py
@@ -63,7 +63,7 @@ class GenericExecAST(DeviceBuffer): # pylint: disable=abstract-method
return ret
class GlobalCounters:
- global_ops, global_mem = 0, 0
+ global_ops, global_mem, time_sum = 0, 0, 0
class GenericShape(GenericExecAST): # pylint: disable=abstract-method
def __init__(self, shape, flops=0): self.shape, self.flops = shape, flops

MODIFY
tinygrad/runtime/opencl.py
tinygrad/runtime/opencl.py
@@ -3,7 +3,7 @@ import numpy as np
import pyopencl as cl # type: ignore
from typing import Dict, Optional, Tuple, List
from collections import defaultdict
-from tinygrad.ops import DEBUG
+from tinygrad.ops import DEBUG, GlobalCounters
from tinygrad.helpers import getenv
OSX = platform.system() == "Darwin"
@@ -13,7 +13,7 @@ CLCACHE = getenv("CLCACHE", 1)
FLOAT16 = getenv("FLOAT16", 0)
class CL:
- CACHE, kernel_count, mem_used, time_sum, ops_sum = None, -1, 0, 0.0, 0.0
+ CACHE, kernel_count, mem_used, ops_sum = None, -1, 0, 0.0
BUFFER_CACHE : Dict[int, List[cl.Buffer]] = defaultdict(list)
cl_ctx : Optional[cl.Context] = None
cl_queue : Optional[cl.CommandQueue] = None
@@ -84,9 +84,9 @@ class CLProgram:
CL.cl_queue.finish()
# NOTE: Profiling is not in ns in OS X, we multiply by a computed ratio
et = (e.profile.end - e.profile.start) * OSX_TIMING_RATIO
- CL.time_sum += et
+ GlobalCounters.time_sum += et
if DEBUG >= 1:
CL.ops_sum += self.op_estimate
print(f"**CL** {CL.kernel_count:6d} {self.name:28s} args {len(args[2:]):5d} kernels {str(args[0]):18s} {str(args[1]):12s} OPs {self.op_estimate/1e6:7.1f}M/{CL.ops_sum/1e9:7.2f}G mem {CL.mem_used/1e9:5.2f} GB " +
- (str() if DEBUG <= 1 or CL.CACHE is not None else f"tm {et/1e3:9.2f}us/{CL.time_sum/1e6:9.2f}ms ({self.op_estimate/et:8.2f} GFLOPS)"))
+ (str() if DEBUG <= 1 or CL.CACHE is not None else f"tm {et/1e3:9.2f}us/{GlobalCounters.time_sum/1e6:9.2f}ms ({self.op_estimate/et:8.2f} GFLOPS)"))
return e if CL.CACHE is None else None

Refactor CL.time_sum into GlobalCounters (#519)