move datasets into datasets
ADD
datasets/__init__.py
+import io
+import numpy as np
+import gzip
+import tarfile
+import pickle
+from extra.utils import fetch
+
+def fetch_mnist():
+ parse = lambda file: np.frombuffer(gzip.open(file).read(), dtype=np.uint8).copy()
+ X_train = parse("datasets/mnist/train-images-idx3-ubyte.gz")[0x10:].reshape((-1, 28*28)).astype(np.float32)
+ Y_train = parse("datasets/mnist/train-labels-idx1-ubyte.gz")[8:]
+ X_test = parse("datasets/mnist/t10k-images-idx3-ubyte.gz")[0x10:].reshape((-1, 28*28)).astype(np.float32)
+ Y_test = parse("datasets/mnist/t10k-labels-idx1-ubyte.gz")[8:]
+ return X_train, Y_train, X_test, Y_test
+
+def fetch_cifar():
+ tt = tarfile.open(fileobj=io.BytesIO(fetch('https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz')), mode='r:gz')
+ db = pickle.load(tt.extractfile('cifar-10-batches-py/data_batch_1'), encoding="bytes")
+ X = db[b'data'].reshape((-1, 3, 32, 32))
+ Y = np.array(db[b'labels'])
+ return X, Y

RENAME
test/mnist/t10k-images-idx3-ubyte.gz
datasets/mnist/t10k-images-idx3-ubyte.gz
RENAME
test/mnist/t10k-labels-idx1-ubyte.gz
datasets/mnist/t10k-labels-idx1-ubyte.gz
RENAME
test/mnist/train-images-idx3-ubyte.gz
datasets/mnist/train-images-idx3-ubyte.gz
RENAME
test/mnist/train-labels-idx1-ubyte.gz
datasets/mnist/train-labels-idx1-ubyte.gz
MODIFY
examples/serious_mnist.py
examples/serious_mnist.py
@@ -9,7 +9,7 @@ import numpy as np
from tinygrad.tensor import Tensor, GPU
from tinygrad.nn import BatchNorm2D
from extra.utils import get_parameters
-from test_mnist import fetch_mnist
+from datasets import fetch_mnist
from extra.training import train, evaluate, sparse_categorical_crossentropy
import tinygrad.optim as optim
from extra.augment import augment_img

MODIFY
examples/train_efficientnet.py
examples/train_efficientnet.py
import os
import time
import numpy as np
-from extra.efficientnet import EfficientNet
+from models.efficientnet import EfficientNet
from tinygrad.tensor import Tensor
-from extra.utils import get_parameters, fetch
+from extra.utils import get_parameters
from tqdm import trange
import tinygrad.optim as optim
-import io
-import tarfile
-import pickle
+from datasets import fetch_cifar
class TinyConvNet:
def __init__(self, classes=10):
@@ -24,15 +22,8 @@ class TinyConvNet:
x = x.reshape(shape=[x.shape[0], -1])
return x.dot(self.l1).logsoftmax()
-def load_cifar():
- tt = tarfile.open(fileobj=io.BytesIO(fetch('https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz')), mode='r:gz')
- db = pickle.load(tt.extractfile('cifar-10-batches-py/data_batch_1'), encoding="bytes")
- X = db[b'data'].reshape((-1, 3, 32, 32))
- Y = np.array(db[b'labels'])
- return X, Y
-
if __name__ == "__main__":
- X_train, Y_train = load_cifar()
+ X_train, Y_train = fetch_cifar()
classes = 10
Tensor.default_gpu = os.getenv("GPU") is not None

MODIFY
test/test_mnist.py
test/test_mnist.py
@@ -6,16 +6,7 @@ from tinygrad.tensor import Tensor
import tinygrad.optim as optim
from extra.training import train, evaluate
from extra.utils import fetch, get_parameters
-
-# mnist loader
-def fetch_mnist():
- import gzip
- parse = lambda file: np.frombuffer(gzip.open(file).read(), dtype=np.uint8).copy()
- X_train = parse("test/mnist/train-images-idx3-ubyte.gz")[0x10:].reshape((-1, 28*28)).astype(np.float32)
- Y_train = parse("test/mnist/train-labels-idx1-ubyte.gz")[8:]
- X_test = parse("test/mnist/t10k-images-idx3-ubyte.gz")[0x10:].reshape((-1, 28*28)).astype(np.float32)
- Y_test = parse("test/mnist/t10k-labels-idx1-ubyte.gz")[8:]
- return X_train, Y_train, X_test, Y_test
+from datasets import fetch_mnist
# load the mnist dataset
X_train, Y_train, X_test, Y_test = fetch_mnist()

move datasets into datasets