# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import torch

dtype = torch.float
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
N, D_in, H, D_out = 64, 1000, 100, 10
x = torch.randn(N, D_in, device=device, dtype=dtype)
y = torch.randn(N, D_out, device=device, dtype=dtype)

w1 = torch.randn(D_in, H, device=device, dtype=dtype, requires_grad=True)
w2 = torch.randn(H, D_out, device=device, dtype=dtype, requires_grad=True)
lr =  1e-6
print(x.size())
print(w1.size())
print(w2.size())
print(y.size())

for i in range(500):
    y_pred = x.mm(w1).clamp(min=0).mm(w2)
    loss = (y_pred - y).pow(2).sum()
    print(i, loss.item())
    loss.backward()

    with torch.no_grad():
        w1 -=  lr * w1.grad
        w2 -=  lr * w2.grad
        w1.grad.zero_()
        w2.grad.zero_()

StarcoderdataJupyterScriptsDedupFiltered