# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ### Cut below here

# But there's another view that you should also get comfortable with: a function can simply return a function.

def make_multiplier(x):
    def multiply_by(y):
        return x * y
    return multiply_by
make_multiplier(5)(2)


# Use this pattern to make a `make_projector`:

def make_projector(weights, bias):
    def forward(x):
        return weights @ x + bias
    return forward
make_projector(w, 1.0)(x)

# *Supplemental Notes (optional)*:
#
# - The inner function is called a *closure*.
# - This programming pattern is used quite frequently in user interface programming, e.g,. JavaScript.
# - Here we only used it for "partial application"; we could have instead used `functools.partial(f, weights = w, bias = 1.0)`.
# - We'll use this later for implementing backpropagation.
#

import numpy as np


def relu(x):
    out = np.maximum(0, x)
    def grad_x(grad_out):
        return (x > 0) * grad_out
    return out, grad_x
act, grad_act = relu(np.array([2.5, -1.3]))
act, grad_act(np.ones(2))


# +
def mse(inputs, targets):
    # Write out the mean by hand, for backprop
    num_elts = len(inputs)
    out = ((targets - inputs) ** 2).sum() / num_elts
    
    def grad_inputs(out_grad):
        return 2.0 * out / num_elts * out_grad

    return out, grad_inputs # don't return a grad for targets. It would be the same thing as grad_inputs anyway.

#loss, grad_loss = mse(y_pred, y_true)
#loss, grad_loss(1.0)


# +
def linear(x, weights, bias):
    out = x @ weights + bias
    
    def grad_x(out_grad):
        return out_grad @ weights.T
    def grad_weights(out_grad):
        return x.T @ out_grad
    def grad_bias(out_grad):
        return out_grad.sum(axis = 0)
    
    return out, grad_x, grad_weights, grad_bias

np.random.seed(0)
x = np.random.randn(5)
y = np.random.randn(1)
weights = np.random.randn(5, 1)
bias = np.random.randn(1)
out, grad_x, grad_weights, grad_bias = linear(x, weights, bias)

# -

def linreg(x, params, targets):
    weights = params['W']
    bias = params['b']
    predictions, _, grad_weights, grad_bias = linear(x, weights, bias)
    loss, grad_loss_fn = mse(predictions, targets)
    
    grad_loss = grad_loss_fn(1.0)
    grads = {
        'W': grad_weights(grad_loss),
        'b': grad_bias(grad_loss)
    }
    return predictions, grads


linreg(x, {'W': weights, 'b': bias}, y)





StarcoderdataJupyterScriptsDedupFiltered