# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] id="yHG6wUbvifuX"
# # Forecasting with machine learning

# + [markdown] id="vidayERjaO5q"
# ## Setup

# + id="gqWabzlJ63nL"
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf

keras = tf.keras


# + id="cg1hfKCPldZG"
def plot_series(time, series, format="-", start=0, end=None, label=None):
    plt.plot(time[start:end], series[start:end], format, label=label)
    plt.xlabel("Time")
    plt.ylabel("Value")
    if label:
        plt.legend(fontsize=14)
    plt.grid(True)
    
def trend(time, slope=0):
    return slope * time
  
  
def seasonal_pattern(season_time):
    """Just an arbitrary pattern, you can change it if you wish"""
    return np.where(season_time < 0.4,
                    np.cos(season_time * 2 * np.pi),
                    1 / np.exp(3 * season_time))

  
def seasonality(time, period, amplitude=1, phase=0):
    """Repeats the same pattern at each period"""
    season_time = ((time + phase) % period) / period
    return amplitude * seasonal_pattern(season_time)
  
  
def white_noise(time, noise_level=1, seed=None):
    rnd = np.random.RandomState(seed)
    return rnd.randn(len(time)) * noise_level


# + id="iL2DDjV3lel6"
time = np.arange(4 * 365 + 1)

slope = 0.05
baseline = 10
amplitude = 40
series = baseline + trend(time, slope) + seasonality(time, period=365, amplitude=amplitude)

noise_level = 5
noise = white_noise(time, noise_level, seed=42)

series += noise

plt.figure(figsize=(10, 6))
plot_series(time, series)
plt.show()


# + [markdown] id="ViWVB9qd8OIR"
# ## Forecasting with Machine Learning
#
# First, we will train a model to forecast the next step given the previous 30 steps, therefore, we need to create a dataset of 30-step windows for training.

# + id="1tl-0BOKkEtk"
def window_dataset(series, window_size, batch_size=32,
                   shuffle_buffer=1000):
    dataset = tf.data.Dataset.from_tensor_slices(series)
    dataset = dataset.window(window_size + 1, shift=1, drop_remainder=True)
    dataset = dataset.flat_map(lambda window: window.batch(window_size + 1))
    dataset = dataset.shuffle(shuffle_buffer)
    dataset = dataset.map(lambda window: (window[:-1], window[-1]))
    dataset = dataset.batch(batch_size).prefetch(1)
    return dataset


# + id="Zmp1JXKxk9Vb"
split_time = 1000
time_train = time[:split_time]
x_train = series[:split_time]
time_valid = time[split_time:]
x_valid = series[split_time:]

# + [markdown] id="T1IvwAFn8OIc"
# ### Linear Model

# + id="ieOKdcEQ0A6k"
keras.backend.clear_session()
tf.random.set_seed(42)                              # do we need to set a random no for new session?
np.random.seed(42)

window_size = 30
train_set = window_dataset(x_train, window_size)    # here we just use the function blindly :)
valid_set = window_dataset(x_valid, window_size)

model = keras.models.Sequential([
  keras.layers.Dense(1, input_shape=[window_size])
])
optimizer = keras.optimizers.SGD(lr=1e-5, momentum=0.9)
model.compile(loss=keras.losses.Huber(),               # what is huber? quadratic for small errors but linear for large errors (MAE) absolute error are large
              optimizer=optimizer,                     # try other like Adam rmsprop  
              metrics=["mae"])
model.fit(train_set, epochs=100, validation_data=valid_set)

# + id="N3N8AGRM8OIc"
keras.backend.clear_session()                          # do we need to set a random no for new session?
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = window_dataset(x_train, window_size)

model = keras.models.Sequential([
  keras.layers.Dense(1, input_shape=[window_size])
])

lr_schedule = keras.callbacks.LearningRateScheduler(
    lambda epoch: 1e-6 * 10**(epoch / 30))                  # what is the logic here?
optimizer = keras.optimizers.SGD(lr=1e-6, momentum=0.9)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])
history = model.fit(train_set, epochs=100, callbacks=[lr_schedule])    # call back here but why validation is not here?

# + id="PF9e7IDm8OId"
plt.semilogx(history.history["lr"], history.history["loss"])
plt.axis([1e-6, 1e-3, 0, 20])

# + id="uMNwyIFE8OIf"
keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = window_dataset(x_train, window_size)
valid_set = window_dataset(x_valid, window_size)

model = keras.models.Sequential([
  keras.layers.Dense(1, input_shape=[window_size])
])
optimizer = keras.optimizers.SGD(lr=1e-5, momentum=0.9)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])
early_stopping = keras.callbacks.EarlyStopping(patience=10)
model.fit(train_set, epochs=500,
          validation_data=valid_set,
          callbacks=[early_stopping])


# + id="_eaAX9g_jS5W"
def model_forecast(model, series, window_size):
    ds = tf.data.Dataset.from_tensor_slices(series)
    ds = ds.window(window_size, shift=1, drop_remainder=True)
    ds = ds.flat_map(lambda w: w.batch(window_size))           # No need to shuffle here because we are not training here.
    ds = ds.batch(32).prefetch(1)
    forecast = model.predict(ds)
    return forecast


# + id="FnIWROQ08OIj"
lin_forecast = model_forecast(model, series[split_time - window_size:-1], window_size)[:, 0]  # what is going on?

# + id="xd7Tj_fA8OIk"
lin_forecast.shape     # what is the vector here?
# -

type(lin_forecast)

x_valid.shape

# + id="F-nftslfgQJs"
plt.figure(figsize=(10, 6))
plot_series(time_valid, x_valid)
plot_series(time_valid, lin_forecast)

# + id="W4E_jXktf7iv"
keras.metrics.mean_absolute_error(x_valid, lin_forecast).numpy()

# + [markdown] id="9nEM33dZ8OIp"
# ### Dense Model Forecasting
# Lets see if we can we do better with additional layers?

# + id="RhGTv4G_8OIp"
keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = window_dataset(x_train, window_size)

model = keras.models.Sequential([
  keras.layers.Dense(10, activation="relu", input_shape=[window_size]),
  keras.layers.Dense(10, activation="relu"),
  keras.layers.Dense(1)
])

lr_schedule = keras.callbacks.LearningRateScheduler(
    lambda epoch: 1e-7 * 10**(epoch / 20))
optimizer = keras.optimizers.SGD(lr=1e-7, momentum=0.9)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])
history = model.fit(train_set, epochs=100, callbacks=[lr_schedule]) # here we don't use validation data b/c we are trying to get the learning rate
                                                                    # learning rate deals with training data only.

# + id="5g-nC_em8OIq"
plt.semilogx(history.history["lr"], history.history["loss"])
plt.axis([1e-7, 5e-3, 0, 30])    #How do we get this learning rate number? Is it by intuition?
# -

# Optional code
import pandas as pd
history.history.keys()
losses = pd.DataFrame(history.history)
losses['mae'].plot()

model.history.history.keys()
losses = pd.DataFrame(model.history.history)
losses['mae'].plot()
# Optional code

# + id="B7t0VrCH8OIr"
keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = window_dataset(x_train, window_size)
valid_set = window_dataset(x_valid, window_size)

model = keras.models.Sequential([
  keras.layers.Dense(10, activation="relu", input_shape=[window_size]),
  keras.layers.Dense(10, activation="relu"),
  keras.layers.Dense(1)
])

optimizer = keras.optimizers.SGD(lr=1e-5, momentum=0.9)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])
early_stopping = keras.callbacks.EarlyStopping(patience=10)    # if it doesn't make any change in last 10 epochs then stop
model.fit(train_set, epochs=500,
          validation_data=valid_set,   # here we use the validation data
          callbacks=[early_stopping])

# + id="RqQbX6DZ8OIu"
dense_forecast = model_forecast(
    model,
    series[split_time - window_size:-1],  # little bit before the validation series
    window_size)[:, 0]

# 30 time steps before the start of validation period. It causes overlap but ignore it here ????
# training 30 steps before?

# + id="98zwAuIo8OIv"
plt.figure(figsize=(10, 6))
plot_series(time_valid, x_valid)
plot_series(time_valid, dense_forecast)

# + id="EgkELN-58OIw"
keras.metrics.mean_absolute_error(x_valid, dense_forecast).numpy()   # little better with the MAE but  graph shows a diff story
# -



StarcoderdataJupyterScriptsDedupFiltered