# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Usages of `tf.keras`

# +
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np

import tensorflow as tf
from tensorflow.keras import layers

tf.enable_eager_execution()

print(tf.VERSION)
print(tf.keras.__version__)
# -

# ## Build a simple model
#
# ### Sequential model
# * Two types of the sequential model
#
# #### Type I

model1 = tf.keras.Sequential()
# Adds a densely-connected layer with 64 units to the model:
model1.add(layers.Dense(64, activation='relu'))
# Add another:
model1.add(layers.Dense(64, activation='relu'))
# Add a softmax layer with 10 output units:
model1.add(layers.Dense(10, activation='softmax'))

# #### Type II

model2 = tf.keras.Sequential([
# Adds a densely-connected layer with 64 units to the model:
layers.Dense(64, activation='relu'),
# Add another:
layers.Dense(64, activation='relu'),
# Add a softmax layer with 10 output units:
layers.Dense(10, activation='softmax')])

model = model2

model.compile(optimizer=tf.train.AdamOptimizer(0.001),
              loss='categorical_crossentropy',
              metrics=['accuracy'])

# #### Two types in `tf.keras.Model.compile`
#
# type I
# ```python
# model.compile(optimizer=tf.train.AdamOptimizer(0.001),
#               loss='categorical_crossentropy',
#               metrics=['accuracy'])```
#               
# type II
# ```python
# model.compile(optimizer=tf.train.AdamOptimizer(0.001),
#               loss=tf.keras.losses.categorical_crossentropy,
#               metrics=[tf.keras.metrics.categorical_accuracy])```

# ## Input
#
# ### Input using numpy data

data = np.random.random((1000, 32))
labels = np.random.random((1000, 10))

model.fit(data, labels, epochs=10, batch_size=32)

# #### validation data

# +
val_data = np.random.random((100, 32))
val_labels = np.random.random((100, 10))

model.fit(data, labels, epochs=10, batch_size=32,
          validation_data=(val_data, val_labels))
# -

# ### Input using `tf.data`

# +
# Instantiates a toy dataset instance:
dataset = tf.data.Dataset.from_tensor_slices((data, labels))
dataset = dataset.batch(32)
dataset = dataset.repeat()

# Don't forget to specify `steps_per_epoch` when calling `fit` on a dataset.
model.fit(dataset, epochs=10, steps_per_epoch=31)
# -

# #### validation data

# +
dataset = tf.data.Dataset.from_tensor_slices((data, labels))
dataset = dataset.batch(32).repeat()

val_dataset = tf.data.Dataset.from_tensor_slices((val_data, val_labels))
val_dataset = val_dataset.batch(32).repeat()

model.fit(dataset, epochs=10, steps_per_epoch=30,
          validation_data=val_dataset,
          validation_steps=3)
# -

# ### Evaluate and predict

# +
data = np.random.random((1000, 32))
labels = np.random.random((1000, 10))

model.evaluate(data, labels, batch_size=32)

model.evaluate(dataset, steps=30)
# -

# ## Build advanced model
#
# ### Functional API
#
# * `tf.keras.Sequential` cannot represent arbitrary models.
#   * Multi-input models
#   * Multi-output models
#   * Models with shared layers (the same layer called several times)
#   * Models with non-sequential data flows (e.g. residual connections)
#   
# #### Building a model with the functional API works like this:
#
# 1. A layer instance is callable and returns a tensor.
# 2. Input tensors and output tensors are used to define a tf.keras.Model instance.
# 3. This model is trained just like the Sequential model.

# +
inputs = tf.keras.Input(shape=(32,))  # Returns a placeholder tensor

# A layer instance is callable on a tensor, and returns a tensor.
x = layers.Dense(64, activation='relu')(inputs)
x = layers.Dense(64, activation='relu')(x)
predictions = layers.Dense(10, activation='softmax')(x)

# +
model = tf.keras.Model(inputs=inputs, outputs=predictions)

# The compile step specifies the training configuration.
model.compile(optimizer=tf.train.RMSPropOptimizer(0.001),
              loss='categorical_crossentropy',
              metrics=['accuracy'])

# Trains for 5 epochs
model.fit(data, labels, batch_size=32, epochs=5)


# -

# ### Build a fully-customizable model
#
# * Subclassing `tf.keras.Model`
# * define `__init__`
#   * Create layers and set them as attributes of the class instance
# * define `call`
#   * Define the forward pass
#   
# Simple class form
# ```python
# class MyModel(tf.keras.Model):
#   def __init__(self):
#     super(MyModel, self).__init__(name='my_model')
#     # Define your layers here.
#     pass
#
#   def call(self, inputs):
#     # Define your forward pass here,
#     # using layers you previously defined (in `__init__`).
#     pass```

class MyModel(tf.keras.Model):

  def __init__(self, num_classes=10):
    super(MyModel, self).__init__(name='my_model')
    self.num_classes = num_classes
    # Define your layers here.
    self.dense_1 = layers.Dense(32, activation='relu')
    self.dense_2 = layers.Dense(num_classes, activation='sigmoid')

  def call(self, inputs):
    # Define your forward pass here,
    # using layers you previously defined (in `__init__`).
    x = self.dense_1(inputs)
    return self.dense_2(x)

  def compute_output_shape(self, input_shape):
    # You need to override this function if you want to use the subclassed model
    # as part of a functional-style model.
    # Otherwise, this method is optional.
    shape = tf.TensorShape(input_shape).as_list()
    shape[-1] = self.num_classes
    return tf.TensorShape(shape)


# +
model = MyModel(num_classes=10)

# The compile step specifies the training configuration.
model.compile(optimizer=tf.train.RMSPropOptimizer(0.001),
              loss='categorical_crossentropy',
              metrics=['accuracy'])

# Trains for 5 epochs.
model.fit(data, labels, batch_size=32, epochs=5)


# -

# ### Custom layers
#
# Create a custom layer by subclassing tf.keras.layers.Layer and implementing the following methods:
#
# * `build`: Create the weights of the layer. Add weights with the `add_weight` method.
# * `call`: Define the forward pass.
# * `compute_output_shape`: Specify how to compute the output shape of the layer given the input shape.
# * Optionally, a layer can be serialized by implementing the `get_config` method and the `from_config` class method.

class MyLayer(layers.Layer):

  def __init__(self, output_dim, **kwargs):
    self.output_dim = output_dim
    super(MyLayer, self).__init__(**kwargs)

  def build(self, input_shape):
    shape = tf.TensorShape((input_shape[1], self.output_dim))
    # Create a trainable weight variable for this layer.
    self.kernel = self.add_weight(name='kernel',
                                  shape=shape,
                                  initializer='uniform',
                                  trainable=True)
    # Be sure to call this at the end
    super(MyLayer, self).build(input_shape)

  def call(self, inputs):
    return tf.matmul(inputs, self.kernel)

  def compute_output_shape(self, input_shape):
    shape = tf.TensorShape(input_shape).as_list()
    shape[-1] = self.output_dim
    return tf.TensorShape(shape)

  def get_config(self):
    base_config = super(MyLayer, self).get_config()
    base_config['output_dim'] = self.output_dim
    return base_config

  @classmethod
  def from_config(cls, config):
    return cls(**config)


# +
model = tf.keras.Sequential([
    MyLayer(10),
    layers.Activation('softmax')])

# The compile step specifies the training configuration
model.compile(optimizer=tf.train.RMSPropOptimizer(0.001),
              loss='categorical_crossentropy',
              metrics=['accuracy'])

# Trains for 5 epochs.
model.fit(data, labels, batch_size=32, epochs=5)
# -

# ## Callbacks
#
# A callback is an object passed to a model to customize and extend its behavior during training. You can write your own custom callback, or use the built-in tf.keras.callbacks that include:
#
# * `tf.keras.callbacks.ModelCheckpoint`: Save checkpoints of your model at regular intervals.
# * `tf.keras.callbacks.LearningRateScheduler`: Dynamically change the learning rate.
# * `tf.keras.callbacks.EarlyStopping`: Interrupt training when validation performance has stopped improving.
# * `tf.keras.callbacks.TensorBoard`: Monitor the model's behavior using TensorBoard.

callbacks = [
  # Interrupt training if `val_loss` stops improving for over 2 epochs
  tf.keras.callbacks.EarlyStopping(patience=2, monitor='val_loss'),
  # Write TensorBoard logs to `./logs` directory
  tf.keras.callbacks.TensorBoard(log_dir='./logs')
]
model.fit(data, labels, batch_size=32, epochs=5, callbacks=callbacks,
          validation_data=(val_data, val_labels))

# ## Save and restore
#
# ### Weights only
#
# * Save and load the weights of a model using `tf.keras.Model.save_weights`:

# +
model = tf.keras.Sequential([
layers.Dense(64, activation='relu'),
layers.Dense(10, activation='softmax')])

model.compile(optimizer=tf.train.AdamOptimizer(0.001),
              loss='categorical_crossentropy',
              metrics=['accuracy'])

# +
# Save weights to a TensorFlow Checkpoint file
model.save_weights('./weights/my_model')

# Restore the model's state,
# this requires a model with the same architecture.
model.load_weights('./weights/my_model')
# -

# ### Configuration only

# Serialize a model to JSON format
json_string = model.to_json()
json_string

import json
import pprint
pprint.pprint(json.loads(json_string))

# ### Entire model

# +
# Create a trivial model
#model = tf.keras.Sequential([
#  layers.Dense(10, activation='relu', input_shape=(32,)),
#  layers.Dense(10, activation='softmax')
#])
#model.compile(optimizer='rmsprop',
#              loss='categorical_crossentropy',
#              metrics=['accuracy'])
#model.fit(data, labels, batch_size=32, epochs=5)

# Save entire model to a HDF5 file
#model.save('my_model')

# Recreate the exact same model, including weights and optimizer.
#model = tf.keras.models.load_model('my_model')

StarcoderdataJupyterScriptsDedupFiltered