# Restore model.
sess = tf.Session()    
#First let's load meta graph and restore weights
saver = tf.train.import_meta_graph('tf_mnist_model_layers/tf_mnist_model.ckpt.meta')
saver.restore(sess,tf.train.latest_checkpoint('tf_mnist_model_layers/'))

"""
Explanation: Loading the model.
End of explanation
"""


dataset = mnist.load_data()
train_data = dataset[0][0] / 255
train_data = train_data[..., np.newaxis].astype('float32')
train_labels = np_utils.to_categorical(dataset[0][1]).astype('float32')
test_data = dataset[1][0] / 255
test_data = test_data[..., np.newaxis].astype('float32')
test_labels = np_utils.to_categorical(dataset[1][1]).astype('float32')
plt.imshow(train_data[0, ..., 0])

# Get the first input tensor.
network_input = sess.graph.get_operations()[0].outputs[0]
network_input

"""
Explanation: Loading the dataset
End of explanation
"""


sess.graph.get_operations()

"""
Explanation: The TensorFlow graph
Look at all operations to find out the ones that are interesting for weights/activations.
End of explanation
"""


conv_op = sess.graph.get_operation_by_name('conv2d_1/convolution')
print('inputs: ', list(conv_op.inputs))
print('outputs: ', list(conv_op.outputs))

weight_tensor = conv_op.inputs[1]
weight_tensor

# Producer ops for the tensor.
weight_tensor.op

# Consumer ops for the tensor.
weight_tensor.consumers()

"""
Explanation: TensorFlow exposes a graph API in which nodes correspond to operations and directed edges correspond to to tensors flowing from one operation to the next. Operations know their respective input and output tensors. Tensors know by which operations they get produced and for which operations they serve as input. c.f. https://www.tensorflow.org/extend/tool_developers/
End of explanation
"""


weight_read_op = sess.graph.get_operation_by_name('conv2d_1/kernel/read')
weight_read_op


conv_op = sess.graph.get_operation_by_name('conv2d_1/convolution')
conv_op

weights_1 = sess.run(weight_read_op.outputs[0])
weights_2 = sess.run(conv_op.inputs[-1])
np.all(weights_1 == weights_2)

"""
Explanation: Getting the weights
For getting the weights we need to run the output tensor of operations that read the weights or the seconde input tensor of the actual MatMul/Conv operations.
End of explanation
"""


print(weights_1.shape)
for i in range(32):
    plt.imshow(weights_1[..., 0, i])
    plt.show()

weights_1[..., 0, 0]

"""
Explanation: Looking at the actual kernels.
End of explanation
"""


bias_add_op = sess.graph.get_operation_by_name('conv2d_1/BiasAdd')
bias_add_op

bias = sess.run(bias_add_op.inputs[-1])
bias

"""
Explanation: The same goes for the bias.
End of explanation
"""


activation_op = sess.graph.get_operation_by_name('conv2d_1/Relu')
activation_op.outputs[0]

activations = sess.run(activation_op.outputs[0], feed_dict={network_input: test_data[0:1]})
activations.shape

"""
Explanation: Getting the activations and net inputs
Activations and net inputs can be found as the output tensors of the respective operations.
Slices of the example code here are used to unit test the network package.
End of explanation
"""


for i in range(32):
    plt.imshow(activations[0, ..., i])
    plt.show()

activations[0, 5:10, 5:10, 0]

net_input_op = sess.graph.get_operation_by_name('conv2d_1/BiasAdd')
net_input_op.outputs[0]

net_inputs = sess.run(net_input_op.outputs[0], feed_dict={network_input: test_data[0:1]})
net_inputs.shape

for i in range(32):
    plt.imshow(net_inputs[0, ..., i])
    plt.show()

net_inputs[0, 5:10, 5:10, 0]

"""
Explanation: Look at the activations
End of explanation
"""


conv_op.inputs[0].shape

conv_op.inputs[0].shape.as_list()

"""
Explanation: Getting input and output shape
For each tensor the shape can be retrieved as a TensorShape object or as a list.
End of explanation
"""


sess.graph_def

activation_op.node_def

"""
Explanation: Getting layer properties
Internally TensorFlow represents graph as protobufs that only define nodes with their inputs. This definition can either be retrieved as globally graph.graph_def as as op.node_def for every single operation.
End of explanation
"""


read_op = sess.graph.get_operation_by_name('conv2d_2/kernel/read')
read_op.outputs[0].shape.as_list()[:2]

"""
Explanation: Kernel size
Kernel size is not defined in the protobuf, but has to be read from the weight tensor shape.
End of explanation
"""


conv_op.node_def

# The padding value has to be decoded from bytes into a string.
conv_op.node_def.attr['padding'].s.decode('utf8')

"""
Explanation: Padding
End of explanation
"""


strides = conv_op.node_def.attr['strides']
strides.list.i[1], strides.list.i[2]

"""
Explanation: Strides
End of explanation
"""


max_pool_op = sess.graph.get_operation_by_name('max_pooling2d_1/MaxPool')
max_pool_op.node_def

kernel = max_pool_op.node_def.attr['ksize']
kernel.list.i[1], kernel.list.i[2]

"""
Explanation: Pool size
Pool size is defined as ksize in the maxpool definition.
End of explanation
"""


