text: # coding: utf-8
  
  
  import matplotlib.pyplot as plt
  import tensorflow as tf
  import numpy as np
  import tensorflow as tf
  
  # *Python Machine Learning 2nd Edition* by [Sebastian Raschka](https://sebastianraschka.com) and Vahid Mirjalili, Packt Publishing Ltd. 2017
  # 
  # Code Repository: https://github.com/rasbt/python-machine-learning-book-2nd-edition
  # 
  # Code License: [MIT License](https://github.com/rasbt/python-machine-learning-book-2nd-edition/blob/master/LICENSE.txt)
  
  # # Python Machine Learning - Code Examples
  
  # Note that the optional watermark extension is a small IPython notebook plugin that I developed to make the code reproducible. You can just skip the following line(s).
  
  
  
  
  
  # *The use of `watermark` is optional. You can install this IPython extension via "`pip install watermark`". For more information, please see: https://github.com/rasbt/watermark.*
  
  # # Chapter 14 - Going Deeper: The Mechanics of TensorFlow
  
  # - [TensorFlow in a nutshell](#TensorFlow-in-a-nutshell)
  # - [Understanding TensorFlow's computation graphs](#Understanding-TensorFlow's-computation-graphs)
  # - [Working with TensorFlow’s placeholders, variables, and operations](#Working-with-TensorFlow’s-placeholders,-variables,-and-operations)
  #   - [Using placeholders for feeding data as input to a model in TensorFlow](#Using-placeholders-for-feeding-data-as-input-to-a-model-in-TensorFlow)
  #     - [Defining placeholders](#Defining-placeholders)
  #     - [Feeding placeholders with data](#Feeding-placeholders-with-data)
  #     - [Defining placeholders for data arrays with varying batchsizes](#Defining-placeholders-for-data-arrays-with-varying-batchsizes)
  #   - [Variables in TensorFlow](#Variables-in-TensorFlow)
  #     - [Defining variables](#Defining-variables)
  #     - [Initializing variables](#Initializing-variables)
  #     - [Variable scope](#Variable-scope)
  #     - [Reusing variables](#Reusing-variables)
  #   - [Building a regression model](#Building-a-regression-model)
  #   - [Executing objects in a TensorFlow graph using their names](#Executing-objects-in-a-TensorFlow-graph-using-their-names)
  # - [Saving and restoring a model in TensorFlow](#Saving-and-restoring-a-model-in-TensorFlow)
  # - [Transforming Tensors as multidimensional data arrays](#Transforming-Tensors-as-multidimensional-data-arrays)
  # - [Utilizing control flow mechanics in building graphs](#Utilizing-control-flow-mechanics-in-building-graphs)
  # - [Visualizing the graph with TensorBoard](#Visualizing-the-graph-with-TensorBoard)
  # - [Summary](#Summary)
  
  
  
  
  
  
  
  
  
  # ## TensorFlow in a nutshell
  
  
  
  
  
  # **How to get the rank and shape of a tensor**
  
  
  
  g = tf.Graph()
  
  ## define the computation graph
  with g.as_default():
      ## define tensors t1, t2, t3:
      t1 = tf.constant(np.pi)
      t2 = tf.constant([1, 2, 3, 4])
      t3 = tf.constant([[1, 2], [3, 4]])
  
      ## get their ranks
      r1 = tf.rank(t1)
      r2 = tf.rank(t2)
      r3 = tf.rank(t3)
  
      ## get their shapes
      s1 = t1.get_shape()
      s2 = t2.get_shape()
      s3 = t3.get_shape()
      print('Shapes:', s1, s2, s3)
  
  with tf.Session(graph=g) as sess:
      print('Ranks:', 
            r1.eval(), 
            r2.eval(), 
            r3.eval())
  
  
  # ## Understanding TensorFlow's computation graph
  
  
  
  Image("images/14_02.png")
  
  
  
  
  g = tf.Graph()
   
  ## add nodes to the graph
  with g.as_default():
      a = tf.constant(1, name='a')
      b = tf.constant(2, name='b') 
      c = tf.constant(3, name='c') 
  
      z = 2*(a-b) + c
      
  ## launch the graph
  with tf.Session(graph=g) as sess:
      print('2*(a-b)+c => ', sess.run(z))
  
  
  # ## Working with TensorFlow’s placeholders, variables, and operations
  
  # ### Using placeholders for feeding data as input to a model in TensorFlow
  
  # #### Defining placeholders
  
  
  
  g = tf.Graph()
  with g.as_default():
      tf_a = tf.placeholder(tf.int32, shape=[],
                            name='tf_a')
      tf_b = tf.placeholder(tf.int32, shape=[],
                            name='tf_b') 
      tf_c = tf.placeholder(tf.int32, shape=[],
                            name='tf_c') 
  
      r1 = tf_a-tf_b
      r2 = 2*r1
      z  = r2 + tf_c
  
  
  # #### Feeding placeholders with data
  
  
  
  ## launch the previous graph
  with tf.Session(graph=g) as sess:
      feed = {tf_a: 1,
              tf_b: 2,
              tf_c: 3}
      print('z:', 
            sess.run(z, feed_dict=feed))
  
  
  # Execution with and without feeding tf_c:
  
  
  
  ## launch the previous graph
  with tf.Session(graph=g) as sess:
      ## execution without feeding tf_c
      feed = {tf_a: 1,
              tf_b: 2}
      print('r1:', 
            sess.run(r1, feed_dict=feed))
      print('r2:', 
            sess.run(r2, feed_dict=feed))
      
      ## execution with feeding tf_c
      feed = {tf_a: 1,
              tf_b: 2,
              tf_c: 3}
      print('r1:', 
            sess.run(r1, feed_dict=feed))
      print('r2:', 
            sess.run(r2, feed_dict=feed))
  
  
  # ### Defining placeholders for data arrays with varying batchsizes
  
  # Placeholder for varying batchsizes:
  
  
  
  g = tf.Graph()
  
  with g.as_default():
      tf_x = tf.placeholder(tf.float32, 
                            shape=[None, 2],
                            name='tf_x')
      
      x_mean = tf.reduce_mean(tf_x, 
                            axis=0, 
                            name='mean')
  
  
  np.random.seed(123)
  np.set_printoptions(precision=2)
  
  with tf.Session(graph=g) as sess:
      x1 = np.random.uniform(low=0, high=1, 
                             size=(5,2))
      print('Feeding data with shape', x1.shape)
      print('Result:', sess.run(x_mean, 
                               feed_dict={tf_x:x1}))
      x2 = np.random.uniform(low=0, high=1, 
                             size=(10,2))
      print('Feeding data with shape', x2.shape)
      print('Result:', sess.run(x_mean, 
                               feed_dict={tf_x:x2}))
  
  
  
  
  print(tf_x)
  
  
  # ### Variables in TensorFlow
  # 
  
  # #### Defining Variables
  
  
  
  g1 = tf.Graph()
  
  with g1.as_default():
      w = tf.Variable(np.array([[1, 2, 3, 4],
                                [5, 6, 7, 8]]), name='w')
      print(w)
  
  
  # #### Initializing variables
  
  
  
  ## initialize w and evaluate it
  with tf.Session(graph=g1) as sess:
      sess.run(tf.global_variables_initializer())
      print(sess.run(w))
  
  
  
  
  ## add the init_op to the graph
  with g1.as_default():
      init_op = tf.global_variables_initializer()
      
  ## initialize w with init_op and evaluate it
  with tf.Session(graph=g1) as sess:
      sess.run(init_op)
      print(sess.run(w))
  
  
  
  
  g2 = tf.Graph()
  
  with g2.as_default():
      w1 = tf.Variable(1, name='w1')
      init_op = tf.global_variables_initializer()
      w2 = tf.Variable(2, name='w2')
  
  
  
  
  with tf.Session(graph=g2) as sess:
      sess.run(init_op)
      print('w1:', sess.run(w1))
  
  
  # Error if a variable is not initialized:
  
  
  
  with tf.Session(graph=g2) as sess:
      
      try:
          sess.run(init_op)
          print('w2:', sess.run(w2))
      except tf.errors.FailedPreconditionError as e:
          print(e)
  
  
  # #### Variable scope
  
  
  
  g = tf.Graph()
  
  with g.as_default():
      with tf.variable_scope('net_A'):
          with tf.variable_scope('layer-1'):
              w1 = tf.Variable(tf.random_normal(
                  shape=(10,4)), name='weights')
          with tf.variable_scope('layer-2'):
              w2 = tf.Variable(tf.random_normal(
                  shape=(20,10)), name='weights')
      with tf.variable_scope('net_B'):
          with tf.variable_scope('layer-1'):
              w3 = tf.Variable(tf.random_normal(
                  shape=(10,4)), name='weights')
  
      print(w1)
      print(w2)
      print(w3)
  
  
  # #### Reusing variables
  
  
  
  ######################
  ## Helper functions ##
  ######################
  
  def build_classifier(data, labels, n_classes=2):
      data_shape = data.get_shape().as_list()
      weights = tf.get_variable(name='weights',
                                shape=(data_shape[1], n_classes),
                                dtype=tf.float32)
      bias = tf.get_variable(name='bias', 
                             initializer=tf.zeros(shape=n_classes))
      print(weights)
      print(bias)
      logits = tf.add(tf.matmul(data, weights), 
                      bias, 
                      name='logits')
      print(logits)
      return logits, tf.nn.softmax(logits)
  
  def build_generator(data, n_hidden):
      data_shape = data.get_shape().as_list()
      w1 = tf.Variable(
          tf.random_normal(shape=(data_shape[1], 
                                  n_hidden)),
          name='w1')
      b1 = tf.Variable(tf.zeros(shape=n_hidden),
                       name='b1')
      hidden = tf.add(tf.matmul(data, w1), b1, 
                      name='hidden_pre-activation')
      hidden = tf.nn.relu(hidden, 'hidden_activation')
          
      w2 = tf.Variable(
          tf.random_normal(shape=(n_hidden, 
                                  data_shape[1])),
          name='w2')
      b2 = tf.Variable(tf.zeros(shape=data_shape[1]),
                       name='b2')
      output = tf.add(tf.matmul(hidden, w2), b2, 
                      name = 'output')
      return output, tf.nn.sigmoid(output)
      
  ########################
  ## Defining the graph ##
  ########################
  
  batch_size=64
  g = tf.Graph()
  
  with g.as_default():
      tf_X = tf.placeholder(shape=(batch_size, 100), 
                            dtype=tf.float32,
                            name='tf_X')
      ## build the generator
      with tf.variable_scope('generator'):
          gen_out1 = build_generator(data=tf_X, 
                                     n_hidden=50)
      
      ## build the classifier
      with tf.variable_scope('classifier') as scope:
          ## classifier for the original data:
          cls_out1 = build_classifier(data=tf_X, 
                                      labels=tf.ones(
                                          shape=batch_size))
          
          ## reuse the classifier for generated data
          scope.reuse_variables()
          cls_out2 = build_classifier(data=gen_out1[1],
                                      labels=tf.zeros(
                                          shape=batch_size))
          
          init_op = tf.global_variables_initializer()
  
  
  
  
  ## alternative way
  
  g = tf.Graph()
  
  with g.as_default():
      tf_X = tf.placeholder(shape=(batch_size, 100), 
                            dtype=tf.float32,
                            name='tf_X')
      ## build the generator
      with tf.variable_scope('generator'):
          gen_out1 = build_generator(data=tf_X, 
                                     n_hidden=50)
      
      ## build the classifier
      with tf.variable_scope('classifier'):
          ## classifier for the original data:
          cls_out1 = build_classifier(data=tf_X, 
                                      labels=tf.ones(
                                          shape=batch_size))
          
      with tf.variable_scope('classifier', reuse=True):
          ## reuse the classifier for generated data
          cls_out2 = build_classifier(data=gen_out1[1],
                                      labels=tf.zeros(
                                          shape=batch_size))
          
          init_op = tf.global_variables_initializer()
  
  
  # ### Building a regression model
  
  
  
  ## define a graph
  g = tf.Graph()
  
  ## define the computation graph
  with g.as_default():
      ## placeholders
      tf.set_random_seed(123)
      tf_x = tf.placeholder(shape=(None), 
                            dtype=tf.float32, 
                            name='tf_x')
      tf_y = tf.placeholder(shape=(None), 
                            dtype=tf.float32,
                            name='tf_y')
      
      ## define the variable (model parameters)
      weight = tf.Variable(
          tf.random_normal(
              shape=(1, 1), 
              stddev=0.25),
          name = 'weight')
      bias = tf.Variable(0.0, name='bias')
      
      ## build the model
      y_hat = tf.add(weight * tf_x, bias, 
                     name='y_hat')
      print(y_hat)
      
      ## compute the cost
      cost = tf.reduce_mean(tf.square(tf_y - y_hat), 
                            name='cost')
      print(cost)
      
      ## train
      optim = tf.train.GradientDescentOptimizer(
          learning_rate=0.001)
      train_op = optim.minimize(cost, name='train_op')
  
  
  
  
  ## create a random toy dataset for regression
  
  
  np.random.seed(0)
  
  def make_random_data():
      x = np.random.uniform(low=-2, high=4, size=200)
      y = []
      for t in x:
          r = np.random.normal(loc=0.0, 
                               scale=(0.5 + t*t/3), 
                               size=None)
          y.append(r)
      return  x, 1.726*x -0.84 + np.array(y)
  
  
  x, y = make_random_data() 
  
  plt.plot(x, y, 'o')
  # plt.savefig('images/14_03.png', dpi=300)
  plt.show()
  
  
  # ### Executing objects in a TensorFlow graph using their names
  
  
  
  ## train/test splits:
  x_train, y_train = x[:100], y[:100]
  x_test, y_test = x[100:], y[100:]
  
  
  ## training the model
  n_epochs = 500
  training_costs = []
  with tf.Session(graph=g) as sess:
      ## first, run the variables initializer
      sess.run(tf.global_variables_initializer())
      
      ## train the model for n_epochs
      for e in range(n_epochs):
          c, _ = sess.run([cost, train_op], 
                          feed_dict={tf_x: x_train,
                                     tf_y: y_train})
          training_costs.append(c)
          if not e % 50:
              print('Epoch %4d: %.4f' % (e, c))
              
  
  plt.plot(training_costs)
  plt.savefig('images/14_04.png', dpi=300)
  
  
  # Executing with variable names:
  
  
  
  ## train/test splits
  x_train, y_train = x[:100], y[:100]
  x_test, y_test = x[100:], y[100:]
  
  ## plot trainng data
  plt.plot(x_train, y_train, 'o')
  plt.show()
  
  ## training the model
  n_epochs = 500
  training_costs = []
  with tf.Session(graph=g) as sess:
      ## first, run the variables initializer
      sess.run(tf.global_variables_initializer())
      
      ## train the model for n_eopchs
      for e in range(n_epochs):
          c, _ = sess.run(['cost:0', 'train_op'], 
                          feed_dict={'tf_x:0': x_train,
                                     'tf_y:0': y_train})
          training_costs.append(c)
          if not e % 50:
              print('Epoch %4d: %.4f' % (e, c))
  
  
  # ## Saving and restoring a model in TensorFlow
  
  
  
  ## add saver to the graph
  with g.as_default():
      saver = tf.train.Saver()
      
  ## training the model
  n_epochs = 500
  training_costs = []
  
  with tf.Session(graph=g) as sess:
      ## first, run the variables initializer
      sess.run(tf.global_variables_initializer())
      
      ## train the model for n_epochs
      for e in range(n_epochs):
          c, _ = sess.run(['cost:0', 'train_op'], 
                          feed_dict={'tf_x:0':x_train,
                                     'tf_y:0':y_train})
          training_costs.append(c)
          if not e % 50:
              print('Epoch %4d: %.4f' % (e, c))
              
      saver.save(sess, './trained-model')
  
  
  # Restoring the saved model:
  
  
  
  ## new file: loading a trained model
  ## and run the model on test set
  
  
  g2 = tf.Graph()
  with tf.Session(graph=g2) as sess:
      new_saver = tf.train.import_meta_graph(
          './trained-model.meta')
      new_saver.restore(sess, './trained-model')
      
      y_pred = sess.run('y_hat:0', 
                        feed_dict={'tf_x:0' : x_test})
  
  
  
  
  print('SSE: %.4f' % (np.sum(np.square(y_pred - y_test))))
  
  
  
  
  x_arr = np.arange(-2, 4, 0.1)
  
  g2 = tf.Graph()
  with tf.Session(graph=g2) as sess:
      new_saver = tf.train.import_meta_graph(
          './trained-model.meta')
      new_saver.restore(sess, './trained-model')
      
      y_arr = sess.run('y_hat:0', 
                        feed_dict={'tf_x:0' : x_arr})
  
  plt.figure()
  plt.plot(x_train, y_train, 'bo')
  plt.plot(x_test, y_test, 'bo', alpha=0.3)
  plt.plot(x_arr, y_arr.T[:, 0], '-r', lw=3)
  plt.savefig('images/14_05.png', dpi=400)
  plt.show()
  
  
  # ## Transforming Tensors as multidimensional data arrays
  
  
  
  g = tf.Graph()
  with g.as_default():
      arr = np.array([[1., 2., 3., 3.5],
                      [4., 5., 6., 6.5],
                      [7., 8., 9., 9.5]])
      T1 = tf.constant(arr, name='T1')
      print(T1)
      s = T1.get_shape()
      print('Shape of T1 is', s)
      T2 = tf.Variable(tf.random_normal(
          shape=s))
      print(T2)
      T3 = tf.Variable(tf.random_normal(
          shape=(s.as_list()[0],)))
      print(T3)
  
  
  
  
  with g.as_default():
      T4 = tf.reshape(T1, shape=[1, 1, -1], 
                      name='T4')
      print(T4)
      T5 = tf.reshape(T1, shape=[1, 3, -1], 
                      name='T5')
      print(T5)
  
  
  
  
  with tf.Session(graph = g) as sess:
      print(sess.run(T4)) 
      print()   
      print(sess.run(T5))
  
  
  
  
  with g.as_default():
      T6 = tf.transpose(T5, perm=[2, 1, 0], 
                       name='T6')
      print(T6)
      T7 = tf.transpose(T5, perm=[0, 2, 1], 
                       name='T7')
      print(T7)
  
  
  
  
  with g.as_default():
      t5_splt = tf.split(T5, 
                         num_or_size_splits=2, 
                         axis=2, name='T8')
      print(t5_splt)
  
  
  
  
  g = tf.Graph()
  with g.as_default():
      t1 = tf.ones(shape=(10, 1), 
                   dtype=tf.float32, name='t1')
      t2 = tf.zeros(shape=(10, 1),
                   dtype=tf.float32, name='t2')
      print(t1)
      print(t2)
      
  with g.as_default():
      t3 = tf.concat([t1, t2], axis=0, name='t3')
      print(t3)
      t4 = tf.concat([t1, t2], axis=1, name='t4')
      print(t4)
  
  
  
  
  with tf.Session(graph = g) as sess:
      print(t3.eval())
      print()
      print(t4.eval())
  
  
  # ## Utilizing control flow mechanics in building graphs
  
  
  
  ## Python control flow
  
  
  x, y = 1.0, 2.0
  
  g = tf.Graph()
  with g.as_default():
      tf_x = tf.placeholder(dtype=tf.float32, 
                             shape=None, name='tf_x')
      tf_y = tf.placeholder(dtype=tf.float32, 
                             shape=None, name='tf_y')
      if x < y:
          res = tf.add(tf_x, tf_y, name='result_add')
      else:
          res = tf.subtract(tf_x, tf_y, name='result_sub')
          
      print('Object: ', res)
          
  with tf.Session(graph=g) as sess:
      print('x < y: %s -> Result:' % (x < y), 
            res.eval(feed_dict={'tf_x:0': x, 
                                'tf_y:0': y}))
      x, y = 2.0, 1.0
      print('x < y: %s -> Result:' % (x < y), 
            res.eval(feed_dict={'tf_x:0': x,
                                'tf_y:0': y}))  
      
      ## uncomment the next line if you want to visualize the graph in TensorBoard:
      file_writer = tf.summary.FileWriter(logdir='./logs/py-cflow/', graph=g)
  
  
  
  
  ## TensorFlow control flow
  
  
  x, y = 1.0, 2.0
  
  g = tf.Graph()
  with g.as_default():
      tf_x = tf.placeholder(dtype=tf.float32, 
                             shape=None, name='tf_x')
      tf_y = tf.placeholder(dtype=tf.float32, 
                             shape=None, name='tf_y')
      res = tf.cond(tf_x < tf_y, 
                    lambda: tf.add(tf_x, tf_y, 
                                   name='result_add'),
                    lambda: tf.subtract(tf_x, tf_y, 
                                   name='result_sub'))
      print('Object:', res)
          
  with tf.Session(graph=g) as sess:
      print('x < y: %s -> Result:' % (x < y), 
            res.eval(feed_dict={'tf_x:0': x, 
                                'tf_y:0': y}))
      x, y = 2.0, 1.0
      print('x < y: %s -> Result:' % (x < y), 
            res.eval(feed_dict={'tf_x:0': x,
                                'tf_y:0': y}))  
  
      #file_writer = tf.summary.FileWriter(logdir='./logs/tf-cond/', graph=g)
  
  
  
  
  
  
  # ## Visualizing the graph with TensorBoard
  
  
  
  
  
  ###########################
  ##   Helper functions    ##
  ###########################
  
  def build_classifier(data, labels, n_classes=2):
      data_shape = data.get_shape().as_list()
      weights = tf.get_variable(name = 'weights',
                                shape=(data_shape[1],
                                       n_classes),
                                dtype=tf.float32)
      bias = tf.get_variable(name='bias', 
                             initializer=tf.zeros(
                                       shape=n_classes))
      print(weights)
      print(bias)
      logits = tf.add(tf.matmul(data, weights), 
                      bias, 
                      name='logits')
      print(logits)
      return logits, tf.nn.softmax(logits)
  
  
  def build_generator(data, n_hidden):
      data_shape = data.get_shape().as_list()
      w1 = tf.Variable(
          tf.random_normal(shape=(data_shape[1], 
                                  n_hidden)),
          name='w1')
      b1 = tf.Variable(tf.zeros(shape=n_hidden),
                       name='b1')
      hidden = tf.add(tf.matmul(data, w1), b1, 
                      name='hidden_pre-activation')
      hidden = tf.nn.relu(hidden, 'hidden_activation')
          
      w2 = tf.Variable(
          tf.random_normal(shape=(n_hidden, 
                                  data_shape[1])),
          name='w2')
      b2 = tf.Variable(tf.zeros(shape=data_shape[1]),
                       name='b2')
      output = tf.add(tf.matmul(hidden, w2), b2, 
                      name = 'output')
      return output, tf.nn.sigmoid(output)
  
  
  ###########################
  ##  Building the graph   ##
  ###########################
  
  batch_size=64
  g = tf.Graph()
  
  with g.as_default():
      tf_X = tf.placeholder(shape=(batch_size, 100), 
                            dtype=tf.float32,
                            name='tf_X')
      
      ## build the generator
      with tf.variable_scope('generator'):
          gen_out1 = build_generator(data=tf_X, 
                                     n_hidden=50)
      
      ## build the classifier
      with tf.variable_scope('classifier') as scope:
          ## classifier for the original data:
          cls_out1 = build_classifier(data=tf_X, 
                                      labels=tf.ones(
                                          shape=batch_size))
          
          ## reuse the classifier for generated data
          scope.reuse_variables()
          cls_out2 = build_classifier(data=gen_out1[1],
                                      labels=tf.zeros(
                                          shape=batch_size))
  
          
  with tf.Session(graph = g) as sess:
      sess.run(tf.global_variables_initializer())
      
      file_writer = tf.summary.FileWriter(logdir='logs/', graph=g)
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  # ## Summary
  
  # ...
  
  # ---
  # 
  # Readers may ignore the next cell.
  
  
  
  
  