text: # -*- coding: utf-8 -*-
  """tf_2_intro.ipynb
  
  Automatically generated by Colaboratory.
  
  Original file is located at
      https://colab.research.google.com/drive/1CQe2Hs9B5Iujn0bKLwSWpaPjgexcs2Cf
  
  # Tensorflow 2.0
  
  ## Changes in Tensorflow 2.0
  
  - There are no more global variables. In 1.x, the graph is global; it doesn't matter if a variable has been defined inside a Python function—it is visible and separate from every other part of the graph.
  - No more tf.variable_scope. A context manager can't change the behavior of a function by setting a boolean flag (reuse). In TensorFlow 2.0, variable sharing is made by the model itself. 
  - Every model is a Python object, every object has its own set of variables, and to share the variables, you just have to use the same model with different input.
  - No more tf.get_variable. As we saw in Chapter 3, TensorFlow Graph Architecture, tf.get_variable allows you to declare variables that can be shared by using tf.variable_scope. Since every variable now matches 1:1 with a Python variable, the possibility of declaring global variables has been removed.
  - No more tf.layers. Every layer that's declared inside the tf.layers module uses tf.get_variable to define its own variables. Use tf.keras.layers instead.
  - No more global collections. Every variable was added to a collection of global variables that were accessible via tf.trainable_variables()—this was contradictory to every good software design principle. 
  - Now, the only way to access the variables of an object is by accessing its trainable_variables attribute, which returns the list of the trainable variables of that specific object.
  - There's no need to manually call an operation that initializes all the variables.
  - API cleanup and the removal of tf.contrib is now used in favor of the creation of several small and well-organized projects.
  """
  
  !pip install tensorflow==2.0.0
  
  import tensorflow as tf
  
  print(tf.__version__)
  
  """## tf.Keras Sequential API"""
  
  from tensorflow.keras.datasets import fashion_mnist
  import numpy as np
  
  n_classes = 10
  model = tf.keras.Sequential()
  model.add(tf.keras.layers.Conv2D(32, (5,5), activation=tf.nn.relu, input_shape=(28,28,1)))
  model.add(tf.keras.layers.MaxPool2D((2,2), (2,2))),
  model.add(tf.keras.layers.Conv2D(64, (3,3), activation=tf.nn.relu))
  model.add(tf.keras.layers.MaxPool2D((2,2)))
  model.add(tf.keras.layers.Flatten())
  model.add(tf.keras.layers.Dense(128, activation=tf.nn.relu))
  model.add(tf.keras.layers.Dropout(0.3))
  model.add(tf.keras.layers.Dense(n_classes, activation=tf.nn.softmax))
  
  model.summary()
  
  (train_x, train_y), (test_x, test_y) = fashion_mnist.load_data()
  
  train_x = (train_x / 255.) * 2 - 1
  test_x = (test_x/255.) * 2 - 1
  train_x = tf.expand_dims(train_x, -1).numpy()
  test_x = tf.expand_dims(test_x, -1).numpy()
  
  model.compile(optimizer=tf.keras.optimizers.Adam(1e-5), loss='sparse_categorical_crossentropy', metrics=['accuracy'])
  
  history = model.fit(train_x, train_y, epochs = 10, batch_size=128, verbose=1, validation_split=0.2)
  
  acc = model.evaluate(test_x, test_y)[1]
  print(acc)
  
  """# The tf.keras Functional API
  
  - The Functional API allows you to define multi-input, multi-output models, easily sharing layers, defines residual connections, and in general define models with arbitrary complex topologies.
  
  - It knows that it is possible to compose the layers by treating them as functions and building a tf.keras.Model object just by passing the input and output layers.
  """
  
  input_shape = (100,)
  inputs = tf.keras.layers.Input(input_shape)
  net = tf.keras.layers.Dense(64, activation=tf.nn.elu, name="fc1") (inputs)
  net = tf.keras.layers.Dense(64, activation=tf.nn.elu, name="fc2") (net)
  net = tf.keras.layers.Dense(1, activation=tf.nn.elu, name="g") (net)
  model = tf.keras.Model(inputs=inputs, outputs=net)
  
  """- Being a Keras model, model can be compiled and trained exactly like any other Keras model that's defined using the Sequential API.
  
  ## Subclassing Method
  
  - The subclassing method is not recommended since it separates the layer definition from its usage, making it easy to make mistakes while refactoring the code. 
  - However, defining the forward pass using this kind of model definition is sometimes the only way to proceed, especially when working with recurrent neural networks.
  """
  
  import tensorflow as tf
  
  class generator(tf.keras.Model):
      def __init__(self):
          super().__init__()
          self.dense_1 = tf.keras.layers.Dense(64, activation=tf.nn.relu, name="fc1")
          self.dense_2 = tf.keras.layers.Dense(32, activation=tf.nn.relu, name="fc2")
          self.output = tf.keras.layers.Dense(1, name="G")
      
      def call(self, target):
          net = self.dense_1(inputs)
          net = self.dense_2(net)
          net = self.output(net)
          return net
  
  """- Subclassing from a tf.keras.Model the Generator object is a tf.keras.Model itself, and as such, it can be trained using the compile and fit commands, as shown earlier.
  
  - Keras can be used to train and evaluate models, but TensorFlow 2.0, with its eager execution, allows us to write our own custom training loop so that we have complete control of the training process and can debug easily.
  
  # Eager Execution
  
  - TensorFlow's eager execution is an imperative programming environment that evaluates operations immediately, without building graphs: operations return concrete values instead of constructing a computational graph to run later.
  """
  
  # tf 2 code
  A = tf.constant([[1,2], [3,4]], dtype=tf.float32)
  x = tf.constant([[0,10], [0,0.5]], dtype=tf.float32)
  b = tf.constant([[1, -1]], dtype=tf.float32)
  
  y = A@x + b
  print(y)
  
  """- The numerical value is, of course, the same, but the object that's returned is no longer a NumPy array—instead, it's a tf.Tensor object.
  
  ### Functions, not sessions
  
  - The tf.Session object has been removed from the TensorFlow API. By focusing on eager execution, you no longer need the concept of a session because the execution of the operation is immediate—we don't build a computational graph before running the computation.
  
  - The two operations that could be executed singularly by calling sess.run (the matrix multiplication and the sum) have been moved to independent functions.
  """
  
  def multiply(x, y):
      assert x.shape == y.shape
      return tf.matmul(x,y)
  
  def add(x,y):
      return(tf.add(x,y))
  
  def main():
      A = tf.constant([[1, 2], [3, 4]], dtype=tf.float32)
      x = tf.constant([[0, 10], [0, 0.5]])
      b = tf.constant([[1, -1]], dtype=tf.float32)
  
      z = multiply(A,x)
      y = add(z, b)
      print(y)
  
  main()
  
  """### No More Globals
  
  - The second and most important advantage that eager execution brings is that global graphs are no longer needed and, by extension, neither are its global collections and variables.
  
  - We can declare a Graph variable by assigning it to a Python variable, but this bond is not tight: the Python variable gets destroyed as soon as it goes out of scope, while the graph variable is still present: it is a global and persistent object.
  
  - Deleting Python variables is completely useless since all the operations defined in the graph are still there and we can access their output tensor, thus restoring the Python variables if needed or creating new Python variables that point to the graph nodes.
  
  - All of these problems are gone in TensorFlow 2.0—we just have to write Python code that we are used to. There's no need to worry about graphs, global scopes, naming conflicts, placeholders, graph dependencies, and side effects.
  
  # Control Flow
  
  - . In TensorFlow 1.x, we have to explicitly force the order of execution using tf.control_dependencies, conditioning the assignment operation so that it's executed before the evaluation of out:
  
  - TensorFlow 2.x, with its eager execution, makes it possible to use the Python interpreter to control the flow of execution:
  """
  
  x = tf.Variable(1, dtype=tf.float32)
  y = tf.Variable(2, dtype=tf.float32)
  
  for _ in range(5):
      y.assign_add(1)
      out = x * y
      print(out)
  
  """# Gradient Tape
  
  - The tf.GradientTape() invocation creates a context that records the operations for automatic differentiation. Every operation that's executed within the context manager is recorded on tape if at least one of their inputs is watchable and is being watched.
  
  - An input is watchable when the following occurs:
  
  - It's a trainable variable that's been created by using tf.Variable
  - It's being explicitly watched by the tape, which is done by calling the watch method on the tf.Tensor object
  
  - The tape records every operation that's executed within the context in order to build a graph of the forward pass that was executed; then, the tape can be unrolled in order to compute the gradients using reverse-mode automatic differentiation.
  """
  
  x = tf.constant(4, dtype=tf.float32)
  with tf.GradientTape() as tape:
      tape.watch(x)
      y = tf.pow(x,2)
  
  dy_dx = tape.gradient(y, x)
  print(dy_dx)
  
  """- In the preceding example, we explicitly asked tape to watch a constant value that, by its nature, is not watchable (since it is not a tf.Variable object).
  
  - A tf.GradientTape object such as tape releases the resources that it's holding as soon as the tf.GradientTape.gradient() method is called. 
  - This is desirable for the most common scenarios, but there are cases in which we need to invoke tf.GradientTape.gradient() more than once. 
  - To do that, we need to create a persistent gradient tape that allows multiple calls to the gradient method without it releasing the resources.
  """
  
  x = tf.Variable(4.0)
  y = tf.Variable(2.0)
  
  with tf.GradientTape(persistent=True) as tape:
      z = x * y
      w = tf.pow(x,2)
  
  dz_dy = tape.gradient(z,y)
  dz_dx = tape.gradient(z,x)
  dw_dx = tape.gradient(w,x)
  
  print(dz_dy, dz_dx, dw_dx)
  del tape
  
  """# Custom training loop
  
  There are several differences between these APIs:
  
  - The optimizers are now Keras optimizers.
  - The losses are now Keras losses.
  - The accuracy is easily computed using the Keras metrics package.
  - There is always a TensorFlow 2.0 version of any TensorFlow 1.x symbol.
  - There are no more global collections. The tape needs a list of the variables it needs to use to compute the gradient and the tf.keras.Model object has to carry its own set of trainable_variables.
  """
  
  from tensorflow.keras.datasets import fashion_mnist
  
  def make_model(n_classes):
      model = tf.keras.Sequential()
      model.add(tf.keras.layers.Conv2D(32, (5,5), activation=tf.nn.relu, input_shape=(28,28,1)))
      model.add(tf.keras.layers.MaxPool2D((2,2), (2,2)))
      model.add(tf.keras.layers.Conv2D(32, (5,5), activation=tf.nn.relu))
      model.add(tf.keras.layers.MaxPool2D((2,2), (2,2)))
      model.add(tf.keras.layers.Flatten())
      model.add(tf.keras.layers.Dense(1024, activation=tf.nn.relu))
      model.add(tf.keras.layers.Dropout(0.3))
      model.add(tf.keras.layers.Dense(n_classes))
  
      return model
  
  def load_data():
      (train_x, train_y), (test_x, test_y) = fashion_mnist.load_data()
      train_x = tf.expand_dims(train_x, -1)
      train_x = (tf.image.convert_image_dtype(train_x, tf.float32) - 0.5) * 2
      train_y = tf.expand_dims(train_y, -1)
  
      test_x = test_x / 255. * 2 - 1
      test_x = (tf.image.convert_image_dtype(test_x, tf.float32) - 0.5) * 2
      test_y = tf.expand_dims(test_y, -1)
  
      return (train_x, train_y), (test_x, test_y)
  
  def train():
      n_classes = 10
      model = make_model(n_classes)
  
      (train_x, train_y), (test_x, test_y) = load_data()
  
      loss = tf.losses.SparseCategoricalCrossentropy(from_logits=True)
      step = tf.Variable(1, name="global_step")
      optimizer = tf.optimizers.Adam(1e-3)
      accuracy = tf.metrics.Accuracy()
  
      def train_step(inputs, labels):
          with tf.GradientTape() as tape:
              logits = model(inputs)
              loss_value = loss(labels, logits)
  
          gradients = tape.gradient(loss_value, model.trainable_variables)
          # TODO: apply gradient clipping here
          optimizer.apply_gradients(zip(gradients, model.trainable_variables))
          step.assign_add(1)
  
          accuracy_value = accuracy(labels, tf.argmax(logits, -1))
          return loss_value, accuracy_value
  
      epochs = 10
      batch_size = 32
      nr_batches_train = int(train_x.shape[0] / batch_size)
      print(f"Batch size: {batch_size}")
      print(f"Number of batches per epoch: {nr_batches_train}")
  
      for epoch in range(epochs):
          for t in range(nr_batches_train):
              start_from = t * batch_size
              to = (t + 1) * batch_size
              features, labels = train_x[start_from:to], train_y[start_from:to]
              loss_value, accuracy_value = train_step(features, labels)
              if t % 10 == 0:
                  print(
                      f"{step.numpy()}: {loss_value} - accuracy: {accuracy_value}"
                  )
          print(f"Epoch {epoch} terminated")
  
  train()
  
  """# Saving and restoring the model's status
  
  - TensorFlow 2.0 introduced the concept of a checkpointable object: every object that inherits from tf.train.Checkpointable is automatically serializable, which means that it is possible to save it in a checkpoint.
  
  - Due to this, it is possible to save whole layers/models instead of worrying about their variables; as usual, Keras introduced an additional abstraction layer that simplifies the usage of the framework. There are two ways of saving a model:
  
  - Using a checkpoint
  - Using a SavedModel
  
  - The SavedModel format, on the other hand, is the serialized description of the computation, in addition to the parameter's value. We can summarize these two objects as follows:
  
  - Checkpoint: An easy way to store variables on disk
  - SavedModel: Model structure and checkpoint
  
  - Restart the training process in the case of failures, without wasting all the previous computation.
  - Save the model parameters at the end of the training loop so that we can test the trained model on the test set.
  - Save the model parameters in different locations so that we can save the status of the models that reached the best validation performance (model selection).
  
  To save and restore the model parameters in TensorFlow 2.0, we can use two objects:
  - tf.train.Checkpoint is the object-based serializer/deserializer.
  - tf.train.CheckpointManager is an object that can use a tf.train.Checkpoint instance to save and manage checkpoints.
  
  Because of this, the save and restore capabilities of the previous model's training are as easy as adding the following lines, right after the model and optimizer definition:
  """
  
  chkpt = tf.train.Checkpoint(step=step, optimizer=optimizer, model=model)
  manager = tf.train.CheckpointManager(chkpt, './checkpoints', max_to_keep=3)
  chkpt.restore(manager.latest_checkpoint)
  if manager.latest_checkpoint:
      print("Restored from %s "%(manager.latest_checkpoint))
  else:
      print("Initialzing from scratch")
  
  """- Trainable and not-trainable variables are automatically added for the checkpoint variables to monitor, allowing you to restore the model and restart the training loop without introducing unwanted fluctuations in the loss functions. 
  
  - When a condition is met (i % 10 == 0, or when the validation metric is improved), is it possible to use the manager.save method invocation to checkpoint the model's status:
  """
  
  save_path = manager.save()
  print("Checkpoint saved %s "%(save_path))
  
  """- The manager can save the model parameters in the directory that's specified during its construction; therefore, to perform model selection,
  
  # Summaries and metrics
  
  - The tf.summary package contains all the required methods to save scalar values, images, plot histograms, distributions, and more.
  - The objects in the tf.metrics package are stateful, which means they are able to accumulate/aggregate values and return a cumulative result when calling .result().
  """
  
  summary_writer = tf.summary.create_file_writer(path)
  
  """- This new object doesn't work like it does in 1.x—its usage is now simplified and more powerful. Instead of using a session and executing the sess.run(summary) line to get the line to write inside the summary, the new tf.summary.* objects are able to detect the context they are used within and log the correct summary inside the writer once the summary line has been computed.
  
  - In fact, the summary writer object defines a context manager by calling .as_default(); every tf.summary.* method that's invoked within this context will add its result to the default summary writer.
  
  The following code shows how the previous example can be extended to log the training loss, accuracy, and three training images
  """
  
  def train():
      n_classes = 10
      model = make_model(n_classes)
  
      (train_x, train_y), (test_x, test_y) = load_data()
  
      loss = tf.losses.SparseCategoricalCrossentropy(from_logits=True)
      step = tf.Variable(1, name="global_step")
      optimizer = tf.optimizers.Adam(1e-3)
  
      chkpt = tf.train.Checkpoint(step=step, optimizer=optimizer, model = model)
      manager = tf.train.CheckpointManager(chkpt, './tf_chkpt', max_to_keep=3)
      chkpt.restore(manager.latest_checkpoint)
  
      if manager.latest_checkpoint:
          print("Restored from %s" %(manager.latest_checkpoint))
      else:
          print("Initializing from scratch")
      
      accuracy = tf.metrics.Accuracy()
      mean_loss = tf.metrics.Mean(name='loss')
  
      def train_step(inputs, labels):
          with tf.GradientTape as tape:
              logits = model(inputs)
              loss_val = loss(labels, logits)
          
          gradient = tape.gradient(loss_value, model.trainable_variables)
          optimizer.apply_gradients(zip(gradients, model.trainable_variables))
          step.assign_add(1)
          accuracy.update_state(labels, tf.argmax(logits, -1))
  
          return loss_value, accuracy.result()
  
      epochs = 10
      batch_size = 32
      nr_batches_train = int(x_train.shape[0] / batch_size)
      print("Batch size: %d" %(batch_size))
      print("Number of batches per epoch: " %(nr_batches_train))
      train_summary_writer = tf.summary.create_file_writer('./log/train')
      with train_summary_writer.as_default():
          for epoch in range(epochs):
              for t in range(nr_batches_train):
                  start_from = t * batch_size
                  to = (t+1) * batch_size
  
                  features, labels = train_x{start_from:to}, train_y[start_from:to]
                  loss_value, accuracy_value = train_step(features, labels)
                  mean_loss.update_state(loss_value)
  
                  if (t%10==0):
                      print(f"{step.numpy()}: {loss_value} - accuracy: {accuracy_value}")
                      save_path = manager.save()
                      print("Checkpoint saved: %s "%(save_path))
                      tf.summary.image('train_set',features,max_outputs=3,step=step.numpy())
                      tf.summary.scalar('accuracy',accuracy_value,step=step.numpy())
                      tf.summary.scalar('loss',mean_loss.result(),step=step.numpy())
                      accuracy.reset_states()
                      mean_loss.reseet_states()
              
              for t in range(nr_batches_train):
                  start_from = t * batch_size
                  to = (t + 1) * batch_size
                  features, labels = train_x[start_from:to], train_y[start_from:to]
                  logits = model(features)
                  accuracy.update_state(labels, tf.argmax(logits, -1))
              print(f"Training accuracy: {accuracy.result()}")
              accuracy.reset_states()
  
  """Eager execution allows you to create and execute models on the fly, without explicitly creating a graph. However, working in eager mode does not mean that a graph can't be built from TensorFlow code. In fact, as we saw in the previous section, by using tf.GradientTape, is it possible to register what happens during a training step, build a computational graph by tracing the operations that are executed, and use this graph to automatically compute the gradient using automatic differentiation.
  
  # AutoGraph
  
  - Automatically converting Python code into its graphical representation is done with the use of AutoGraph.
  - In TensorFlow 2.0, AutoGraph is automatically applied to a function when it is decorated with @tf.function. This decorator creates callable graphs from Python functions.
  - A function, once decorated correctly, is processed by tf.function and the tf.autograph module in order to convert it into its graphical representation.
  
  - Schematic representation of what happens when a function, f, decorated with @tf.function, which is called on the first call and on any other subsequent call
  
  On the first call of the annotated function, the following occurs:
  
  - The function is executed and traced. Eager execution is disabled in this context, and so every tf.* method defines a tf.Operation node that produces a tf.Tensor output, exactly like it does in TensorFlow 1.x.
  - The tf.autograph module is used to detect Python constructs that can be converted into their graph equivalent. The graph representation is built from the function trace and AutoGraph information. This is done in order to preserve the execution order that's defined in Python.
  - The tf.Graph object has now been built.
  - Based on the function name and the input parameters, a unique ID is created and associated with the graph. The graph is then cached into a map so that it can be reused when a second invocation occurs and the ID matches.
  
  - tf.function parses the function body multiple times while looking for the tf.Variable definition. If, at the second invocation, it finds out that a variable object is being recreated, it raises an exception:
  
  - In practice, if we have defined a function that performs a simple operation that uses a tf.Variable inside it, we have to ensure that the object is only created once.
  
  - The following function works correctly in eager mode, but it fails to execute if it is decorated with @tf.function and is raising the preceding exception:
  """
  
  def f():
      a = tf.constant([[10,10], [11,1]])
      x = tf.constant([[1,0], [0,1]])
      b = tf.Variable(12)
      y = tf.matmul(a,x) + b
      return y
  
  """Handling functions that create a state means that we have to rethink our usage of graph-mode. A state is a persistent object, such as a variable, and the variable can't be redeclared more than once. Due to this, the function definition can be changed in two ways:
  
  - By passing the variable as an input parameter
  - By breaking the function scope and inheriting a variable from the external scope
  """
  
  @tf.function
  def f(b):
      a = tf.constant([[10,10], [11,1]])
      x = tf.constant([[1,1], [0,1]])
      y = tf.matmul(a,x) + b
      return y
  
  var = tf.Variable(12)
  f(var)
  f(15)
  f(tf.constant(1))
  
  """f now accepts a Python input variable, b. This variable can be a tf.Variable, a tf.Tensor, and also a NumPy object or a Python type. 
  - Every time the input type changes, a new graph is created in order to make an accelerated version of the function that works for any required input type
  
  The second option, on the other hand, requires breaking down the function scope,making the variable available outside the scope of the function itself. In this case, there are two paths we can follow:
  
  - Not recommended: Use global variables
  - Recommended: Use Keras-like objects
  """
  
  # NOT RECOMMENDED PATH BUT EASY
  b = None
  @tf.function
  def f():
      a = tf.constant([[10, 10], [11.,1]])
      x = tf.constant([[1.,0], [0.,1]])
      global b
      if b is None:
          b = tf.Variable(12.)
      y = tf.matmul(a,x) + b
      return y
  f()
  
  """The second path, which is recommended, is to use an object-oriented approach and declare the variable as a private attribute of a class. Then, you need to make the objects that were instantiated callable by putting the function body inside the __call__ method:"""
  
  class F():
      def __init__(self):
          self._b = None
      
      @tf.function
      def __call__(self):
          a = tf.constant([[10.,10.], [11.,1.]])
          x = tf.constant([[1., 0.], [0., 1.]])
          if self._b is None:
              self._b = tf.Variable(12.)
          y = tf.matmul(a,x) + self._b
          return y
  
  f = F()
  
  f()
  
  """- AutoGraph and the graph acceleration process work best when it comes to optimizing the training process.
  
  The speedup is not guaranteed since eager execution is already fast and there are simple scenarios in which eager execution is as fast as its graphical counterpart. However, the performance boost is visible when the models become more complex and deeper.
  
  # Codebase migration
  
  1. Run the migration script.
  2. Manually remove every tf.contrib symbol, looking for the new location of the project that was used in the contrib namespace.
  3. Manually switch the models to their Keras equivalent. Remove the sessions.
  4. Define the training loop in eager execution mode.
  5. Accelerate the computationally-intensive parts using tf.function.
  
  The migration tool, tf_upgrade_v2, is installed automatically when TensorFlow 2.0 is installed via pip. The upgrade script works on single Python files, notebooks, or complete project directories.
  """
  
  # For a single file
  tf_upgrade_v2 --infile file.py --outfile file-migrated.py
  # For a directory
  tf_upgrade_v2 --intree project --outtree project-migrate
  
  """Moreover, it always reports a list of detailed changes in the report.txt file, which can help us understand why certain changes have been applied by the tool; for example:"""