class_docstr: Defines Temporal Fusion Transformer.
  
  Attributes:
    name: Name of model
    time_steps: Total number of input time steps per forecast date (i.e. Width
      of Temporal fusion decoder N)
    input_size: Total number of inputs
    output_size: Total number of outputs
    category_counts: Number of categories per categorical variable
    n_multiprocessing_workers: Number of workers to use for parallel
      computations
    column_definition: List of tuples of (string, DataType, InputType) that
      define each column
    quantiles: Quantiles to forecast for TFT
    use_cudnn: Whether to use Keras CuDNNLSTM or standard LSTM layers
    hidden_layer_size: Internal state size of TFT
    dropout_rate: Dropout discard rate
    max_gradient_norm: Maximum norm for gradient clipping
    learning_rate: Initial learning rate of ADAM optimizer
    minibatch_size: Size of minibatches for training
    num_epochs: Maximum number of epochs for training
    early_stopping_patience: Maximum number of iterations of non-improvement
      before early stopping kicks in
    num_encoder_steps: Size of LSTM encoder -- i.e. number of past time steps
      before forecast date to use
    num_stacks: Number of self-attention layers to apply (default is 1 for basic
      TFT)
    num_heads: Number of heads for interpretable mulit-head attention
    model: Keras model for TFT
class_docstr_tok: ["Defines","Temporal","Fusion","Transformer",".","Attributes",":","name",":","Name","of","model","time_steps",":","Total","number","of","input","time","steps","per","forecast","date","(","i.e",".","Width","of","Temporal","fusion","decoder","N",")","input_size",":","Total","number","of","inputs","output_size",":","Total","number","of","outputs","category_counts",":","Number","of","categories","per","categorical","variable","n_multiprocessing_workers",":","Number","of","workers","to","use","for","parallel","computations","column_definition",":","List","of","tuples","of","(","string",",","DataType",",","InputType",")","that","define","each","column","quantiles",":","Quantiles","to","forecast","for","TFT","use_cudnn",":","Whether","to","use","Keras","CuDNNLSTM","or","standard","LSTM","layers","hidden_layer_size",":","Internal","state","size","of","TFT","dropout_rate",":","Dropout","discard","rate","max_gradient_norm",":","Maximum","norm","for","gradient","clipping","learning_rate",":","Initial","learning","rate","of","ADAM","optimizer","minibatch_size",":","Size","of","minibatches","for","training","num_epochs",":","Maximum","number","of","epochs","for","training","early_stopping_patience",":","Maximum","number","of","iterations","of","non-improvement","before","early","stopping","kicks","in","num_encoder_steps",":","Size","of","LSTM","encoder","--","i.e",".","number","of","past","time","steps","before","forecast","date","to","use","num_stacks",":","Number","of","self-attention","layers","to","apply","(","default","is","1","for","basic","TFT",")","num_heads",":","Number","of","heads","for","interpretable","mulit-head","attention","model",":","Keras","model","for","TFT"]
docstr: Returns graph defining layers of the TFT.
docstr_tok: ["Returns","graph","defining","layers","of","the","TFT","."]
code: def _build_base_graph(self):
      """Returns graph defining layers of the TFT."""
  
      # Size definitions.
      time_steps = self.time_steps
      combined_input_size = self.input_size
      encoder_steps = self.num_encoder_steps
  
      # Inputs.
      all_inputs = tf.keras.layers.Input(
          shape=(
              time_steps,
              combined_input_size,
          )
      )
  
      (
          unknown_inputs,
          known_combined_layer,
          obs_inputs,
          static_inputs,
      ) = self.get_tft_embeddings(all_inputs)
  
      # Isolate known and observed historical inputs.
      if unknown_inputs is not None:
          historical_inputs = concat(
              [
                  unknown_inputs[:, :encoder_steps, :],
                  known_combined_layer[:, :encoder_steps, :],
                  obs_inputs[:, :encoder_steps, :],
              ],
              axis=-1,
          )
      else:
          historical_inputs = concat(
              [
                  known_combined_layer[:, :encoder_steps, :],
                  obs_inputs[:, :encoder_steps, :],
              ],
              axis=-1,
          )
  
      # Isolate only known future inputs.
      future_inputs = known_combined_layer[:, encoder_steps:, :]
  
      def static_combine_and_mask(embedding):
          """Applies variable selection network to static inputs.
  
          Args:
            embedding: Transformed static inputs
  
          Returns:
            Tensor output for variable selection network
          """
  
          # Add temporal features
          _, num_static, _ = embedding.get_shape().as_list()
  
          flatten = tf.keras.layers.Flatten()(embedding)
  
          # Nonlinear transformation with gated residual network.
          mlp_outputs = gated_residual_network(
              flatten,
              self.hidden_layer_size,
              output_size=num_static,
              dropout_rate=self.dropout_rate,
              use_time_distributed=False,
              additional_context=None,
          )
  
          sparse_weights = tf.keras.layers.Activation("softmax")(
              mlp_outputs
          )
          sparse_weights = K.expand_dims(sparse_weights, axis=-1)
  
          trans_emb_list = []
          for i in range(num_static):
              e = gated_residual_network(
                  embedding[:, i : i + 1, :],
                  self.hidden_layer_size,
                  dropout_rate=self.dropout_rate,
                  use_time_distributed=False,
              )
              trans_emb_list.append(e)
  
          transformed_embedding = concat(trans_emb_list, axis=1)
  
          combined = tf.keras.layers.Multiply()(
              [sparse_weights, transformed_embedding]
          )
  
          static_vec = K.sum(combined, axis=1)
  
          return static_vec, sparse_weights
  
      static_encoder, static_weights = static_combine_and_mask(
          static_inputs
      )
  
      static_context_variable_selection = gated_residual_network(
          static_encoder,
          self.hidden_layer_size,
          dropout_rate=self.dropout_rate,
          use_time_distributed=False,
      )
      static_context_enrichment = gated_residual_network(
          static_encoder,
          self.hidden_layer_size,
          dropout_rate=self.dropout_rate,
          use_time_distributed=False,
      )
      static_context_state_h = gated_residual_network(
          static_encoder,
          self.hidden_layer_size,
          dropout_rate=self.dropout_rate,
          use_time_distributed=False,
      )
      static_context_state_c = gated_residual_network(
          static_encoder,
          self.hidden_layer_size,
          dropout_rate=self.dropout_rate,
          use_time_distributed=False,
      )
  
      def lstm_combine_and_mask(embedding):
          """Apply temporal variable selection networks.
  
          Args:
            embedding: Transformed inputs.
  
          Returns:
            Processed tensor outputs.
          """
  
          # Add temporal features
          (
              _,
              time_steps,
              embedding_dim,
              num_inputs,
          ) = embedding.get_shape().as_list()
  
          flatten = K.reshape(
              embedding, [-1, time_steps, embedding_dim * num_inputs]
          )
  
          expanded_static_context = K.expand_dims(
              static_context_variable_selection, axis=1
          )
  
          # Variable selection weights
          mlp_outputs, static_gate = gated_residual_network(
              flatten,
              self.hidden_layer_size,
              output_size=num_inputs,
              dropout_rate=self.dropout_rate,
              use_time_distributed=True,
              additional_context=expanded_static_context,
              return_gate=True,
          )
  
          sparse_weights = tf.keras.layers.Activation("softmax")(
              mlp_outputs
          )
          sparse_weights = tf.expand_dims(sparse_weights, axis=2)
  
          # Non-linear Processing & weight application
          trans_emb_list = []
          for i in range(num_inputs):
              grn_output = gated_residual_network(
                  embedding[Ellipsis, i],
                  self.hidden_layer_size,
                  dropout_rate=self.dropout_rate,
                  use_time_distributed=True,
              )
              trans_emb_list.append(grn_output)
  
          transformed_embedding = stack(trans_emb_list, axis=-1)
  
          combined = tf.keras.layers.Multiply()(
              [sparse_weights, transformed_embedding]
          )
          temporal_ctx = K.sum(combined, axis=-1)
  
          return temporal_ctx, sparse_weights, static_gate
  
      historical_features, historical_flags, _ = lstm_combine_and_mask(
          historical_inputs
      )
      future_features, future_flags, _ = lstm_combine_and_mask(
          future_inputs
      )
  
      # LSTM layer
      def get_lstm(return_state):
          """Returns LSTM cell initialized with default parameters."""
          if self.use_cudnn:
              lstm = tf.keras.layers.CuDNNLSTM(
                  self.hidden_layer_size,
                  return_sequences=True,
                  return_state=return_state,
                  stateful=False,
              )
          else:
              lstm = tf.keras.layers.LSTM(
                  self.hidden_layer_size,
                  return_sequences=True,
                  return_state=return_state,
                  stateful=False,
                  # Additional params to ensure LSTM matches CuDNN, See TF 2.0 :
                  # (https://www.tensorflow.org/api_docs/python/tf/keras/layers/LSTM)
                  activation="tanh",
                  recurrent_activation="sigmoid",
                  recurrent_dropout=0,
                  unroll=False,
                  use_bias=True,
              )
          return lstm
  
      history_lstm, state_h, state_c = get_lstm(return_state=True)(
          historical_features,
          initial_state=[
              static_context_state_h,
              static_context_state_c,
          ],
      )
  
      future_lstm = get_lstm(return_state=False)(
          future_features, initial_state=[state_h, state_c]
      )
  
      lstm_layer = concat([history_lstm, future_lstm], axis=1)
  
      # Apply gated skip connection
      input_embeddings = concat(
          [historical_features, future_features], axis=1
      )
  
      lstm_layer, _ = apply_gating_layer(
          lstm_layer,
          self.hidden_layer_size,
          self.dropout_rate,
          activation=None,
      )
      temporal_feature_layer = add_and_norm(
          [lstm_layer, input_embeddings]
      )
  
      # Static enrichment layers
      expanded_static_context = K.expand_dims(
          static_context_enrichment, axis=1
      )
      enriched, _ = gated_residual_network(
          temporal_feature_layer,
          self.hidden_layer_size,
          dropout_rate=self.dropout_rate,
          use_time_distributed=True,
          additional_context=expanded_static_context,
          return_gate=True,
      )
  
      # Decoder self attention
      self_attn_layer = InterpretableMultiHeadAttention(
          self.num_heads,
          self.hidden_layer_size,
          dropout=self.dropout_rate,
      )
  
      mask = get_decoder_mask(enriched)
      x, self_att = self_attn_layer(
          enriched, enriched, enriched, mask=mask
      )
  
      x, _ = apply_gating_layer(
          x,
          self.hidden_layer_size,
          dropout_rate=self.dropout_rate,
          activation=None,
      )
      x = add_and_norm([x, enriched])
  
      # Nonlinear processing on outputs
      decoder = gated_residual_network(
          x,
          self.hidden_layer_size,
          dropout_rate=self.dropout_rate,
          use_time_distributed=True,
      )
  
      # Final skip connection
      decoder, _ = apply_gating_layer(
          decoder, self.hidden_layer_size, activation=None
      )
      transformer_layer = add_and_norm(
          [decoder, temporal_feature_layer]
      )
  
      # Attention components for explainability
      attention_components = {
          # Temporal attention weights
          "decoder_self_attn": self_att,
          # Static variable selection weights
          "static_flags": static_weights[Ellipsis, 0],
          # Variable selection weights of past inputs
          "historical_flags": historical_flags[Ellipsis, 0, :],
          # Variable selection weights of future inputs
          "future_flags": future_flags[Ellipsis, 0, :],
      }
  
      return transformer_layer, all_inputs, attention_components
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import gc
  import json
  import os
  import shutil
  import data_formatters.base
  import libs.utils
  import numpy
  import pandas
  import tensorflow
label_desc: Use image node_id 9 for calling the TemporalFusionTransformer obj's underlying member method code with example usage: obj._build_base_graph() and returns: transformer_layer,  all_inputs,  attention_components, static_vec,  sparse_weights, temporal_ctx,  sparse_weights,  static_gate, lstm