class_docstr: Main class that defines training graph and defines training run method for language modeling
class_docstr_tok: ["Main","class","that","defines","training","graph","and","defines","training","run","method","for","language","modeling"]
docstr: Method that builds the graph for training
  Args:
      num_words: int, number of words in the vocabulary
      word_embeddings: numpy array, optional numpy array to initialize embeddings
      max_gradient_norm: float, maximum gradient norm value for clipping
      em_dropout: float, dropout rate for embeddings
  
  Returns:
      None
docstr_tok: ["Method","that","builds","the","graph","for","training","Args",":","num_words",":","int",",","number","of","words","in","the","vocabulary","word_embeddings",":","numpy","array",",","optional","numpy","array","to","initialize","embeddings","max_gradient_norm",":","float",",","maximum","gradient","norm","value","for","clipping","em_dropout",":","float",",","dropout","rate","for","embeddings","Returns",":","None"]
code: def build_train_graph(
      self,
      num_words=20000,
      word_embeddings=None,
      max_gradient_norm=None,
      em_dropout=0.4,
  ):
      """
      Method that builds the graph for training
      Args:
          num_words: int, number of words in the vocabulary
          word_embeddings: numpy array, optional numpy array to initialize embeddings
          max_gradient_norm: float, maximum gradient norm value for clipping
          em_dropout: float, dropout rate for embeddings
  
      Returns:
          None
      """
      self.num_words = num_words
      with tf.variable_scope("input", reuse=True):
          self.input_placeholder_tokens = tf.placeholder(
              tf.int32, [None, self.max_len], name="input_tokens"
          )
          self.label_placeholder_tokens = tf.placeholder(
              tf.int32,
              [None, self.max_len],
              name="input_tokens_shifted",
          )
          self.learning_rate = tf.placeholder(
              tf.float32, shape=(), name="learning_rate"
          )
  
      self.input_embeddings = self.define_input_layer(
          self.input_placeholder_tokens,
          word_embeddings,
          embeddings_trainable=True,
      )
  
      input_embeddings_dropped = tf.layers.dropout(
          self.input_embeddings,
          rate=em_dropout,
          training=self.training_mode,
      )
      self.prediction = self.build_network_graph(
          input_embeddings_dropped, last_timepoint=False
      )
  
      if self.prediction.shape[-1] != self.n_features_in:
          print("Not tying weights")
          tied_weights = False
      else:
          print("Tying weights")
          tied_weights = True
      self.projection_out = self.define_projection_layer(
          self.prediction, tied_weights=tied_weights
      )
      self.gen_seq_prob = tf.nn.softmax(self.projection_out)
  
      with tf.variable_scope("training"):
          params = tf.trainable_variables()
  
          soft_ce = tf.nn.sparse_softmax_cross_entropy_with_logits(
              labels=self.label_placeholder_tokens,
              logits=self.projection_out,
          )
          ce_last_tokens = tf.slice(
              soft_ce,
              [0, int(self.max_len / 2)],
              [-1, int(self.max_len / 2)],
          )
          self.training_loss = tf.reduce_mean(ce_last_tokens)
  
          summary_ops_train = [
              tf.summary.scalar("Training Loss", self.training_loss),
              tf.summary.scalar(
                  "Training perplexity", tf.exp(self.training_loss)
              ),
          ]
          self.merged_summary_op_train = tf.summary.merge(
              summary_ops_train
          )
  
          self.validation_loss = tf.placeholder(tf.float32, shape=())
          summary_ops_val = [
              tf.summary.scalar(
                  "Validation Loss", self.validation_loss
              ),
              tf.summary.scalar(
                  "Validation perplexity", tf.exp(self.validation_loss)
              ),
          ]
          self.merged_summary_op_val = tf.summary.merge(summary_ops_val)
  
          self.test_loss = tf.placeholder(tf.float32, shape=())
          summary_ops_test = [
              tf.summary.scalar("Test Loss", self.test_loss),
              tf.summary.scalar(
                  "Test perplexity", tf.exp(self.test_loss)
              ),
          ]
          self.merged_summary_op_test = tf.summary.merge(
              summary_ops_test
          )
  
          # Calculate and clip gradients
          gradients = tf.gradients(self.training_loss, params)
  
          if max_gradient_norm is not None:
              clipped_gradients, _ = tf.clip_by_global_norm(
                  gradients, max_gradient_norm
              )
          else:
              clipped_gradients = gradients
  
          grad_norm = tf.global_norm(clipped_gradients)
          summary_ops_train.append(
              tf.summary.scalar("Grad Norm", grad_norm)
          )
  
          # Optimization
          update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
          summary_ops_train.append(
              tf.summary.scalar("Learning rate", self.learning_rate)
          )
          self.merged_summary_op_train = tf.summary.merge(
              summary_ops_train
          )
  
          optimizer = tf.train.GradientDescentOptimizer(
              learning_rate=self.learning_rate
          )
          with tf.control_dependencies(update_ops):
              self.training_update_step = optimizer.apply_gradients(
                  zip(clipped_gradients, params)
              )
  
file_path: nlp-architect/examples/word_language_model_with_tcn/mle_language_model/lm_model.py
imports: import os
  import numpy
  import tensorflow
  from tqdm import tqdm
  from nlp_architect.models.temporal_convolutional_network import TCN, CommonLayers
label_desc: Use image node_id 4 for calling the TCNForLM obj's underlying member method code with example usage: obj.build_train_graph(num_words, word_embeddings, max_gradient_norm, em_dropout) without return types