code: def main():
      """
      The core of the model consists of an LSTM cell that processes one word at
      a time and computes probabilities of the possible continuations of the
      sentence. The memory state of the network is initialized with a vector
      of zeros and gets updated after reading each word. Also, for computational
      reasons, we will process data in mini-batches of size batch_size.
  
      """
      param = process_args(sys.argv[1:])
  
      if param.model == "small":
          init_scale = 0.1
          learning_rate = 1e-3
          max_grad_norm = 5
          num_steps = 20
          hidden_size = 200
          max_epoch = 4
          max_max_epoch = 13
          keep_prob = 1.0
          lr_decay = 0.5
          batch_size = 20
          vocab_size = 10000
      elif param.model == "medium":
          init_scale = 0.05
          learning_rate = 1e-3
          max_grad_norm = 5
          # num_layers = 2
          num_steps = 35
          hidden_size = 650
          max_epoch = 6
          max_max_epoch = 39
          keep_prob = 0.5
          lr_decay = 0.8
          batch_size = 20
          vocab_size = 10000
      elif param.model == "large":
          init_scale = 0.04
          learning_rate = 1e-3
          max_grad_norm = 10
          # num_layers = 2
          num_steps = 35
          hidden_size = 1500
          max_epoch = 14
          max_max_epoch = 55
          keep_prob = 0.35
          lr_decay = 1 / 1.15
          batch_size = 20
          vocab_size = 10000
      else:
          raise ValueError("Invalid model: %s", param.model)
  
      # Load PTB dataset
      (
          train_data,
          valid_data,
          test_data,
          vocab_size,
      ) = tl.files.load_ptb_dataset()
      # train_data = train_data[0:int(100000/5)]    # for fast testing
      print(
          "len(train_data) {}".format(len(train_data))
      )  # 929589 a list of int
      print(
          "len(valid_data) {}".format(len(valid_data))
      )  # 73760  a list of int
      print(
          "len(test_data)  {}".format(len(test_data))
      )  # 82430  a list of int
      print("vocab_size      {}".format(vocab_size))  # 10000
  
      # One int represents one word, the meaning of batch_size here is not the
      # same with MNIST example, it is the number of concurrent processes for
      # computational reasons.
  
      init = tf.random_uniform_initializer(-init_scale, init_scale)
      net = PTB_Net(
          hidden_size=hidden_size,
          vocab_size=vocab_size,
          init=init,
          keep=keep_prob,
      )
  
      # Truncated Backpropagation for training
      lr = tf.Variable(0.0, trainable=False)
      train_weights = net.weights
      optimizer = tf.optimizers.Adam(lr=lr)
  
      print(net)
  
      print("\nStart learning a language model by using PTB dataset")
      for i in range(max_max_epoch):
          # decreases the initial learning rate after several
          # epoachs (defined by ``max_epoch``), by multipling a ``lr_decay``.
          new_lr_decay = lr_decay ** max(i - max_epoch, 0.0)
          lr.assign(learning_rate * new_lr_decay)
  
          # Training
          net.train()
          print(
              "Epoch: %d/%d Learning rate: %.3f"
              % (i + 1, max_max_epoch, lr.value())
          )
          epoch_size = (
              (len(train_data) // batch_size) - 1
          ) // num_steps
          start_time = time.time()
          costs = 0.0
          iters = 0
          # reset all states at the begining of every epoch
          lstm1_state = None
          lstm2_state = None
  
          for step, (x, y) in enumerate(
              tl.iterate.ptb_iterator(train_data, batch_size, num_steps)
          ):
              with tf.GradientTape() as tape:
                  ## compute outputs
                  logits, lstm1_state, lstm2_state = net(
                      x,
                      lstm1_initial_state=lstm1_state,
                      lstm2_initial_state=lstm2_state,
                  )
                  ## compute loss and update model
                  cost = tl.cost.cross_entropy(
                      logits, tf.reshape(y, [-1]), name="train_loss"
                  )
  
              grad, _ = tf.clip_by_global_norm(
                  tape.gradient(cost, train_weights), max_grad_norm
              )
              optimizer.apply_gradients(zip(grad, train_weights))
  
              costs += cost
              iters += 1
  
              if step % (epoch_size // 10) == 10:
                  print(
                      "%.3f perplexity: %.3f speed: %.0f wps"
                      % (
                          step * 1.0 / epoch_size,
                          np.exp(costs / iters),
                          iters
                          * batch_size
                          * num_steps
                          / (time.time() - start_time),
                      )
                  )
          train_perplexity = np.exp(costs / iters)
          print(
              "Epoch: %d/%d Train Perplexity: %.3f"
              % (i + 1, max_max_epoch, train_perplexity)
          )
  
          # Validing
          net.eval()
          start_time = time.time()
          costs = 0.0
          iters = 0
          # reset all states at the begining of every epoch
          lstm1_state = None
          lstm2_state = None
          for step, (x, y) in enumerate(
              tl.iterate.ptb_iterator(valid_data, batch_size, num_steps)
          ):
              ## compute outputs
              logits, lstm1_state, lstm2_state = net(
                  x,
                  lstm1_initial_state=lstm1_state,
                  lstm2_initial_state=lstm2_state,
              )
              ## compute loss and update model
              cost = tl.cost.cross_entropy(
                  logits, tf.reshape(y, [-1]), name="train_loss"
              )
              costs += cost
              iters += 1
          valid_perplexity = np.exp(costs / iters)
          print(
              "Epoch: %d/%d Valid Perplexity: %.3f"
              % (i + 1, max_max_epoch, valid_perplexity)
          )
  
      print("Evaluation")
      # Testing
      net.eval()
      # go through the test set step by step, it will take a while.
      start_time = time.time()
      costs = 0.0
      iters = 0
      # reset all states at the begining
      lstm1_state = None
      lstm2_state = None
      for step, (x, y) in enumerate(
          tl.iterate.ptb_iterator(test_data, batch_size=1, num_steps=1)
      ):
          ## compute outputs
          logits, lstm1_state, lstm2_state = net(
              x,
              lstm1_initial_state=lstm1_state,
              lstm2_initial_state=lstm2_state,
          )
          ## compute loss and update model
          cost = tl.cost.cross_entropy(
              logits, tf.reshape(y, [-1]), name="train_loss"
          )
          costs += cost
          iters += 1
      test_perplexity = np.exp(costs / iters)
      print(
          "Test Perplexity: %.3f took %.2fs"
          % (test_perplexity, time.time() - start_time)
      )
  
      print(
          "More example: Text generation using Trump's speech data: https://github.com/tensorlayer/tensorlayer/blob/master/example/tutorial_generate_text.py -- def main_lstm_generate_text():"
      )
  
file_path: TensorLayer/examples/text_ptb/tutorial_ptb_lstm.py
imports: import argparse
  import sys
  import time
  import numpy
  import tensorflow
  import tensorlayer
  from tensorlayer.models import Model
label_desc: Use image node_id 2 for calling a global function with example usage: main() without return types