def main(_):
if FLAGS.model == "<STR_LIT>":<EOL><INDENT>init_scale = <NUM_LIT:0.1><EOL>learning_rate = <NUM_LIT:1.><EOL>max_grad_norm = <NUM_LIT:5><EOL>num_steps = <NUM_LIT:20><EOL>hidden_size = <NUM_LIT:200><EOL>max_epoch = <NUM_LIT:4><EOL>max_max_epoch = <NUM_LIT><EOL>keep_prob = <NUM_LIT:1.0><EOL>lr_decay = <NUM_LIT:0.5><EOL>batch_size = <NUM_LIT:20><EOL>vocab_size = <NUM_LIT><EOL><DEDENT>elif FLAGS.model == "<STR_LIT>":<EOL><INDENT>init_scale = <NUM_LIT><EOL>learning_rate = <NUM_LIT:1.0><EOL>max_grad_norm = <NUM_LIT:5><EOL>num_steps = <NUM_LIT><EOL>hidden_size = <NUM_LIT><EOL>max_epoch = <NUM_LIT:6><EOL>max_max_epoch = <NUM_LIT><EOL>keep_prob = <NUM_LIT:0.5><EOL>lr_decay = <NUM_LIT><EOL>batch_size = <NUM_LIT:20><EOL>vocab_size = <NUM_LIT><EOL><DEDENT>elif FLAGS.model == "<STR_LIT>":<EOL><INDENT>init_scale = <NUM_LIT><EOL>learning_rate = <NUM_LIT:1.0><EOL>max_grad_norm = <NUM_LIT:10><EOL>num_steps = <NUM_LIT><EOL>hidden_size = <NUM_LIT><EOL>max_epoch = <NUM_LIT><EOL>max_max_epoch = <NUM_LIT><EOL>keep_prob = <NUM_LIT><EOL>lr_decay = <NUM_LIT:1> / <NUM_LIT><EOL>batch_size = <NUM_LIT:20><EOL>vocab_size = <NUM_LIT><EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>", FLAGS.model)<EOL><DEDENT>train_data, valid_data, test_data, vocab_size = tl.files.load_ptb_dataset()<EOL>print('<STR_LIT>'.format(len(train_data)))  <EOL>print('<STR_LIT>'.format(len(valid_data)))  <EOL>print('<STR_LIT>'.format(len(test_data)))  <EOL>print('<STR_LIT>'.format(vocab_size))  <EOL>sess = tf.InteractiveSession()<EOL>input_data = tf.placeholder(tf.int32, [batch_size, num_steps])<EOL>targets = tf.placeholder(tf.int32, [batch_size, num_steps])<EOL>input_data_test = tf.placeholder(tf.int32, [<NUM_LIT:1>, <NUM_LIT:1>])<EOL>targets_test = tf.placeholder(tf.int32, [<NUM_LIT:1>, <NUM_LIT:1>])<EOL>def inference(x, is_training, num_steps, reuse=None):<EOL><INDENT>"""<STR_LIT>"""<EOL>print("<STR_LIT>" % (num_steps, is_training, reuse))<EOL>init = tf.random_uniform_initializer(-init_scale, init_scale)<EOL>with tf.variable_scope("<STR_LIT>", reuse=reuse):<EOL><INDENT>net = tl.layers.EmbeddingInputlayer(x, vocab_size, hidden_size, init, name='<STR_LIT>')<EOL>net = tl.layers.DropoutLayer(net, keep=keep_prob, is_fix=True, is_train=is_training, name='<STR_LIT>')<EOL>net = tl.layers.RNNLayer(<EOL>net,<EOL>cell_fn=tf.contrib.rnn.BasicLSTMCell,  <EOL>cell_init_args={<EOL>'<STR_LIT>': <NUM_LIT:0.0>,<EOL>'<STR_LIT>': True<EOL>},<EOL>n_hidden=hidden_size,<EOL>initializer=init,<EOL>n_steps=num_steps,<EOL>return_last=False,<EOL>name='<STR_LIT>'<EOL>)<EOL>lstm1 = net<EOL>net = tl.layers.DropoutLayer(net, keep=keep_prob, is_fix=True, is_train=is_training, name='<STR_LIT>')<EOL>net = tl.layers.RNNLayer(<EOL>net,<EOL>cell_fn=tf.contrib.rnn.BasicLSTMCell,  <EOL>cell_init_args={<EOL>'<STR_LIT>': <NUM_LIT:0.0>,<EOL>'<STR_LIT>': True<EOL>},<EOL>n_hidden=hidden_size,<EOL>initializer=init,<EOL>n_steps=num_steps,<EOL>return_last=False,<EOL>return_seq_2d=True,<EOL>name='<STR_LIT>'<EOL>)<EOL>lstm2 = net<EOL>net = tl.layers.DropoutLayer(net, keep=keep_prob, is_fix=True, is_train=is_training, name='<STR_LIT>')<EOL>net = tl.layers.DenseLayer(net, vocab_size, W_init=init, b_init=init, act=None, name='<STR_LIT>')<EOL><DEDENT>return net, lstm1, lstm2<EOL><DEDENT>net, lstm1, lstm2 = inference(input_data, is_training=True, num_steps=num_steps, reuse=None)<EOL>net_val, lstm1_val, lstm2_val = inference(input_data, is_training=False, num_steps=num_steps, reuse=True)<EOL>net_test, lstm1_test, lstm2_test = inference(input_data_test, is_training=False, num_steps=<NUM_LIT:1>, reuse=True)<EOL>sess.run(tf.global_variables_initializer())<EOL>def loss_fn(outputs, targets, batch_size):<EOL><INDENT>loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example(<EOL>[outputs], [tf.reshape(targets, [-<NUM_LIT:1>])], [tf.ones_like(tf.reshape(targets, [-<NUM_LIT:1>]), dtype=tf.float32)]<EOL>)<EOL>cost = tf.reduce_sum(loss) / batch_size<EOL>return cost<EOL><DEDENT>cost = loss_fn(net.outputs, targets, batch_size)<EOL>cost_val = loss_fn(net_val.outputs, targets, batch_size)<EOL>cost_test = loss_fn(net_test.outputs, targets_test, <NUM_LIT:1>)<EOL>with tf.variable_scope('<STR_LIT>'):<EOL><INDENT>lr = tf.Variable(<NUM_LIT:0.0>, trainable=False)<EOL><DEDENT>tvars = tf.trainable_variables()<EOL>grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars), max_grad_norm)<EOL>optimizer = tf.train.GradientDescentOptimizer(lr)<EOL>train_op = optimizer.apply_gradients(zip(grads, tvars))<EOL>sess.run(tf.global_variables_initializer())<EOL>net.print_params()<EOL>net.print_layers()<EOL>tl.layers.print_all_variables()<EOL>print("<STR_LIT>")<EOL>for i in range(max_max_epoch):<EOL><INDENT>new_lr_decay = lr_decay**max(i - max_epoch, <NUM_LIT:0.0>)<EOL>sess.run(tf.assign(lr, learning_rate * new_lr_decay))<EOL>print("<STR_LIT>" % (i + <NUM_LIT:1>, max_max_epoch, sess.run(lr)))<EOL>epoch_size = ((len(train_data) // batch_size) - <NUM_LIT:1>) // num_steps<EOL>start_time = time.time()<EOL>costs = <NUM_LIT:0.0><EOL>iters = <NUM_LIT:0><EOL>state1 = tl.layers.initialize_rnn_state(lstm1.initial_state)<EOL>state2 = tl.layers.initialize_rnn_state(lstm2.initial_state)<EOL>for step, (x, y) in enumerate(tl.iterate.ptb_iterator(train_data, batch_size, num_steps)):<EOL><INDENT>feed_dict = {<EOL>input_data: x,<EOL>targets: y,<EOL>lstm1.initial_state.c: state1[<NUM_LIT:0>],<EOL>lstm1.initial_state.h: state1[<NUM_LIT:1>],<EOL>lstm2.initial_state.c: state2[<NUM_LIT:0>],<EOL>lstm2.initial_state.h: state2[<NUM_LIT:1>],<EOL>}<EOL>feed_dict.update(net.all_drop)<EOL>_cost, state1_c, state1_h, state2_c, state2_h, _ = sess.run(<EOL>[cost, lstm1.final_state.c, lstm1.final_state.h, lstm2.final_state.c, lstm2.final_state.h, train_op],<EOL>feed_dict=feed_dict<EOL>)<EOL>state1 = (state1_c, state1_h)<EOL>state2 = (state2_c, state2_h)<EOL>costs += _cost<EOL>iters += num_steps<EOL>if step % (epoch_size // <NUM_LIT:10>) == <NUM_LIT:10>:<EOL><INDENT>print(<EOL>"<STR_LIT>" %<EOL>(step * <NUM_LIT:1.0> / epoch_size, np.exp(costs / iters), iters * batch_size / (time.time() - start_time))<EOL>)<EOL><DEDENT><DEDENT>train_perplexity = np.exp(costs / iters)<EOL>print("<STR_LIT>" % (i + <NUM_LIT:1>, max_max_epoch, train_perplexity))<EOL>start_time = time.time()<EOL>costs = <NUM_LIT:0.0><EOL>iters = <NUM_LIT:0><EOL>state1 = tl.layers.initialize_rnn_state(lstm1_val.initial_state)<EOL>state2 = tl.layers.initialize_rnn_state(lstm2_val.initial_state)<EOL>for step, (x, y) in enumerate(tl.iterate.ptb_iterator(valid_data, batch_size, num_steps)):<EOL><INDENT>feed_dict = {<EOL>input_data: x,<EOL>targets: y,<EOL>lstm1_val.initial_state.c: state1[<NUM_LIT:0>],<EOL>lstm1_val.initial_state.h: state1[<NUM_LIT:1>],<EOL>lstm2_val.initial_state.c: state2[<NUM_LIT:0>],<EOL>lstm2_val.initial_state.h: state2[<NUM_LIT:1>],<EOL>}<EOL>_cost, state1_c, state1_h, state2_c, state2_h, _ = sess.run(<EOL>[<EOL>cost_val, lstm1_val.final_state.c, lstm1_val.final_state.h, lstm2_val.final_state.c,<EOL>lstm2_val.final_state.h,<EOL>tf.no_op()<EOL>], feed_dict=feed_dict<EOL>)<EOL>state1 = (state1_c, state1_h)<EOL>state2 = (state2_c, state2_h)<EOL>costs += _cost<EOL>iters += num_steps<EOL><DEDENT>valid_perplexity = np.exp(costs / iters)<EOL>print("<STR_LIT>" % (i + <NUM_LIT:1>, max_max_epoch, valid_perplexity))<EOL><DEDENT>print("<STR_LIT>")<EOL>start_time = time.time()<EOL>costs = <NUM_LIT:0.0><EOL>iters = <NUM_LIT:0><EOL>state1 = tl.layers.initialize_rnn_state(lstm1_test.initial_state)<EOL>state2 = tl.layers.initialize_rnn_state(lstm2_test.initial_state)<EOL>for step, (x, y) in enumerate(tl.iterate.ptb_iterator(test_data, batch_size=<NUM_LIT:1>, num_steps=<NUM_LIT:1>)):<EOL><INDENT>feed_dict = {<EOL>input_data_test: x,<EOL>targets_test: y,<EOL>lstm1_test.initial_state.c: state1[<NUM_LIT:0>],<EOL>lstm1_test.initial_state.h: state1[<NUM_LIT:1>],<EOL>lstm2_test.initial_state.c: state2[<NUM_LIT:0>],<EOL>lstm2_test.initial_state.h: state2[<NUM_LIT:1>],<EOL>}<EOL>_cost, state1_c, state1_h, state2_c, state2_h = sess.run(<EOL>[<EOL>cost_test,<EOL>lstm1_test.final_state.c,<EOL>lstm1_test.final_state.h,<EOL>lstm2_test.final_state.c,<EOL>lstm2_test.final_state.h,<EOL>], feed_dict=feed_dict<EOL>)<EOL>state1 = (state1_c, state1_h)<EOL>state2 = (state2_c, state2_h)<EOL>costs += _cost<EOL>iters += <NUM_LIT:1><EOL><DEDENT>test_perplexity = np.exp(costs / iters)<EOL>print("<STR_LIT>" % (test_perplexity, time.time() - start_time))<EOL>print(<EOL>"<STR_LIT>"<EOL>)<EOL>
The core of the model consists of an LSTM cell that processes one word at
a time and computes probabilities of the possible continuations of the
sentence. The memory state of the network is initialized with a vector
of zeros and gets updated after reading each word. Also, for computational
reasons, we will process data in mini-batches of size batch_size.