def main_lstm_generate_text():
<EOL>init_scale = <NUM_LIT:0.1><EOL>learning_rate = <NUM_LIT:1.0><EOL>max_grad_norm = <NUM_LIT:5><EOL>sequence_length = <NUM_LIT:20><EOL>hidden_size = <NUM_LIT:200><EOL>max_epoch = <NUM_LIT:4><EOL>max_max_epoch = <NUM_LIT:100><EOL>lr_decay = <NUM_LIT><EOL>batch_size = <NUM_LIT:20><EOL>top_k_list = [<NUM_LIT:1>, <NUM_LIT:3>, <NUM_LIT:5>, <NUM_LIT:10>]<EOL>print_length = <NUM_LIT:30><EOL>model_file_name = "<STR_LIT>"<EOL>words = customized_read_words(input_fpath="<STR_LIT>")<EOL>vocab = tl.nlp.create_vocab([words], word_counts_output_file='<STR_LIT>', min_word_count=<NUM_LIT:1>)<EOL>vocab = tl.nlp.Vocabulary('<STR_LIT>', unk_word="<STR_LIT>")<EOL>vocab_size = vocab.unk_id + <NUM_LIT:1><EOL>train_data = [vocab.word_to_id(word) for word in words]<EOL>seed = "<STR_LIT>"<EOL>seed = nltk.tokenize.word_tokenize(seed)<EOL>print('<STR_LIT>' % seed)<EOL>sess = tf.InteractiveSession()<EOL>input_data = tf.placeholder(tf.int32, [batch_size, sequence_length])<EOL>targets = tf.placeholder(tf.int32, [batch_size, sequence_length])<EOL>input_data_test = tf.placeholder(tf.int32, [<NUM_LIT:1>, <NUM_LIT:1>])<EOL>def inference(x, is_train, sequence_length, reuse=None):<EOL><INDENT>"""<STR_LIT>"""<EOL>print("<STR_LIT>" % (sequence_length, is_train, reuse))<EOL>rnn_init = tf.random_uniform_initializer(-init_scale, init_scale)<EOL>with tf.variable_scope("<STR_LIT>", reuse=reuse):<EOL><INDENT>network = EmbeddingInputlayer(x, vocab_size, hidden_size, rnn_init, name='<STR_LIT>')<EOL>network = RNNLayer(<EOL>network, cell_fn=tf.contrib.rnn.BasicLSTMCell, cell_init_args={<EOL>'<STR_LIT>': <NUM_LIT:0.0>,<EOL>'<STR_LIT>': True<EOL>}, n_hidden=hidden_size, initializer=rnn_init, n_steps=sequence_length, return_last=False,<EOL>return_seq_2d=True, name='<STR_LIT>'<EOL>)<EOL>lstm1 = network<EOL>network = DenseLayer(network, vocab_size, W_init=rnn_init, b_init=rnn_init, act=None, name='<STR_LIT>')<EOL><DEDENT>return network, lstm1<EOL><DEDENT>network, lstm1 = inference(input_data, is_train=True, sequence_length=sequence_length, reuse=None)<EOL>network_test, lstm1_test = inference(input_data_test, is_train=False, sequence_length=<NUM_LIT:1>, reuse=True)<EOL>y_linear = network_test.outputs<EOL>y_soft = tf.nn.softmax(y_linear)<EOL>def loss_fn(outputs, targets, batch_size, sequence_length):<EOL><INDENT>loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example(<EOL>[outputs], [tf.reshape(targets, [-<NUM_LIT:1>])], [tf.ones([batch_size * sequence_length])]<EOL>)<EOL>cost = tf.reduce_sum(loss) / batch_size<EOL>return cost<EOL><DEDENT>cost = loss_fn(network.outputs, targets, batch_size, sequence_length)<EOL>with tf.variable_scope('<STR_LIT>'):<EOL><INDENT>lr = tf.Variable(<NUM_LIT:0.0>, trainable=False)<EOL><DEDENT>tvars = network.all_params<EOL>grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars), max_grad_norm)<EOL>optimizer = tf.train.GradientDescentOptimizer(lr)<EOL>train_op = optimizer.apply_gradients(zip(grads, tvars))<EOL>sess.run(tf.global_variables_initializer())<EOL>print("<STR_LIT>")<EOL>for i in range(max_max_epoch):<EOL><INDENT>new_lr_decay = lr_decay**max(i - max_epoch, <NUM_LIT:0.0>)<EOL>sess.run(tf.assign(lr, learning_rate * new_lr_decay))<EOL>print("<STR_LIT>" % (i + <NUM_LIT:1>, max_max_epoch, sess.run(lr)))<EOL>epoch_size = ((len(train_data) // batch_size) - <NUM_LIT:1>) // sequence_length<EOL>start_time = time.time()<EOL>costs = <NUM_LIT:0.0><EOL>iters = <NUM_LIT:0><EOL>state1 = tl.layers.initialize_rnn_state(lstm1.initial_state)<EOL>for step, (x, y) in enumerate(tl.iterate.ptb_iterator(train_data, batch_size, sequence_length)):<EOL><INDENT>_cost, state1, _ = sess.run(<EOL>[cost, lstm1.final_state, train_op], feed_dict={<EOL>input_data: x,<EOL>targets: y,<EOL>lstm1.initial_state: state1<EOL>}<EOL>)<EOL>costs += _cost<EOL>iters += sequence_length<EOL>if step % (epoch_size // <NUM_LIT:10>) == <NUM_LIT:1>:<EOL><INDENT>print(<EOL>"<STR_LIT>" %<EOL>(step * <NUM_LIT:1.0> / epoch_size, np.exp(costs / iters), iters * batch_size / (time.time() - start_time))<EOL>)<EOL><DEDENT><DEDENT>train_perplexity = np.exp(costs / iters)<EOL>print("<STR_LIT>" % (i + <NUM_LIT:1>, max_max_epoch, train_perplexity))<EOL>for top_k in top_k_list:<EOL><INDENT>state1 = tl.layers.initialize_rnn_state(lstm1_test.initial_state)<EOL>outs_id = [vocab.word_to_id(w) for w in seed]<EOL>for ids in outs_id[:-<NUM_LIT:1>]:<EOL><INDENT>a_id = np.asarray(ids).reshape(<NUM_LIT:1>, <NUM_LIT:1>)<EOL>state1 = sess.run(<EOL>[lstm1_test.final_state], feed_dict={<EOL>input_data_test: a_id,<EOL>lstm1_test.initial_state: state1<EOL>}<EOL>)<EOL><DEDENT>a_id = outs_id[-<NUM_LIT:1>]<EOL>for _ in range(print_length):<EOL><INDENT>a_id = np.asarray(a_id).reshape(<NUM_LIT:1>, <NUM_LIT:1>)<EOL>out, state1 = sess.run(<EOL>[y_soft, lstm1_test.final_state], feed_dict={<EOL>input_data_test: a_id,<EOL>lstm1_test.initial_state: state1<EOL>}<EOL>)<EOL>a_id = tl.nlp.sample_top(out[<NUM_LIT:0>], top_k=top_k)<EOL>outs_id.append(a_id)<EOL><DEDENT>sentence = [vocab.id_to_word(w) for w in outs_id]<EOL>sentence = "<STR_LIT:U+0020>".join(sentence)<EOL>print(top_k, '<STR_LIT::>', sentence)<EOL><DEDENT><DEDENT>print("<STR_LIT>")<EOL>tl.files.save_npz(network_test.all_params, name=model_file_name)<EOL>
Generate text by Synced sequence input and output.