code: # -*- coding: utf-8 -*-
  #
  # Implementing an LSTM RNN Model
  # ------------------------------
  #  Here we implement an LSTM model on all a data set of Shakespeare works.
  #
  #
  #
  
  import os
  import re
  import string
  import requests
  import numpy as np
  import collections
  import random
  import pickle
  import matplotlib.pyplot as plt
  import tensorflow as tf
  from tensorflow.python.framework import ops
  
  ops.reset_default_graph()
  
  # Start a session
  sess = tf.Session()
  
  # Set RNN Parameters
  min_word_freq = 5  # Trim the less frequent words off
  rnn_size = 1024  # RNN Model size, has to equal embedding size
  epochs = 10  # Number of epochs to cycle through data
  batch_size = 32  # Train on this many examples at once
  learning_rate = 0.001  # Learning rate
  training_seq_len = 11  # how long of a word group to consider
  embedding_size = rnn_size
  save_every = 500  # How often to save model checkpoints
  eval_every = 50  # How often to evaluate the test sentences
  prime_texts = ['1_yr_srv_0'] # EOD10_ND_3'] MAR_STAT_2', '1_yr_srv_0 MAR_STAT_1', '1_yr_srv_0 MAR_STAT_3', '1_yr_srv_0 MAR_STAT_9', '1_yr_srv_0 MAR_STAT_4', '1_yr_srv_0 MAR_STAT_5', '1_yr_srv_0 MAR_STAT_4', '1_yr_srv_0 MAR_STAT_2', '1_yr_srv_0 MAR_STAT_9', '1_yr_srv_0', '1_yr_srv_0 MAR_STAT_5']
  
  # Download/store Shakespeare data
  data_dir = 'temp'
  data_dir = 'data'
  data_file = 'shakespeare.txt'
  data_file = 'feature_paragraph.txt'
  model_path = 'shakespeare_model'
  full_model_dir = os.path.join(data_dir, model_path)
  
  # Declare punctuation to remove, everything except hyphens and apostrophes
  punctuation = string.punctuation
  punctuation = ''.join([x for x in punctuation if x not in ['-', "'"]])
  
  # Make Model Directory
  if not os.path.exists(full_model_dir):
      os.makedirs(full_model_dir)
  
  # Make data directory
  if not os.path.exists(data_dir):
      os.makedirs(data_dir)
  
  print('Loading Shakespeare Data')
  print('Loading the feature data')
  # Check if file is downloaded.
  if not os.path.isfile(os.path.join(data_dir, data_file)):
      print('Not found, downloading Shakespeare texts from www.gutenberg.org')
      shakespeare_url = 'http://www.gutenberg.org/cache/epub/100/pg100.txt'
      # Get Shakespeare text
      response = requests.get(shakespeare_url)
      shakespeare_file = response.content
      # Decode binary into string
      s_text = shakespeare_file.decode('utf-8')
      # Drop first few descriptive paragraphs.
      s_text = s_text[7675:]
      # Remove newlines
      s_text = s_text.replace('\r\n', '')
      s_text = s_text.replace('\n', '')
  
      # Write to file
      with open(os.path.join(data_dir, data_file), 'w') as out_conn:
          out_conn.write(s_text)
  else:
      print('opening file: ', data_file)
      # If file has been saved, load from that file
      with open(os.path.join(data_dir, data_file), 'r') as file_conn:
          s_text = file_conn.read().replace('\n', '')
  
  # Clean text
  print('first line: ', s_text[:100])
  
  print('1_yr_srv_0' in s_text)
  
  print('Cleaning Text')
  #s_text = re.sub(r'[{}]'.format(punctuation), ' ', s_text)
  #s_text = re.sub('\s+', ' ', s_text).strip().lower()
  
  print('first line: ', s_text[:100])
  print('1_yr_srv_0' in s_text)
  
  
  # Build word vocabulary function
  def build_vocab(text, min_word_freq):
      word_counts = collections.Counter(text.split(' '))
      print ('word count: ', len(word_counts), 'text len: ', len(text.split(' ')))
      # limit word counts to those more frequent than cutoff
      word_counts = {key: val for key, val in word_counts.items() if val > min_word_freq}
      # Create vocab --> index mapping
      words = word_counts.keys()
      vocab_to_ix_dict = {key: (ix + 1) for ix, key in enumerate(words)}
      # Add unknown key --> 0 index
      vocab_to_ix_dict['unknown'] = 0
      # Create index --> vocab mapping
      ix_to_vocab_dict = {val: key for key, val in vocab_to_ix_dict.items()}
      return (ix_to_vocab_dict, vocab_to_ix_dict)
  
  
  # Build Shakespeare vocabulary
  print('Building Vocab')
  ix2vocab, vocab2ix = build_vocab(s_text, min_word_freq)
  vocab_size = len(ix2vocab) + 1
  print('Vocabulary Length = {}'.format(vocab_size))
  # Sanity Check
  assert (len(ix2vocab) == len(vocab2ix))
  
  # Convert text to word vectors
  s_text_words = s_text.split(' ')
  s_text_ix = []
  for ix, x in enumerate(s_text_words):
      try:
          s_text_ix.append(vocab2ix[x])
      except:
          s_text_ix.append(0)
  s_text_ix = np.array(s_text_ix)
  
  # Define LSTM RNN Model
  class LSTM_Model():
      def __init__(self, rnn_size, batch_size, learning_rate,
                   training_seq_len, vocab_size, infer_sample=False):
          self.rnn_size = rnn_size
          self.vocab_size = vocab_size
          self.infer_sample = infer_sample
          self.learning_rate = learning_rate
  
          if infer_sample:
              self.batch_size = 1
              self.training_seq_len = 1
          else:
              self.batch_size = batch_size
              self.training_seq_len = training_seq_len
  
          self.lstm_cell = tf.contrib.rnn.core_rnn_cell.BasicLSTMCell(rnn_size)
          self.initial_state = self.lstm_cell.zero_state(self.batch_size, tf.float32)
  
          self.x_data = tf.placeholder(tf.int32, [self.batch_size, self.training_seq_len])
          self.y_output = tf.placeholder(tf.int32, [self.batch_size, self.training_seq_len])
  
          with tf.variable_scope('lstm_vars'):
              # Softmax Output Weights
              W = tf.get_variable('W', [self.rnn_size, self.vocab_size], tf.float32, tf.random_normal_initializer())
              b = tf.get_variable('b', [self.vocab_size], tf.float32, tf.constant_initializer(0.0))
  
              # Define Embedding
              embedding_mat = tf.get_variable('embedding_mat', [self.vocab_size, self.rnn_size],
                                              tf.float32, tf.random_normal_initializer())
              print('xdata:', self.x_data.get_shape())
              print('emb_mat: ', embedding_mat.get_shape())
              embedding_output = tf.nn.embedding_lookup(embedding_mat, self.x_data)
              print('emb_output: ', embedding_output.get_shape())
              rnn_inputs = tf.split(axis=1, num_or_size_splits=self.training_seq_len, value=embedding_output)
              print('rnninputs: ', len(rnn_inputs), rnn_inputs[0].get_shape())
              rnn_inputs_trimmed = [tf.squeeze(x, [1]) for x in rnn_inputs]
              print('rnninput trimmed:', len(rnn_inputs_trimmed), rnn_inputs_trimmed[0].get_shape())
  
              # If we are inferring (generating text), we add a 'loop' function
              # Define how to get the i+1 th input from the i th output
              def inferred_loop(prev, count):
                  # Apply hidden layer
                  prev_transformed = tf.matmul(prev, W) + b
                  # Get the index of the output (also don't run the gradient)
                  prev_symbol = tf.stop_gradient(tf.argmax(prev_transformed, 1))
                  # Get embedded vector
                  output = tf.nn.embedding_lookup(embedding_mat, prev_symbol)
                  return (output)
  
              decoder = tf.contrib.legacy_seq2seq.rnn_decoder
              outputs, last_state = decoder(rnn_inputs_trimmed,
                                            self.initial_state,
                                            self.lstm_cell,
                                            loop_function=inferred_loop if infer_sample else None)
              # Non inferred outputs
              output = tf.reshape(tf.concat(axis=1, values=outputs), [-1, self.rnn_size])
              # Logits and output
              self.logit_output = tf.matmul(output, W) + b
              self.model_output = tf.nn.softmax(self.logit_output)
  
              loss_fun = tf.contrib.legacy_seq2seq.sequence_loss_by_example
              loss = loss_fun([self.logit_output], [tf.reshape(self.y_output, [-1])],
                              [tf.ones([self.batch_size * self.training_seq_len])],
                              self.vocab_size)
              self.cost = tf.reduce_sum(loss) / (self.batch_size * self.training_seq_len)
              self.final_state = last_state
              gradients, _ = tf.clip_by_global_norm(tf.gradients(self.cost, tf.trainable_variables()), 4.5)
              optimizer = tf.train.AdamOptimizer(self.learning_rate)
              self.train_op = optimizer.apply_gradients(zip(gradients, tf.trainable_variables()))
  
      def sample(self, sess, words=ix2vocab, vocab=vocab2ix, num=10, prime_text='thou art'):
          state = sess.run(self.lstm_cell.zero_state(1, tf.float32))
          word_list = prime_text.split()
          for word in word_list[:-1]:
              x = np.zeros((1, 1))
              x[0, 0] = vocab[word]
              feed_dict = {self.x_data: x, self.initial_state: state}
              [state] = sess.run([self.final_state], feed_dict=feed_dict)
  
          out_sentence = prime_text
          word = word_list[-1]
          for n in range(num):
              x = np.zeros((1, 1))
              x[0, 0] = vocab[word]
              feed_dict = {self.x_data: x, self.initial_state: state}
              [model_output, state] = sess.run([self.model_output, self.final_state], feed_dict=feed_dict)
              sample = np.argmax(model_output[0])
              if sample == 0:
                  break
              word = words[sample]
              out_sentence = out_sentence + ' ' + word
          return (out_sentence)
  
  
  with tf.variable_scope('lstm_model') as scope:
      # Define LSTM Model
      lstm_model = LSTM_Model(rnn_size, batch_size, learning_rate,
                              training_seq_len, vocab_size)
      scope.reuse_variables()
      test_lstm_model = LSTM_Model(rnn_size, batch_size, learning_rate,
                                   training_seq_len, vocab_size, infer_sample=True)
  
  # Create model saver
  saver = tf.train.Saver(tf.global_variables())
  
  # Create batches for each epoch
  num_batches = int(len(s_text_ix) / (batch_size * training_seq_len)) + 1
  # Split up text indices into subarrays, of equal size
  batches = np.array_split(s_text_ix, num_batches)
  # Reshape each split into [batch_size, training_seq_len]
  batches = [np.resize(x, [batch_size, training_seq_len]) for x in batches]
  
  # Initialize all variables
  init = tf.global_variables_initializer()
  sess.run(init)
  
  # Train model
  train_loss = []
  iteration_count = 1
  for epoch in range(epochs):
      # Shuffle word indices
      random.shuffle(batches)
      # Create targets from shuffled batches
      targets = [np.roll(x, -1, axis=1) for x in batches]
      # Run a through one epoch
      print('Starting Epoch #{} of {}.'.format(epoch + 1, epochs))
      # Reset initial LSTM state every epoch
      state = sess.run(lstm_model.initial_state)
      for ix, batch in enumerate(batches):
          training_dict = {lstm_model.x_data: batch, lstm_model.y_output: targets[ix]}
          c, h = lstm_model.initial_state
          training_dict[c] = state.c
          training_dict[h] = state.h
  
          temp_loss, state, _ = sess.run([lstm_model.cost, lstm_model.final_state, lstm_model.train_op],
                                         feed_dict=training_dict)
          train_loss.append(temp_loss)
  
          # Print status every 10 gens
          if iteration_count % 10 == 0:
              summary_nums = (iteration_count, epoch + 1, ix + 1, num_batches + 1, temp_loss)
              print('Iteration: {}, Epoch: {}, Batch: {} out of {}, Loss: {:.2f}'.format(*summary_nums))
  
          # Save the model and the vocab
          if iteration_count % save_every == 0:
              # Save model
              model_file_name = os.path.join(full_model_dir, 'model')
              saver.save(sess, model_file_name, global_step=iteration_count)
              print('Model Saved To: {}'.format(model_file_name))
              # Save vocabulary
              dictionary_file = os.path.join(full_model_dir, 'vocab.pkl')
              with open(dictionary_file, 'wb') as dict_file_conn:
                  pickle.dump([vocab2ix, ix2vocab], dict_file_conn)
  
          if iteration_count % eval_every == 0:
              for sample in prime_texts:
                  print(test_lstm_model.sample(sess, ix2vocab, vocab2ix, num=10, prime_text=sample))
  
          iteration_count += 1
  
  # Plot loss over time
  plt.plot(train_loss, 'k-')
  plt.title('Sequence to Sequence Loss')
  plt.xlabel('Generation')
  plt.ylabel('Loss')
  plt.show()
  
apis: numpy.resize tensorflow.concat tensorflow.contrib.rnn.core_rnn_cell.BasicLSTMCell tensorflow.reduce_sum tensorflow.global_variables matplotlib.pyplot.plot tensorflow.train.AdamOptimizer numpy.roll tensorflow.squeeze numpy.argmax tensorflow.Session tensorflow.trainable_variables tensorflow.argmax tensorflow.random_normal_initializer numpy.zeros tensorflow.python.framework.ops.reset_default_graph tensorflow.matmul matplotlib.pyplot.title tensorflow.placeholder tensorflow.global_variables_initializer tensorflow.split numpy.array matplotlib.pyplot.show tensorflow.nn.embedding_lookup matplotlib.pyplot.ylabel tensorflow.nn.softmax tensorflow.reshape tensorflow.ones tensorflow.constant_initializer tensorflow.variable_scope matplotlib.pyplot.xlabel numpy.array_split
api_extract: [(22, 'tensorflow.python.framework.ops.reset_default_graph', 'ops.reset_default_graph', ([], {}), False, 'from tensorflow.python.framework import ops\n'), (25, 'tensorflow.Session', 'tf.Session', ([], {}), True, 'import tensorflow as tf\n'), (45, 'os.path.join', 'os.path.join', (['data_dir', 'model_path'], {}), False, 'import os\n'), (130, 'numpy.array', 'np.array', (['s_text_ix'], {}), True, 'import numpy as np\n'), (241, 'numpy.array_split', 'np.array_split', (['s_text_ix', 'num_batches'], {}), True, 'import numpy as np\n'), (246, 'tensorflow.global_variables_initializer', 'tf.global_variables_initializer', ([], {}), True, 'import tensorflow as tf\n'), (294, 'matplotlib.pyplot.plot', 'plt.plot', (['train_loss', '"""k-"""'], {}), True, 'import matplotlib.pyplot as plt\n'), (295, 'matplotlib.pyplot.title', 'plt.title', (['"""Sequence to Sequence Loss"""'], {}), True, 'import matplotlib.pyplot as plt\n'), (296, 'matplotlib.pyplot.xlabel', 'plt.xlabel', (['"""Generation"""'], {}), True, 'import matplotlib.pyplot as plt\n'), (297, 'matplotlib.pyplot.ylabel', 'plt.ylabel', (['"""Loss"""'], {}), True, 'import matplotlib.pyplot as plt\n'), (298, 'matplotlib.pyplot.show', 'plt.show', ([], {}), True, 'import matplotlib.pyplot as plt\n'), (52, 'os.path.exists', 'os.path.exists', (['full_model_dir'], {}), False, 'import os\n'), (53, 'os.makedirs', 'os.makedirs', (['full_model_dir'], {}), False, 'import os\n'), (56, 'os.path.exists', 'os.path.exists', (['data_dir'], {}), False, 'import os\n'), (57, 'os.makedirs', 'os.makedirs', (['data_dir'], {}), False, 'import os\n'), (66, 'requests.get', 'requests.get', (['shakespeare_url'], {}), False, 'import requests\n'), (227, 'tensorflow.variable_scope', 'tf.variable_scope', (['"""lstm_model"""'], {}), True, 'import tensorflow as tf\n'), (236, 'tensorflow.global_variables', 'tf.global_variables', ([], {}), True, 'import tensorflow as tf\n'), (243, 'numpy.resize', 'np.resize', (['x', '[batch_size, training_seq_len]'], {}), True, 'import numpy as np\n'), (254, 'random.shuffle', 'random.shuffle', (['batches'], {}), False, 'import random\n'), (62, 'os.path.join', 'os.path.join', (['data_dir', 'data_file'], {}), False, 'import os\n'), (148, 'tensorflow.contrib.rnn.core_rnn_cell.BasicLSTMCell', 'tf.contrib.rnn.core_rnn_cell.BasicLSTMCell', (['rnn_size'], {}), True, 'import tensorflow as tf\n'), (151, 'tensorflow.placeholder', 'tf.placeholder', (['tf.int32', '[self.batch_size, self.training_seq_len]'], {}), True, 'import tensorflow as tf\n'), (152, 'tensorflow.placeholder', 'tf.placeholder', (['tf.int32', '[self.batch_size, self.training_seq_len]'], {}), True, 'import tensorflow as tf\n'), (256, 'numpy.roll', 'np.roll', (['x', '(-1)'], {'axis': '(1)'}), True, 'import numpy as np\n'), (77, 'os.path.join', 'os.path.join', (['data_dir', 'data_file'], {}), False, 'import os\n'), (82, 'os.path.join', 'os.path.join', (['data_dir', 'data_file'], {}), False, 'import os\n'), (154, 'tensorflow.variable_scope', 'tf.variable_scope', (['"""lstm_vars"""'], {}), True, 'import tensorflow as tf\n'), (164, 'tensorflow.nn.embedding_lookup', 'tf.nn.embedding_lookup', (['embedding_mat', 'self.x_data'], {}), True, 'import tensorflow as tf\n'), (166, 'tensorflow.split', 'tf.split', ([], {'axis': '(1)', 'num_or_size_splits': 'self.training_seq_len', 'value': 'embedding_output'}), True, 'import tensorflow as tf\n'), (191, 'tensorflow.nn.softmax', 'tf.nn.softmax', (['self.logit_output'], {}), True, 'import tensorflow as tf\n'), (200, 'tensorflow.train.AdamOptimizer', 'tf.train.AdamOptimizer', (['self.learning_rate'], {}), True, 'import tensorflow as tf\n'), (207, 'numpy.zeros', 'np.zeros', (['(1, 1)'], {}), True, 'import numpy as np\n'), (215, 'numpy.zeros', 'np.zeros', (['(1, 1)'], {}), True, 'import numpy as np\n'), (219, 'numpy.argmax', 'np.argmax', (['model_output[0]'], {}), True, 'import numpy as np\n'), (279, 'os.path.join', 'os.path.join', (['full_model_dir', '"""model"""'], {}), False, 'import os\n'), (283, 'os.path.join', 'os.path.join', (['full_model_dir', '"""vocab.pkl"""'], {}), False, 'import os\n'), (156, 'tensorflow.random_normal_initializer', 'tf.random_normal_initializer', ([], {}), True, 'import tensorflow as tf\n'), (157, 'tensorflow.constant_initializer', 'tf.constant_initializer', (['(0.0)'], {}), True, 'import tensorflow as tf\n'), (161, 'tensorflow.random_normal_initializer', 'tf.random_normal_initializer', ([], {}), True, 'import tensorflow as tf\n'), (168, 'tensorflow.squeeze', 'tf.squeeze', (['x', '[1]'], {}), True, 'import tensorflow as tf\n'), (179, 'tensorflow.nn.embedding_lookup', 'tf.nn.embedding_lookup', (['embedding_mat', 'prev_symbol'], {}), True, 'import tensorflow as tf\n'), (188, 'tensorflow.concat', 'tf.concat', ([], {'axis': '(1)', 'values': 'outputs'}), True, 'import tensorflow as tf\n'), (190, 'tensorflow.matmul', 'tf.matmul', (['output', 'W'], {}), True, 'import tensorflow as tf\n'), (197, 'tensorflow.reduce_sum', 'tf.reduce_sum', (['loss'], {}), True, 'import tensorflow as tf\n'), (285, 'pickle.dump', 'pickle.dump', (['[vocab2ix, ix2vocab]', 'dict_file_conn'], {}), False, 'import pickle\n'), (175, 'tensorflow.matmul', 'tf.matmul', (['prev', 'W'], {}), True, 'import tensorflow as tf\n'), (177, 'tensorflow.argmax', 'tf.argmax', (['prev_transformed', '(1)'], {}), True, 'import tensorflow as tf\n'), (194, 'tensorflow.reshape', 'tf.reshape', (['self.y_output', '[-1]'], {}), True, 'import tensorflow as tf\n'), (195, 'tensorflow.ones', 'tf.ones', (['[self.batch_size * self.training_seq_len]'], {}), True, 'import tensorflow as tf\n'), (199, 'tensorflow.trainable_variables', 'tf.trainable_variables', ([], {}), True, 'import tensorflow as tf\n'), (201, 'tensorflow.trainable_variables', 'tf.trainable_variables', ([], {}), True, 'import tensorflow as tf\n')]