class_docstr: An LSTM architecture for named entity recognition.
  Uses a character embedding layer followed by an LSTM to generate vector 
  representation from characters for each token.
  Then the character vector is concatenated with token embedding vector, 
  which is input to another LSTM  followed by a CRF layer.
code: def __init__(self, dataset, parameters):
      self.verbose = False
  
      # Placeholders for input, output and dropout
      self.input_token_indices = tf.placeholder(
          tf.int32, [None], name="input_token_indices"
      )
      self.input_label_indices_vector = tf.placeholder(
          tf.float32,
          [None, dataset.number_of_classes],
          name="input_label_indices_vector",
      )
      self.input_label_indices_flat = tf.placeholder(
          tf.int32, [None], name="input_label_indices_flat"
      )
      self.input_token_character_indices = tf.placeholder(
          tf.int32, [None, None], name="input_token_indices"
      )
      self.input_token_lengths = tf.placeholder(
          tf.int32, [None], name="input_token_lengths"
      )
      self.dropout_keep_prob = tf.placeholder(
          tf.float32, name="dropout_keep_prob"
      )
  
      # Internal parameters
      initializer = tf.contrib.layers.xavier_initializer()
  
      if parameters["use_character_lstm"]:
          # Character-level LSTM
          # Idea: reshape so that we have a tensor [number_of_token, max_token_length,
          # token_embeddings_size], which we pass to the LSTM
  
          # Character embedding layer
          with tf.variable_scope("character_embedding"):
              self.character_embedding_weights = tf.get_variable(
                  "character_embedding_weights",
                  shape=[
                      dataset.alphabet_size,
                      parameters["character_embedding_dimension"],
                  ],
                  initializer=initializer,
              )
              embedded_characters = tf.nn.embedding_lookup(
                  self.character_embedding_weights,
                  self.input_token_character_indices,
                  name="embedded_characters",
              )
              if self.verbose:
                  print(
                      "embedded_characters: {0}".format(
                          embedded_characters
                      )
                  )
              utils_tf.variable_summaries(
                  self.character_embedding_weights
              )
  
          # Character LSTM layer
          with tf.variable_scope("character_lstm") as vs:
              character_lstm_output = bidirectional_LSTM(
                  embedded_characters,
                  parameters["character_lstm_hidden_state_dimension"],
                  initializer,
                  sequence_length=self.input_token_lengths,
                  output_sequence=False,
              )
              self.character_lstm_variables = tf.get_collection(
                  tf.GraphKeys.TRAINABLE_VARIABLES, scope=vs.name
              )
  
      # Token embedding layer
      with tf.variable_scope("token_embedding"):
          self.token_embedding_weights = tf.get_variable(
              "token_embedding_weights",
              shape=[
                  dataset.vocabulary_size,
                  parameters["token_embedding_dimension"],
              ],
              initializer=initializer,
              trainable=not parameters["freeze_token_embeddings"],
          )
          embedded_tokens = tf.nn.embedding_lookup(
              self.token_embedding_weights, self.input_token_indices
          )
          utils_tf.variable_summaries(self.token_embedding_weights)
  
      # Concatenate character LSTM outputs and token embeddings
      if parameters["use_character_lstm"]:
          with tf.variable_scope(
              "concatenate_token_and_character_vectors"
          ):
              if self.verbose:
                  print("embedded_tokens: {0}".format(embedded_tokens))
  
              token_lstm_input = tf.concat(
                  [character_lstm_output, embedded_tokens],
                  axis=1,
                  name="token_lstm_input",
              )
  
              if self.verbose:
                  print(
                      "token_lstm_input: {0}".format(token_lstm_input)
                  )
      else:
          token_lstm_input = embedded_tokens
  
      # Add dropout
      with tf.variable_scope("dropout"):
          token_lstm_input_drop = tf.nn.dropout(
              token_lstm_input,
              self.dropout_keep_prob,
              name="token_lstm_input_drop",
          )
  
          if self.verbose:
              print(
                  "token_lstm_input_drop: {0}".format(
                      token_lstm_input_drop
                  )
              )
  
          # https://www.tensorflow.org/api_guides/python/contrib.rnn
          # Prepare data shape to match `rnn` function requirements
          # Current data input shape: (batch_size, n_steps, n_input)
          # Required shape: 'n_steps' tensors list of shape (batch_size, n_input)
          token_lstm_input_drop_expanded = tf.expand_dims(
              token_lstm_input_drop,
              axis=0,
              name="token_lstm_input_drop_expanded",
          )
  
          if self.verbose:
              print(
                  "token_lstm_input_drop_expanded: {0}".format(
                      token_lstm_input_drop_expanded
                  )
              )
  
      # Token LSTM layer
      with tf.variable_scope("token_lstm") as vs:
          token_lstm_output = bidirectional_LSTM(
              token_lstm_input_drop_expanded,
              parameters["token_lstm_hidden_state_dimension"],
              initializer,
              output_sequence=True,
          )
  
          token_lstm_output_squeezed = tf.squeeze(
              token_lstm_output, axis=0
          )
  
          self.token_lstm_variables = tf.get_collection(
              tf.GraphKeys.TRAINABLE_VARIABLES, scope=vs.name
          )
  
      # Needed only if Bidirectional LSTM is used for token level
      with tf.variable_scope("feedforward_after_lstm") as vs:
          W = tf.get_variable(
              "W",
              shape=[
                  2 * parameters["token_lstm_hidden_state_dimension"],
                  parameters["token_lstm_hidden_state_dimension"],
              ],
              initializer=initializer,
          )
  
          b = tf.Variable(
              tf.constant(
                  0.0,
                  shape=[
                      parameters["token_lstm_hidden_state_dimension"]
                  ],
              ),
              name="bias",
          )
  
          outputs = tf.nn.xw_plus_b(
              token_lstm_output_squeezed,
              W,
              b,
              name="output_before_tanh",
          )
          outputs = tf.nn.tanh(outputs, name="output_after_tanh")
          utils_tf.variable_summaries(W)
          utils_tf.variable_summaries(b)
          self.token_lstm_variables += tf.get_collection(
              tf.GraphKeys.TRAINABLE_VARIABLES, scope=vs.name
          )
  
      with tf.variable_scope("feedforward_before_crf") as vs:
          W = tf.get_variable(
              "W",
              shape=[
                  parameters["token_lstm_hidden_state_dimension"],
                  dataset.number_of_classes,
              ],
              initializer=initializer,
          )
  
          b = tf.Variable(
              tf.constant(0.0, shape=[dataset.number_of_classes]),
              name="bias",
          )
  
          scores = tf.nn.xw_plus_b(outputs, W, b, name="scores")
          self.unary_scores = scores
          self.predictions = tf.argmax(
              self.unary_scores, 1, name="predictions"
          )
          utils_tf.variable_summaries(W)
          utils_tf.variable_summaries(b)
          self.feedforward_variables = tf.get_collection(
              tf.GraphKeys.TRAINABLE_VARIABLES, scope=vs.name
          )
  
      # CRF layer
      if parameters["use_crf"]:
          with tf.variable_scope("crf") as vs:
              # Add start and end tokens
              small_score = -1000.0
              large_score = 0.0
              sequence_length = tf.shape(self.unary_scores)[0]
              unary_scores_with_start_and_end = tf.concat(
                  [
                      self.unary_scores,
                      tf.tile(
                          tf.constant(small_score, shape=[1, 2]),
                          [sequence_length, 1],
                      ),
                  ],
                  1,
              )
              start_unary_scores = [
                  [small_score] * dataset.number_of_classes
                  + [large_score, small_score]
              ]
              end_unary_scores = [
                  [small_score] * dataset.number_of_classes
                  + [small_score, large_score]
              ]
              self.unary_scores = tf.concat(
                  [
                      start_unary_scores,
                      unary_scores_with_start_and_end,
                      end_unary_scores,
                  ],
                  0,
              )
              start_index = dataset.number_of_classes
              end_index = dataset.number_of_classes + 1
              input_label_indices_flat_with_start_and_end = tf.concat(
                  [
                      tf.constant(start_index, shape=[1]),
                      self.input_label_indices_flat,
                      tf.constant(end_index, shape=[1]),
                  ],
                  0,
              )
  
              # Apply CRF layer
              sequence_length = tf.shape(self.unary_scores)[0]
              sequence_lengths = tf.expand_dims(
                  sequence_length, axis=0, name="sequence_lengths"
              )
              unary_scores_expanded = tf.expand_dims(
                  self.unary_scores,
                  axis=0,
                  name="unary_scores_expanded",
              )
              input_label_indices_flat_batch = tf.expand_dims(
                  input_label_indices_flat_with_start_and_end,
                  axis=0,
                  name="input_label_indices_flat_batch",
              )
  
              if self.verbose:
                  print(
                      "unary_scores_expanded: {0}".format(
                          unary_scores_expanded
                      )
                  )
                  print(
                      "input_label_indices_flat_batch: {0}".format(
                          input_label_indices_flat_batch
                      )
                  )
                  print(
                      "sequence_lengths: {0}".format(sequence_lengths)
                  )
  
              # https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/crf
              # Compute the log-likelihood of the gold sequences and keep the
              # transition params for inference at test time.
              self.transition_parameters = tf.get_variable(
                  "transitions",
                  shape=[
                      dataset.number_of_classes + 2,
                      dataset.number_of_classes + 2,
                  ],
                  initializer=initializer,
              )
              utils_tf.variable_summaries(self.transition_parameters)
              log_likelihood, _ = tf.contrib.crf.crf_log_likelihood(
                  unary_scores_expanded,
                  input_label_indices_flat_batch,
                  sequence_lengths,
                  transition_params=self.transition_parameters,
              )
  
              self.loss = tf.reduce_mean(
                  -log_likelihood, name="cross_entropy_mean_loss"
              )
              self.accuracy = tf.constant(1)
              self.crf_variables = tf.get_collection(
                  tf.GraphKeys.TRAINABLE_VARIABLES, scope=vs.name
              )
  
      # Do not use CRF layer
      else:
          with tf.variable_scope("crf") as vs:
              self.transition_parameters = tf.get_variable(
                  "transitions",
                  shape=[
                      dataset.number_of_classes + 2,
                      dataset.number_of_classes + 2,
                  ],
                  initializer=initializer,
              )
              utils_tf.variable_summaries(self.transition_parameters)
              self.crf_variables = tf.get_collection(
                  tf.GraphKeys.TRAINABLE_VARIABLES, scope=vs.name
              )
  
          # Calculate mean cross-entropy loss
          with tf.variable_scope("loss"):
              losses = tf.nn.softmax_cross_entropy_with_logits(
                  logits=self.unary_scores,
                  labels=self.input_label_indices_vector,
                  name="softmax",
              )
              self.loss = tf.reduce_mean(
                  losses, name="cross_entropy_mean_loss"
              )
          with tf.variable_scope("accuracy"):
              correct_predictions = tf.equal(
                  self.predictions,
                  tf.argmax(self.input_label_indices_vector, 1),
              )
              self.accuracy = tf.reduce_mean(
                  tf.cast(correct_predictions, "float"), name="accuracy"
              )
  
      self.define_training_procedure(parameters)
      self.summary_op = tf.summary.merge_all()
      # defaults to saving all variables
      self.saver = tf.train.Saver(
          max_to_keep=parameters["maximum_number_of_epochs"]
      )
  
file_path: NeuroNER/neuroner/entity_lstm.py
imports: import os
  import pickle
  import re
  import time
  import tensorflow
  from neuroner import utils
  from neuroner import utils_tf
  from neuroner import utils_nlp
label_desc: Use image node_id 1 to create a new EntityLSTM object from inherited base classes: object with example: obj = EntityLSTM(dataset, parameters)