code: import tensorflow as tf
  from tensorflow.contrib import layers
  
  from config import MovieQAPath
  from raw_input import Input
  
  _mp = MovieQAPath()
  hp = {'emb_dim': 300, 'feat_dim': 512, 'dropout_rate': 0.1}
  
  
  def dropout(x, training):
      return tf.layers.dropout(x, hp['dropout_rate'], training=training)
  
  
  def l2_norm(x, axis=1):
      return tf.nn.l2_normalize(x, axis=axis)
  
  
  def unit_norm(x, dim=2):
      return layers.unit_norm(x, dim=dim, epsilon=1e-12)
  
  
  def l1_norm(x, axis=None, epsilon=1e-6, name=None):
      with tf.name_scope(name, "l1_normalize", [x]) as name:
          x = tf.convert_to_tensor(x, name="x")
          square_sum = tf.reduce_sum(x, axis, keepdims=True)
          x_inv_norm = tf.reciprocal(tf.maximum(square_sum, epsilon))
          return tf.multiply(x, x_inv_norm, name=name)
  
  
  def bhattacharyya_norm(x, axis=None, epsilon=1e-6, name=None):
      with tf.name_scope(name, "l1_normalize", [x]) as name:
          x = tf.convert_to_tensor(x, name="x")
          x = tf.sqrt(x)
          square_sum = tf.reduce_sum(x, axis, keepdims=True)
          x_inv_norm = tf.reciprocal(tf.maximum(square_sum, epsilon))
          return tf.multiply(x, x_inv_norm, name=name)
  
  
  class Model(object):
      def __init__(self, data, beta=0.0, training=False):
          self.data = data
          reg = layers.l2_regularizer(beta)
          # reg = layers.l1_regularizer(beta)
          initializer = tf.glorot_normal_initializer(seed=0)
          # constraint = tf.keras.constraints.NonNeg()
          group = 32
          skip = True
          norm = True
          drop = True
          with tf.variable_scope('Embedding_Linear'):
              self.raw_ques = self.data.ques
              self.raw_ans = self.data.ans
              # self.raw_subt = self.data.subt
              self.raw_subt = tf.boolean_mask(self.data.subt, tf.cast(self.data.spec, tf.bool))
              self.raw_ques = l2_norm(self.raw_ques)
              self.raw_ans = l2_norm(self.raw_ans)
              self.raw_subt = l2_norm(self.raw_subt)
  
              self.q_pass = tf.layers.dense(self.raw_ques, group, tf.nn.relu, True, initializer,
                                            kernel_regularizer=reg)
              self.q_pass = tf.reshape(self.q_pass, [group, 1, 1])
              self.q_pass = dropout(self.q_pass, training)
              self.a_pass = tf.layers.dense(self.raw_ques, group, tf.nn.relu, True, initializer,
                                            kernel_regularizer=reg)
              self.a_pass = tf.reshape(self.a_pass, [group, 1, 1])
              self.a_pass = dropout(self.a_pass, training)
              self.s_pass = tf.layers.dense(self.raw_ques, group, tf.nn.relu, True, initializer,
                                            kernel_regularizer=reg)
              self.s_pass = tf.reshape(self.s_pass, [group, 1, 1])
              self.s_pass = dropout(self.s_pass, training)
              # for i in range(5):
              #     self.group_pass = self.group_pass + \
              #                       tf.layers.dense(self.raw_ans[i], tf.nn.relu, True, initializer,
              #                                       kernel_regularizer=reg)
  
              self.ques = tf.layers.dense(self.raw_ques, hp['emb_dim'] * group, tf.nn.tanh, True, initializer,
                                          kernel_regularizer=reg)
              self.ques = tf.split(self.ques, group, axis=1)
              self.ques = tf.stack(self.ques)
              self.ques = dropout(self.ques, training)
              self.ques = tf.reduce_sum(self.ques * self.q_pass, axis=0)
  
              self.ans = tf.layers.dense(self.raw_ans, hp['emb_dim'] * group, tf.nn.tanh, True, initializer,
                                         kernel_regularizer=reg)
              self.ans = tf.split(self.ans, group, axis=1)
              self.ans = tf.stack(self.ans)
              self.ans = dropout(self.ans, training)
              self.ans = tf.reduce_sum(self.ans * self.a_pass, axis=0)
  
              self.subt = tf.layers.dense(self.raw_subt, hp['emb_dim'] * group, tf.nn.tanh, True, initializer,
                                          kernel_regularizer=reg)
              self.subt = tf.split(self.subt, group, axis=1)
              self.subt = tf.stack(self.subt)
              self.subt = dropout(self.subt, training)
              self.subt = tf.reduce_sum(self.subt * self.s_pass, axis=0)
  
              if skip:
                  self.ques = self.raw_ques + self.ques
                  self.ans = self.raw_ans + self.ans
                  self.subt = self.raw_subt + self.subt
  
              if norm:
                  self.ques = l2_norm(self.ques)
                  self.ans = l2_norm(self.ans)
                  self.subt = l2_norm(self.subt)
  
              if drop:
                  self.ques = dropout(self.ques, training)
                  self.ans = dropout(self.ans, training)
                  self.subt = dropout(self.subt, training)
  
          with tf.variable_scope('Response'):
              # (N, 1)
              self.sq = tf.matmul(self.subt, self.ques, transpose_b=True)
              self.sq = tf.nn.relu(self.sq)
              # self.sq = tf.nn.softmax(self.sq, 0)
              self.sq = dropout(self.sq, training)
              # self.sq = l2_norm(self.sq, 0)
              # self.sq = l1_norm(self.sq, 0)
  
              # (N, 5)
              self.sa = tf.matmul(self.subt, self.ans, transpose_b=True)
              self.sa = tf.nn.relu(self.sa)
              # self.sa = tf.nn.softmax(self.sa)
              self.sa = dropout(self.sa, training)
              # self.sa = l2_norm(self.sa, 0)
              # self.sa = l1_norm(self.sa, 0)
  
              # (N, 5)
              self.attn = self.sq + self.sa
              # self.attn = l1_norm(self.attn, axis=0)
              # (1, N, 5)
              self.attn = tf.expand_dims(self.attn, 0)
              # (5, N, 1)
              self.attn = tf.transpose(self.attn, [2, 1, 0])
              # (5, N, E_t)
              self.abs = tf.expand_dims(self.subt, 0) * self.attn
              # (5, E_t)
              self.abs = tf.reduce_sum(self.abs, axis=1)
              self.abs = l2_norm(self.abs, 1)
              # (5, 1)
              self.output = tf.reduce_sum(self.abs * self.ans, axis=1, keepdims=True)
              # (1, 5)
              self.output = tf.transpose(self.output)
  
  
  def main():
      data = Input(split='train', mode='subt')
      model = Model(data)
  
      for v in tf.global_variables():
          print(v)
      config = tf.ConfigProto(allow_soft_placement=True)
      config.gpu_options.allow_growth = True
      # config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1
      with tf.Session(config=config) as sess:
          sess.run([model.data.initializer, tf.global_variables_initializer()],
                   feed_dict=data.feed_dict)
  
          # q, a, s = sess.run([model.ques_enc, model.ans_enc, model.subt_enc])
          # print(q.shape, a.shape, s.shape)
          # a, b, c, d = sess.run(model.tri_word_encodes)
          # print(a, b, c, d)
          # print(a.shape, b.shape, c.shape, d.shape)
          a, b = sess.run([model.subt, model.output])
          print(a, b)
          print(a.shape, b.shape)
  
  
  if __name__ == '__main__':
      main()
   import tensorflow as tf
  
  import utils.model_utils as mu
  from model.basic_model import BasicModel
  from raw_input import Input
  
  
  class SetupModel(BasicModel):
      def __init__(self, attn, checkpoint):
          super(SetupModel, self).__init__()
          self.data = Input()
          self.output = None
          self.saver = None
          self.best_saver = None
          self.train_gv_summaries_op = None
          self.val_init_op_list = None
          self.train_init_op_list = None
          self.train_op_list = None
          self.train_summaries_op = None
          self.val_op_list = None
          self.attn = attn
          self.checkpoint = checkpoint
  
      def eval_setup(self):
          val_answer = tf.argmax(self.output, axis=1)
          val_accuracy, val_accuracy_update, val_accuracy_initializer \
              = mu.get_acc(self.data.gt, tf.argmax(self.output, axis=1), name='val_accuracy')
  
          val_summaries_op = tf.summary.scalar('val_accuracy', val_accuracy)
          self.val_init_op_list = [self.data.initializer, val_accuracy_initializer]
          self.val_op_list = [val_accuracy, val_accuracy_update, val_summaries_op]
  
      def train_setup(self):
          main_loss = mu.get_loss(self._hp['loss'], self.data.gt, self.output)
          regu_loss = tf.losses.get_regularization_loss()
  
          loss = main_loss + regu_loss
  
          train_answer = tf.argmax(self.output, axis=1)
          train_accuracy, train_accuracy_update, train_accuracy_initializer \
              = mu.get_acc(self.data.gt, train_answer, name='train_accuracy')
  
          global_step = tf.train.get_or_create_global_step()
  
          decay_step = int(self._hp['decay_epoch'] * len(self.data))
          learning_rate = mu.get_lr(self._hp['decay_type'], self._hp['learning_rate'], global_step,
                                    decay_step, self._hp['decay_rate'])
  
          optimizer = mu.get_opt(self._hp['opt'], learning_rate, decay_step)
  
          grads_and_vars = optimizer.compute_gradients(loss)
          # grads_and_vars = [(tf.clip_by_norm(grad, 0.01, axes=[0]), var) if grad is not None else (grad, var)
          #                   for grad, var in grads_and_vars ]
          gradients, variables = list(zip(*grads_and_vars))
          update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
          with tf.control_dependencies(update_ops):
              train_op = tf.group(optimizer.apply_gradients(grads_and_vars, global_step),
                                  train_accuracy_update)
  
          self.saver = tf.train.Saver(tf.global_variables())
          self.best_saver = tf.train.Saver(tf.global_variables())
  
          # Summary
          train_gv_summaries = []
          for idx, grad in enumerate(gradients):
              if grad is not None:
                  train_gv_summaries.append(tf.summary.histogram('gradients/' + variables[idx].name, grad))
                  train_gv_summaries.append(tf.summary.histogram(variables[idx].name, variables[idx]))
  
          train_summaries = [
              tf.summary.scalar('train_loss', loss),
              tf.summary.scalar('train_accuracy', train_accuracy),
              tf.summary.scalar('learning_rate', learning_rate)
          ]
          self.train_summaries_op = tf.summary.merge(train_summaries)
          self.train_gv_summaries_op = tf.summary.merge(train_gv_summaries + train_summaries)
  
          self.train_init_op_list = [self.data.initializer, train_accuracy_initializer]
  
          self.train_op_list = [train_op, loss, train_accuracy, global_step]
  
          # if attn:
          #     self.train_op_list += [self.train_model.sq, self.train_model.sa,
          #                            self.train_data.gt, self.train_answer]
          #     self.val_op_list += [self.val_model.sq, self.val_model.sa,
          #                          self.val_data.gt, self.val_answer]
  
apis: tensorflow.reshape tensorflow.variable_scope tensorflow.matmul tensorflow.name_scope tensorflow.convert_to_tensor tensorflow.split tensorflow.reduce_sum tensorflow.global_variables_initializer tensorflow.multiply tensorflow.contrib.layers.unit_norm tensorflow.contrib.layers.l2_regularizer tensorflow.transpose tensorflow.nn.relu tensorflow.stack tensorflow.expand_dims tensorflow.global_variables tensorflow.cast tensorflow.Session tensorflow.layers.dense tensorflow.layers.dropout tensorflow.ConfigProto tensorflow.nn.l2_normalize tensorflow.sqrt tensorflow.glorot_normal_initializer tensorflow.maximum
apis: tensorflow.summary.scalar tensorflow.summary.histogram tensorflow.get_collection tensorflow.losses.get_regularization_loss tensorflow.global_variables tensorflow.train.get_or_create_global_step tensorflow.argmax tensorflow.summary.merge tensorflow.control_dependencies