text: import keras
  import keras.backend as K
  from keras.models import Sequential
  from keras.layers import Dense, Dropout, Activation, Reshape, Flatten, LSTM, Dense, Dropout, Embedding, Bidirectional, GRU
  from keras.optimizers import Adam
  from keras import initializers, regularizers
  from keras import optimizers
  from keras.engine.topology import Layer
  from keras import constraints
  
  
  def dot_product(x, kernel):
      """
      Wrapper for dot product operation, in order to be compatible with both
      Theano and Tensorflow
      Args:
          x (): input
          kernel (): weights
      Returns:
      """
      if K.backend() == 'tensorflow':
          return K.squeeze(K.dot(x, K.expand_dims(kernel)), axis=-1)
      else:
          return K.dot(x, kernel)
  
  class AttentionWithContext(Layer):
      """
      Attention operation, with a context/query vector, for temporal data.
      Supports Masking.
      follows these equations:
  
      (1) u_t = tanh(W h_t + b)
      (2) \alpha_t = \frac{exp(u^T u)}{\sum_t(exp(u_t^T u))}, this is the attention weight
      (3) v_t = \alpha_t * h_t, v in time t
      # Input shape
          3D tensor with shape: `(samples, steps, features)`.
      # Output shape
          3D tensor with shape: `(samples, steps, features)`.
      """
  
      def __init__(self,
                   W_regularizer=None, u_regularizer=None, b_regularizer=None,
                   W_constraint=None, u_constraint=None, b_constraint=None,
                   bias=True, **kwargs):
  
          self.supports_masking = True
          self.init = initializers.get('glorot_uniform')
  
          self.W_regularizer = regularizers.get(W_regularizer)
          self.u_regularizer = regularizers.get(u_regularizer)
          self.b_regularizer = regularizers.get(b_regularizer)
  
          self.W_constraint = constraints.get(W_constraint)
          self.u_constraint = constraints.get(u_constraint)
          self.b_constraint = constraints.get(b_constraint)
  
          self.bias = bias
          super(AttentionWithContext, self).__init__(**kwargs)
  
      def build(self, input_shape):
          assert len(input_shape) == 3
  
          self.W = self.add_weight((input_shape[-1], input_shape[-1],),
                                   initializer=self.init,
                                   name='{}_W'.format(self.name),
                                   regularizer=self.W_regularizer,
                                   constraint=self.W_constraint)
          if self.bias:
              self.b = self.add_weight((input_shape[-1],),
                                       initializer='zero',
                                       name='{}_b'.format(self.name),
                                       regularizer=self.b_regularizer,
                                       constraint=self.b_constraint)
  
          self.u = self.add_weight((input_shape[-1],),
                                   initializer=self.init,
                                   name='{}_u'.format(self.name),
                                   regularizer=self.u_regularizer,
                                   constraint=self.u_constraint)
  
          super(AttentionWithContext, self).build(input_shape)
  
      def compute_mask(self, input, input_mask=None):
          # do not pass the mask to the next layers
          return None
  
      def call(self, x, mask=None):
          uit = dot_product(x, self.W)
  
          if self.bias:
              uit += self.b
  
          uit = K.tanh(uit)
          ait = dot_product(uit, self.u)
  
          a = K.exp(ait)
  
          # apply mask after the exp. will be re-normalized next
          if mask is not None:
              # Cast the mask to floatX to avoid float64 upcasting in theano
              a *= K.cast(mask, K.floatx())
  
          # in some cases especially in the early stages of training the sum may be almost zero and this results in NaN's. 
          # Should add a small epsilon as the workaround
          # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())
          a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())
  
          a = K.expand_dims(a)
          weighted_input = x * a
  
          return weighted_input
  
      def compute_output_shape(self, input_shape):
          return input_shape[0], input_shape[1], input_shape[2]
      
  class Addition(Layer):
      """
      This layer is supposed to add of all activation weight.
      We split this from AttentionWithContext to help us getting the activation weights
      follows this equation:
      (1) v = \sum_t(\alpha_t * h_t)
  
      # Input shape
          3D tensor with shape: `(samples, steps, features)`.
      # Output shape
          2D tensor with shape: `(samples, features)`.
      """
  
      def __init__(self, **kwargs):
          super(Addition, self).__init__(**kwargs)
  
      def build(self, input_shape):
          self.output_dim = input_shape[-1]
          super(Addition, self).build(input_shape)
  
      def call(self, x):
          return K.sum(x, axis=1)
  
      def compute_output_shape(self, input_shape):
          return (input_shape[0], self.output_dim)