text: """
  @file: layers.py.py
  @time: 2020-12-02 19:32:42
  """
  from tensorflow.keras.layers import Layer
  from tensorflow.keras import initializers, regularizers, constraints
  from tensorflow.keras import backend as K
  
  
  class Attention(Layer):
      """
      Custom Keras attention layer
      Reference: https://www.kaggle.com/qqgeogor/keras-lstm-attention-glove840b-lb-0-043
      """
  
      def __init__(self, step_dim, W_regularizer=None, b_regularizer=None,
                   W_constraint=None, b_constraint=None, bias=True, **kwargs):
  
          self.supports_masking = True
  
          self.bias = bias
          self.step_dim = step_dim
          self.features_dim = None
          super(Attention, self).__init__(**kwargs)
  
          self.param_W = {
              'initializer': initializers.get('glorot_uniform'),
              'name': '{}_W'.format(self.name),
              'regularizer': regularizers.get(W_regularizer),
              'constraint': constraints.get(W_constraint)
          }
          self.W = None
  
          self.param_b = {
              'initializer': 'zero',
              'name': '{}_b'.format(self.name),
              'regularizer': regularizers.get(b_regularizer),
              'constraint': constraints.get(b_constraint)
          }
          self.b = None
  
      def build(self, input_shape):
          assert len(input_shape) == 3
  
          self.features_dim = input_shape[-1]
          self.W = self.add_weight(shape=(input_shape[-1],),
                                   **self.param_W)
  
          if self.bias:
              self.b = self.add_weight(shape=(input_shape[1],),
                                       **self.param_b)
  
          self.built = True
  
      def compute_mask(self, input, input_mask=None):
          return None
  
      def call(self, x, mask=None):
          step_dim = self.step_dim
          features_dim = self.features_dim
  
          eij = K.reshape(
              K.dot(K.reshape(x, (-1, features_dim)), K.reshape(self.W, (features_dim, 1))),
              (-1, step_dim))
  
          if self.bias:
              eij += self.b
          eij = K.tanh(eij)
          a = K.exp(eij)
  
          if mask is not None:
              a *= K.cast(mask, K.floatx())
  
          a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())
          a = K.expand_dims(a)
          weighted_input = x * a
          return K.sum(weighted_input, axis=1)
  
      def compute_output_shape(self, input_shape):
          return input_shape[0], self.features_dim
  