code: # -*- coding: utf-8 -*-
  #/usr/bin/python2
  '''
  June 2017 by kyubyong park. 
  kbpark.linguist@gmail.com.
  https://www.github.com/kyubyong/transformer
  '''
  
  from __future__ import print_function
  import tensorflow as tf
  import numpy as np
  
  def normalize(inputs, 
                epsilon = 1e-8,
                scope="ln",
                reuse=None):
      '''Applies layer normalization.
      
      Args:
        inputs: A tensor with 2 or more dimensions, where the first dimension has
          `batch_size`.
        epsilon: A floating number. A very small number for preventing ZeroDivision Error.
        scope: Optional scope for `variable_scope`.
        reuse: Boolean, whether to reuse the weights of a previous layer
          by the same name.
        
      Returns:
        A tensor with the same shape and data dtype as `inputs`.
      '''
      with tf.variable_scope(scope, reuse=reuse):
          inputs_shape = inputs.get_shape()
          params_shape = inputs_shape[-1:]
          #  Tensor("encoder/num_blocks_0/multihead_attention/ln/moments/mean:0", shape=(32, 10, 1), dtype=float32)
          mean, variance = tf.nn.moments(inputs, [-1], keep_dims=True)
          beta= tf.Variable(tf.zeros(params_shape))
          gamma = tf.Variable(tf.ones(params_shape))
          normalized = (inputs - mean) / ( (variance + epsilon) ** (.5) )
          outputs = gamma * normalized + beta
          
      return outputs
  
  def embedding(inputs, 
                vocab_size, 
                num_units, 
                zero_pad=True, 
                scale=True,
                scope="embedding", 
                reuse=None):
      '''Embeds a given tensor.
  
      data_load 之后产生的x,y的大小
      x，y的shape为[N,T]，N即batch_size的大小，T为最大句子长度maxlen，默认为10。
      这个之后应该是再加一个hidden_size (num_units)
  
      Args:
        inputs: A `Tensor` with type `int32` or `int64` containing the ids
           to be looked up in `lookup table`.
        vocab_size: An int. Vocabulary size.
        num_units: An int. Number of embedding hidden units.
        zero_pad: A boolean. If True, all the values of the fist row (id 0)
          should be constant zeros.
        scale: A boolean. If True. the outputs is multiplied by sqrt num_units.
        scope: Optional scope for `variable_scope`.
        reuse: Boolean, whether to reuse the weights of a previous layer
          by the same name.
  
      Returns:
        A `Tensor` with one more rank than inputs's. The last dimensionality
          should be `num_units`.
          
      For example,
      
      ```
      import tensorflow as tf
      
      inputs = tf.to_int32(tf.reshape(tf.range(2*3), (2, 3)))
      outputs = embedding(inputs, 6, 2, zero_pad=True)
      with tf.Session() as sess:
          sess.run(tf.global_variables_initializer())
          print sess.run(outputs)
      >>
      [[[ 0.          0.        ]
        [ 0.09754146  0.67385566]
        [ 0.37864095 -0.35689294]]
  
       [[-1.01329422 -1.09939694]
        [ 0.7521342   0.38203377]
        [-0.04973143 -0.06210355]]]
      ```
      
      ```
      import tensorflow as tf
      
      inputs = tf.to_int32(tf.reshape(tf.range(2*3), (2, 3)))
      outputs = embedding(inputs, 6, 2, zero_pad=False)
      with tf.Session() as sess:
          sess.run(tf.global_variables_initializer())
          print sess.run(outputs)
      >>
      [[[-0.19172323 -0.39159766]
        [-0.43212751 -0.66207761]
        [ 1.03452027 -0.26704335]]
  
       [[-0.11634696 -0.35983452]
        [ 0.50208133  0.53509563]
        [ 1.22204471 -0.96587461]]]    
      ```    
      '''
      with tf.variable_scope(scope, reuse=reuse):
          lookup_table = tf.get_variable('lookup_table',
                                         dtype=tf.float32,
                                         shape=[vocab_size, num_units],
                                         initializer=tf.contrib.layers.xavier_initializer())
          if zero_pad:
              lookup_table = tf.concat((tf.zeros(shape=[1, num_units]),
                                        lookup_table[1:, :]), 0)
          # todo: embedding_lookup
          outputs = tf.nn.embedding_lookup(lookup_table, inputs)
          
          if scale:
              outputs = outputs * (num_units ** 0.5) 
              
      return outputs
      
  
  def positional_encoding(inputs,
                          num_units,
                          zero_pad=True,
                          scale=True,
                          scope="positional_encoding",
                          reuse=None):
      '''Sinusoidal Positional_Encoding.
      Equivalent to adding a constant matrix each time
      Args:
        inputs: A 2d Tensor with shape of (N, T).
        num_units: Output dimensionality
        zero_pad: Boolean. If True, all the values of the first row (id = 0) should be constant zero
        scale: Boolean. If True, the output will be multiplied by sqrt num_units(check details from paper)
        scope: Optional scope for `variable_scope`.
        reuse: Boolean, whether to reuse the weights of a previous layer
          by the same name.
  
      Returns:
          A 'Tensor' with one more rank than inputs's, with the dimensionality should be 'num_units'
      '''
  
      N, T = inputs.get_shape().as_list()
      with tf.variable_scope(scope, reuse=reuse):
          #    [1,T] [[0, 1, 2, ..., T]] -> [N, T]
          position_ind = tf.tile(tf.expand_dims(tf.range(T), 0), [N, 1])
  
          # First part of the PE function: sin and cos argument
          #    [T, num_units] sin(pos/10000^(2i/dmodel) )
          position_enc = np.array([
              [pos / np.power(10000, 2.*i/num_units) for i in range(num_units)]
              for pos in range(T)])
  
          # Second part, apply the cosine to even columns and sin to odds. 奇数偶数
          position_enc[:, 0::2] = np.sin(position_enc[:, 0::2])  # dim 2i
          position_enc[:, 1::2] = np.cos(position_enc[:, 1::2])  # dim 2i+1
  
          # Convert to a tensor
          lookup_table = tf.convert_to_tensor(position_enc)
  
          if zero_pad:
              lookup_table = tf.concat((tf.zeros(shape=[1, num_units]),
                                        lookup_table[1:, :]), 0)
          outputs = tf.nn.embedding_lookup(lookup_table, position_ind)
  
          if scale:
              outputs = outputs * num_units**0.5
  
          return outputs
  
  
  
  def multihead_attention(queries, 
                          keys, 
                          num_units=None, 
                          num_heads=8, 
                          dropout_rate=0,
                          is_training=True,
                          causality=False,
                          scope="multihead_attention", 
                          reuse=None):
      '''Applies multihead attention.
  
      data_load 输出
      x，y的shape为[N,T]，N即batch_size的大小，T为最大句子长度maxlen，默认为10。
      Form libei
      Input Embedding: Batch * Length * Hidden
      Posional Encoding : Length * Hidden
      Encoder Input = Input Embedding + Posional Encoding
      Query,Key,Value = Conv(Encoder Input,Hidden,3*Hidden)
  
      Query = SplitHeads(Query)
      Key = SplitHeads(Key)
      Value = SplitHeads(Value)
  
      Shape (Batch,Length,Heads,Channels)
  
  
      Args:
        queries: A 3d tensor with shape of [N, T_q, C_q].
        keys: A 3d tensor with shape of [N, T_k, C_k].
        num_units: A scalar. Attention size.
        dropout_rate: A floating point number.
        is_training: Boolean. Controller of mechanism for dropout.
        causality: Boolean. If true, units that reference the future are masked. 
        num_heads: An int. Number of heads.
        scope: Optional scope for `variable_scope`.
        reuse: Boolean, whether to reuse the weights of a previous layer
          by the same name.
          
      Returns
        A 3d tensor with shape of (N, T_q, C)  
      '''
      with tf.variable_scope(scope, reuse=reuse):
          # Set the fall back option for num_units
          if num_units is None:
              num_units = queries.get_shape().as_list[-1]
          
          # Linear projections Q、K、V 使用不同的矩阵进行映射学习
          # 新版代码使用一个大矩阵然后split为三份
          # self attention 在 encoder-decoder attention时候Q来自encoder K、V来自decoder; encoder或decoder时候 Q、K、V一样
          Q = tf.layers.dense(queries, num_units, activation=tf.nn.relu) # (N, T_q, C)
          K = tf.layers.dense(keys, num_units, activation=tf.nn.relu) # (N, T_k, C)
          V = tf.layers.dense(keys, num_units, activation=tf.nn.relu) # (N, T_k, C)
          
          # Split and concat
          # 分为8头
          Q_ = tf.concat(tf.split(Q, num_heads, axis=2), axis=0) # (h*N, T_q, C/h)
          K_ = tf.concat(tf.split(K, num_heads, axis=2), axis=0) # (h*N, T_k, C/h) 
          V_ = tf.concat(tf.split(V, num_heads, axis=2), axis=0) # (h*N, T_k, C/h) 
  
          # Multiplication                             (h*batch,length_q,length_k)
          outputs = tf.matmul(Q_, tf.transpose(K_, [0, 2, 1])) # (h*N, T_q, T_k)
  
          # Scale
          outputs = outputs / (K_.get_shape().as_list()[-1] ** 0.5)
  
          # Key Masking
              # N, T padding 0
          key_masks = tf.sign(tf.abs(tf.reduce_sum(keys, axis=-1))) # (N, T_k)
          key_masks = tf.tile(key_masks, [num_heads, 1]) # (h*N, T_k)
          key_masks = tf.tile(tf.expand_dims(key_masks, 1), [1, tf.shape(queries)[1], 1]) # (h*N, T_q, T_k)
          # tf.expand_dims(input, dim, name=None) add 1 at dim
          paddings = tf.ones_like(outputs)*(-2**32+1)
          outputs = tf.where(tf.equal(key_masks, 0), paddings, outputs) # (h*N, T_q, T_k)
  
          # Causality = Future blinding
          if causality:
              diag_vals = tf.ones_like(outputs[0, :, :]) # (T_q, T_k)
              tril = tf.contrib.linalg.LinearOperatorTriL(diag_vals).to_dense() # (T_q, T_k)  triangular matrix
              masks = tf.tile(tf.expand_dims(tril, 0), [tf.shape(outputs)[0], 1, 1]) # (h*N, T_q, T_k)
     
              paddings = tf.ones_like(masks)*(-2**32+1)
              outputs = tf.where(tf.equal(masks, 0), paddings, outputs) # (h*N, T_q, T_k)
    
          # Activation
          outputs = tf.nn.softmax(outputs) # (h*N, T_q, T_k)
           
          # Query Masking
          query_masks = tf.sign(tf.abs(tf.reduce_sum(queries, axis=-1))) # (N, T_q)
          query_masks = tf.tile(query_masks, [num_heads, 1]) # (h*N, T_q)
          query_masks = tf.tile(tf.expand_dims(query_masks, -1), [1, 1, tf.shape(keys)[1]]) # (h*N, T_q, T_k)
          outputs *= query_masks # broadcasting. (N, T_q, C)  should be  (h*N, T_q, T_k)。
            
          # Dropouts
          outputs = tf.layers.dropout(outputs, rate=dropout_rate, training=tf.convert_to_tensor(is_training))
                 
          # Weighted sum
          outputs = tf.matmul(outputs, V_) # ( h*N, T_q, C/h)
          # 少了个线性变换
          # Restore shape
          outputs = tf.concat(tf.split(outputs, num_heads, axis=0), axis=2 ) # (N, T_q, C)
                
          # Residual connection
          outputs += queries
                
          # Normalize
          outputs = normalize(outputs) # (N, T_q, C)
   
      return outputs
  
  def feedforward(inputs, 
                  num_units=[2048, 512],
                  scope="multihead_attention", 
                  reuse=None):
      '''Point-wise feed forward net.
      
      Args:
        inputs: A 3d tensor with shape of [N, T, C].
        num_units: A list of two integers.
        scope: Optional scope for `variable_scope`.
        reuse: Boolean, whether to reuse the weights of a previous layer
          by the same name.
          
      Returns:
        A 3d tensor with the same shape and dtype as inputs
      '''
      with tf.variable_scope(scope, reuse=reuse):
          # Inner layer
          params = {"inputs": inputs, "filters": num_units[0], "kernel_size": 1,
                    "activation": tf.nn.relu, "use_bias": True}
          outputs = tf.layers.conv1d(**params)
          
          # Readout layer
          params = {"inputs": outputs, "filters": num_units[1], "kernel_size": 1,
                    "activation": None, "use_bias": True}
          outputs = tf.layers.conv1d(**params)
          
          # Residual connection
          outputs += inputs
          
          # Normalize
          outputs = normalize(outputs)
      
      return outputs
  
  def label_smoothing(inputs, epsilon=0.1):
      '''Applies label smoothing. See https://arxiv.org/abs/1512.00567.
      
      Args:
        inputs: A 3d tensor with shape of [N, T, V], where V is the number of vocabulary.
        epsilon: Smoothing rate.
      
      For example,
      
      ```
      import tensorflow as tf
      inputs = tf.convert_to_tensor([[[0, 0, 1], 
         [0, 1, 0],
         [1, 0, 0]],
  
        [[1, 0, 0],
         [1, 0, 0],
         [0, 1, 0]]], tf.float32)
         
      outputs = label_smoothing(inputs)
      
      with tf.Session() as sess:
          print(sess.run([outputs]))
      
      >>
      [array([[[ 0.03333334,  0.03333334,  0.93333334],
          [ 0.03333334,  0.93333334,  0.03333334],
          [ 0.93333334,  0.03333334,  0.03333334]],
  
         [[ 0.93333334,  0.03333334,  0.03333334],
          [ 0.93333334,  0.03333334,  0.03333334],
          [ 0.03333334,  0.93333334,  0.03333334]]], dtype=float32)]   
      ```    
      '''
      K = inputs.get_shape().as_list()[-1] # number of channels
      return ((1-epsilon) * inputs) + (epsilon / K)
      
      
  
              
  
apis: tensorflow.ones tensorflow.variable_scope tensorflow.matmul tensorflow.convert_to_tensor tensorflow.nn.softmax tensorflow.split tensorflow.reduce_sum numpy.cos tensorflow.transpose tensorflow.layers.conv1d tensorflow.shape tensorflow.ones_like tensorflow.expand_dims numpy.power tensorflow.contrib.layers.xavier_initializer tensorflow.tile tensorflow.layers.dense tensorflow.nn.embedding_lookup tensorflow.nn.moments tensorflow.zeros tensorflow.equal tensorflow.contrib.linalg.LinearOperatorTriL tensorflow.range numpy.sin