def apply_main_layers(self, inputs, index):
    'T5的Dencoder主体是基于Self-Attention、Cross-Attention的模块\n        顺序：LN --> Att1 --> Add --> LN --> Att2 --> Add -->  LN --> FFN --> Add\n        '
    (c, x) = inputs
    z = self.layer_norm_conds[0]
    self_attention_name = ('Decoder-Transformer-%d-MultiHeadSelfAttention' % index)
    cross_attention_name = ('Decoder-Transformer-%d-MultiHeadCrossAttention' % index)
    feed_forward_name = ('Decoder-Transformer-%d-FeedForward' % index)
    attention_mask = self.compute_attention_bias(index)
    position_bias = self.compute_position_bias([x, c])
    xi = x
    x = self.apply(inputs=self.simplify([x, z]), layer=LayerNormalization, center=False, epsilon=1e-06, conditional=(z is not None), hidden_units=self.layer_norm_conds[1], hidden_activation=self.layer_norm_conds[2], hidden_initializer=self.initializer, name=('%s-Norm' % self_attention_name))
    x = self.apply(inputs=[x, x, x, attention_mask, position_bias[0]], layer=MultiHeadAttention, arguments={'a_bias': True, 'p_bias': 't5_relative'}, heads=self.num_attention_heads, head_size=self.attention_head_size, out_dim=self.hidden_size, key_size=self.attention_key_size, use_bias=False, attention_scale=False, kernel_initializer=self.initializer, name=self_attention_name)
    x = self.apply(inputs=x, layer=Dropout, rate=self.dropout_rate, name=('%s-Dropout' % self_attention_name))
    x = self.apply(inputs=[xi, x], layer=Add, name=('%s-Add' % self_attention_name))
    xi = x
    x = self.apply(inputs=self.simplify([x, z]), layer=LayerNormalization, center=False, epsilon=1e-06, conditional=(z is not None), hidden_units=self.layer_norm_conds[1], hidden_activation=self.layer_norm_conds[2], hidden_initializer=self.initializer, name=('%s-Norm' % cross_attention_name))
    x = self.apply(inputs=[x, c, c, position_bias[1]], layer=MultiHeadAttention, arguments={'a_bias': None, 'p_bias': 't5_relative'}, heads=self.num_attention_heads, head_size=self.attention_head_size, out_dim=self.hidden_size, key_size=self.attention_key_size, use_bias=False, attention_scale=False, kernel_initializer=self.initializer, name=cross_attention_name)
    x = self.apply(inputs=x, layer=Dropout, rate=self.dropout_rate, name=('%s-Dropout' % cross_attention_name))
    x = self.apply(inputs=[xi, x], layer=Add, name=('%s-Add' % cross_attention_name))
    xi = x
    x = self.apply(inputs=self.simplify([x, z]), layer=LayerNormalization, center=False, epsilon=1e-06, conditional=(z is not None), hidden_units=self.layer_norm_conds[1], hidden_activation=self.layer_norm_conds[2], hidden_initializer=self.initializer, name=('%s-Norm' % feed_forward_name))
    x = self.apply(inputs=x, layer=FeedForward, units=self.intermediate_size, activation=self.hidden_act, use_bias=False, kernel_initializer=self.initializer, name=feed_forward_name)
    x = self.apply(inputs=x, layer=Dropout, rate=self.dropout_rate, name=('%s-Dropout' % feed_forward_name))
    x = self.apply(inputs=[xi, x], layer=Add, name=('%s-Add' % feed_forward_name))
    return [c, x]
T5的Dencoder主体是基于Self-Attention、Cross-Attention的模块
顺序：LN --> Att1 --> Add --> LN --> Att2 --> Add -->  LN --> FFN --> Add
def apply_main_layers(self, inputs, index):
    'T5的Dencoder主体是基于Self-Attention、Cross-Attention的模块\n        顺序：LN --> Att1 --> Add --> LN --> Att2 --> Add -->  LN --> FFN --> Add\n        '
    (c, x) = inputs
    z = self.layer_norm_conds[0]
    self_attention_name = ('Decoder-Transformer-%d-MultiHeadSelfAttention' % index)
    cross_attention_name = ('Decoder-Transformer-%d-MultiHeadCrossAttention' % index)
    feed_forward_name = ('Decoder-Transformer-%d-FeedForward' % index)
    attention_mask = self.compute_attention_bias(index)
    position_bias = self.compute_position_bias([x, c])
    xi = x
    x = self.apply(inputs=self.simplify([x, z]), layer=LayerNormalization, center=False, epsilon=1e-06, conditional=(z is not None), hidden_units=self.layer_norm_conds[1], hidden_activation=self.layer_norm_conds[2], hidden_initializer=self.initializer, name=('%s-Norm' % self_attention_name))
    x = self.apply(inputs=[x, x, x, attention_mask, position_bias[0]], layer=MultiHeadAttention, arguments={'a_bias': True, 'p_bias': 't5_relative'}, heads=self.num_attention_heads, head_size=self.attention_head_size, out_dim=self.hidden_size, key_size=self.attention_key_size, use_bias=False, attention_scale=False, kernel_initializer=self.initializer, name=self_attention_name)
    x = self.apply(inputs=x, layer=Dropout, rate=self.dropout_rate, name=('%s-Dropout' % self_attention_name))
    x = self.apply(inputs=[xi, x], layer=Add, name=('%s-Add' % self_attention_name))
    xi = x
    x = self.apply(inputs=self.simplify([x, z]), layer=LayerNormalization, center=False, epsilon=1e-06, conditional=(z is not None), hidden_units=self.layer_norm_conds[1], hidden_activation=self.layer_norm_conds[2], hidden_initializer=self.initializer, name=('%s-Norm' % cross_attention_name))
    x = self.apply(inputs=[x, c, c, position_bias[1]], layer=MultiHeadAttention, arguments={'a_bias': None, 'p_bias': 't5_relative'}, heads=self.num_attention_heads, head_size=self.attention_head_size, out_dim=self.hidden_size, key_size=self.attention_key_size, use_bias=False, attention_scale=False, kernel_initializer=self.initializer, name=cross_attention_name)
    x = self.apply(inputs=x, layer=Dropout, rate=self.dropout_rate, name=('%s-Dropout' % cross_attention_name))
    x = self.apply(inputs=[xi, x], layer=Add, name=('%s-Add' % cross_attention_name))
    xi = x
    x = self.apply(inputs=self.simplify([x, z]), layer=LayerNormalization, center=False, epsilon=1e-06, conditional=(z is not None), hidden_units=self.layer_norm_conds[1], hidden_activation=self.layer_norm_conds[2], hidden_initializer=self.initializer, name=('%s-Norm' % feed_forward_name))
    x = self.apply(inputs=x, layer=FeedForward, units=self.intermediate_size, activation=self.hidden_act, use_bias=False, kernel_initializer=self.initializer, name=feed_forward_name)
    x = self.apply(inputs=x, layer=Dropout, rate=self.dropout_rate, name=('%s-Dropout' % feed_forward_name))
    x = self.apply(inputs=[xi, x], layer=Add, name=('%s-Add' % feed_forward_name))
    return [c, x]