def apply_embeddings(self, inputs):
    'GPT的embedding是token、position、segment三者embedding之和\n        跟BERT的主要区别是三者相加之后没有加LayerNormalization层。\n        '
    inputs = inputs[:]
    x = inputs.pop(0)
    if (self.segment_vocab_size > 0):
        s = inputs.pop(0)
    if self.custom_position_ids:
        p = inputs.pop(0)
    else:
        p = None
    x = self.apply(inputs=x, layer=Embedding, input_dim=self.vocab_size, output_dim=self.embedding_size, embeddings_initializer=self.initializer, mask_zero=True, name='Embedding-Token')
    if (self.segment_vocab_size > 0):
        if self.shared_segment_embeddings:
            name = 'Embedding-Token'
        else:
            name = 'Embedding-Segment'
        s = self.apply(inputs=s, layer=Embedding, input_dim=self.segment_vocab_size, output_dim=self.embedding_size, embeddings_initializer=self.initializer, name=name)
        x = self.apply(inputs=[x, s], layer=Add, name='Embedding-Token-Segment')
    x = self.apply(inputs=self.simplify([x, p]), layer=PositionEmbedding, input_dim=self.max_position, output_dim=self.embedding_size, merge_mode='add', hierarchical=self.hierarchical_position, embeddings_initializer=self.initializer, custom_position_ids=self.custom_position_ids, name='Embedding-Position')
    x = self.apply(inputs=x, layer=Dropout, rate=self.dropout_rate, name='Embedding-Dropout')
    if (self.embedding_size != self.hidden_size):
        x = self.apply(inputs=x, layer=Dense, units=self.hidden_size, kernel_initializer=self.initializer, name='Embedding-Mapping')
    return x
GPT的embedding是token、position、segment三者embedding之和
跟BERT的主要区别是三者相加之后没有加LayerNormalization层。
def apply_embeddings(self, inputs):
    'GPT的embedding是token、position、segment三者embedding之和\n        跟BERT的主要区别是三者相加之后没有加LayerNormalization层。\n        '
    inputs = inputs[:]
    x = inputs.pop(0)
    if (self.segment_vocab_size > 0):
        s = inputs.pop(0)
    if self.custom_position_ids:
        p = inputs.pop(0)
    else:
        p = None
    x = self.apply(inputs=x, layer=Embedding, input_dim=self.vocab_size, output_dim=self.embedding_size, embeddings_initializer=self.initializer, mask_zero=True, name='Embedding-Token')
    if (self.segment_vocab_size > 0):
        if self.shared_segment_embeddings:
            name = 'Embedding-Token'
        else:
            name = 'Embedding-Segment'
        s = self.apply(inputs=s, layer=Embedding, input_dim=self.segment_vocab_size, output_dim=self.embedding_size, embeddings_initializer=self.initializer, name=name)
        x = self.apply(inputs=[x, s], layer=Add, name='Embedding-Token-Segment')
    x = self.apply(inputs=self.simplify([x, p]), layer=PositionEmbedding, input_dim=self.max_position, output_dim=self.embedding_size, merge_mode='add', hierarchical=self.hierarchical_position, embeddings_initializer=self.initializer, custom_position_ids=self.custom_position_ids, name='Embedding-Position')
    x = self.apply(inputs=x, layer=Dropout, rate=self.dropout_rate, name='Embedding-Dropout')
    if (self.embedding_size != self.hidden_size):
        x = self.apply(inputs=x, layer=Dense, units=self.hidden_size, kernel_initializer=self.initializer, name='Embedding-Mapping')
    return x