mydeepctr/examples/fm/model.py
def create_optimizer(loss, init_lr):
    """Creates an optimizer training op."""
    global_step = tf.train.get_or_create_global_step()
    optimizer = tf.train.AdamOptimizer(init_lr)
    # optimizer = tf.train.FtrlOptimizer(init_lr)
    tvars = tf.trainable_variables()
    grads = tf.gradients(loss, tvars)


    # This is how the model was pre-trained.
    (grads, _) = tf.clip_by_global_norm(grads, clip_norm=1.0)
    # 梯度的计算是固定的，和优化器无关，优化器只是去自适应的确定学习率，所以操作流程是
    #     1. 梯度计算
    #     2. 梯度截断
    #     3. 优化器把梯度加到原变量中

    value_summaries = []
    for g, v in zip(grads, tvars):
        if g is not None:
            value_hist_summary = tf.summary.histogram(
                "{}/value/hist".format(v.name), v)
            value_summaries.append(value_hist_summary)


    grad_summaries = []
    for g, v in zip(grads, tvars):
        if g is not None:
            grad_hist_summary = tf.summary.histogram(
                "{}/grad/hist".format(v.name), g)
            sparsity_summary = tf.summary.scalar(
                "{}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g))
                # 稀疏度检验， 计算0元素占的比例， relu中梯度为0的比例应该很大
            grad_summaries.append(grad_hist_summary)
            grad_summaries.append(sparsity_summary)

    grad_summaries_merged = tf.summary.merge(grad_summaries)
    loss_summary = tf.summary.scalar("loss", loss)
    train_summary_op = tf.summary.merge([loss_summary, grad_summaries_merged])
    train_summary_op = tf.summary.merge([loss_summary, value_summaries])
    train_op = optimizer.apply_gradients(
        zip(grads, tvars), global_step=global_step)
    return (train_op, train_summary_op)
Creates an optimizer training op.
Creates an optimizer training op.
Creates an optimizer training op .
def create_optimizer(loss, init_lr):
    global_step = tf.train.get_or_create_global_step()
    optimizer = tf.train.AdamOptimizer(init_lr)
    tvars = tf.trainable_variables()
    grads = tf.gradients(loss, tvars)
    (grads, _) = tf.clip_by_global_norm(grads, clip_norm=1.0)
    value_summaries = []
    for g, v in zip(grads, tvars):
        if g is not None:
            value_hist_summary = tf.summary.histogram(
                "{}/value/hist".format(v.name), v)
            value_summaries.append(value_hist_summary)
    grad_summaries = []
    for g, v in zip(grads, tvars):
        if g is not None:
            grad_hist_summary = tf.summary.histogram(
                "{}/grad/hist".format(v.name), g)
            sparsity_summary = tf.summary.scalar(
                "{}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g))
            grad_summaries.append(grad_hist_summary)
            grad_summaries.append(sparsity_summary)
    grad_summaries_merged = tf.summary.merge(grad_summaries)
    loss_summary = tf.summary.scalar("loss", loss)
    train_summary_op = tf.summary.merge([loss_summary, grad_summaries_merged])
    train_summary_op = tf.summary.merge([loss_summary, value_summaries])
    train_op = optimizer.apply_gradients(
        zip(grads, tvars), global_step=global_step)
    return (train_op, train_summary_op)
def create_optimizer ( loss , init_lr ) : global_step = tf . train . get_or_create_global_step ( ) optimizer = tf . train . AdamOptimizer ( init_lr ) tvars = tf . trainable_variables ( ) grads = tf . gradients ( loss , tvars ) ( grads , _ ) = tf . clip_by_global_norm ( grads , clip_norm = 1.0 ) value_summaries = [ ] for g , v in zip ( grads , tvars ) : if g is not None : value_hist_summary = tf . summary . histogram ( "{}/value/hist" . format ( v . name ) , v ) value_summaries . append ( value_hist_summary ) grad_summaries = [ ] for g , v in zip ( grads , tvars ) : if g is not None : grad_hist_summary = tf . summary . histogram ( "{}/grad/hist" . format ( v . name ) , g ) sparsity_summary = tf . summary . scalar ( "{}/grad/sparsity" . format ( v . name ) , tf . nn . zero_fraction ( g ) ) grad_summaries . append ( grad_hist_summary ) grad_summaries . append ( sparsity_summary ) grad_summaries_merged = tf . summary . merge ( grad_summaries ) loss_summary = tf . summary . scalar ( "loss" , loss ) train_summary_op = tf . summary . merge ( [ loss_summary , grad_summaries_merged ] ) train_summary_op = tf . summary . merge ( [ loss_summary , value_summaries ] ) train_op = optimizer . apply_gradients ( zip ( grads , tvars ) , global_step = global_step ) return ( train_op , train_summary_op )
Creates an optimizer training op.
Creates an optimizer training op .
"""Creates an optimizer training op.""" # optimizer = tf.train.FtrlOptimizer(init_lr) # This is how the model was pre-trained. # 梯度的计算是固定的，和优化器无关，优化器只是去自适应的确定学习率，所以操作流程是 #     1. 梯度计算 #     2. 梯度截断 #     3. 优化器把梯度加到原变量中 # 稀疏度检验， 计算0元素占的比例， relu中梯度为0的比例应该很大
loss
init_lr