import tensorflow as tf

    # Bellman training error
    if double_q:
        q_max = gather_2d(q_target,tf.argmax(q_online_tp1,axis=1,output_type=tf.int32))
    else:
        q_max = tf.reduce_max(q_target,axis=1)
    target = rew_t_ph + gamma * q_max * (1.0 - done_mask_ph)
    q_t_act = gather_2d(q_online_t,act_t_ph)
    total_error = tf.reduce_mean(huber_loss(target - q_t_act))
    ######

    # construct optimization op (with gradient clipping)
    learning_rate = tf.placeholder(tf.float32, (), name="learning_rate")
    optimizer = optimizer_spec.constructor(learning_rate=learning_rate, **optimizer_spec.kwargs)
    train_fn = minimize_and_clip(optimizer, total_error,
                 var_list=q_func_vars, clip_val=grad_norm_clipping)

    # update_target_fn will be called periodically to copy Q network to target Q network
    update_target_fn = []
    for var, var_target in zip(sorted(q_func_vars,        key=lambda v: v.name),
                               sorted(target_q_func_vars, key=lambda v: v.name)):
        update_target_fn.append(var_target.assign(var))
    update_target_fn = tf.group(*update_target_fn)
