import tensorflow as tf

    # These should hold all of the variables of the Q-function network and target network,
    # respectively. A convenient way to get these is to make use of TF's "scope" feature.
    # For example, you can create your Q-function network with the scope "q_func" like this:
    # <something> = q_func(obs_t_float, num_actions, scope="q_func", reuse=False)
    # And then you can obtain the variables like this:
    # q_func_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='q_func')
    # Older versions of TensorFlow may require using "VARIABLES" instead of "GLOBAL_VARIABLES"
    ######
    def q_online(obs_float):
        return q_func(obs_float,num_actions,scope="online_q_func",reuse=tf.AUTO_REUSE)

    # Q-function network and target network
    q_online_t = q_online(obs_t_float)
    q_online_tp1 = q_online(obs_tp1_float)
    q_func_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,scope='online_q_func')
    q_target = q_func(obs_tp1_float,num_actions,scope="target_q_func",reuse=False)
    target_q_func_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,scope='target_q_func')
    # Bellman training error
    if double_q:
        q_max = gather_2d(q_target,tf.argmax(q_online_tp1,axis=1,output_type=tf.int32))
    else:
        q_max = tf.reduce_max(q_target,axis=1)
    target = rew_t_ph + gamma * q_max * (1.0 - done_mask_ph)
    q_t_act = gather_2d(q_online_t,act_t_ph)
    total_error = tf.reduce_mean(huber_loss(target - q_t_act))
    ######

    # construct optimization op (with gradient clipping)
