class_docstr: DQN learner.
  
  This is the learning component of a DQN agent. It takes a dataset as input
  and implements update functionality to learn from this dataset. Optionally
  it takes a replay client as well to allow for updating of priorities.
docstr: Initializes the learner.
  
  Args:
    network: the online Q network (the one being optimized)
    target_network: the target Q critic (which lags behind the online net).
    discount: discount to use for TD updates.
    importance_sampling_exponent: power to which importance weights are raised
      before normalizing.
    learning_rate: learning rate for the q-network update.
    target_update_period: number of learner steps to perform before updating
      the target networks.
    dataset: dataset to learn from, whether fixed or from a replay buffer (see
      `acme.datasets.reverb.make_reverb_dataset` documentation).
    max_abs_reward: Optional maximum absolute value for the reward.
    huber_loss_parameter: Quadratic-linear boundary for Huber loss.
    replay_client: client to replay to allow for updating priorities.
    counter: Counter object for (potentially distributed) counting.
    logger: Logger object for writing logs to.
    checkpoint: boolean indicating whether to checkpoint the learner.
    save_directory: string indicating where the learner should save
      checkpoints and snapshots.
    max_gradient_norm: used for gradient clipping.
docstr_tok: ["Initializes","the","learner",".","Args",":","network",":","the","online","Q","network","(","the","one","being","optimized",")","target_network",":","the","target","Q","critic","(","which","lags","behind","the","online","net",")",".","discount",":","discount","to","use","for","TD","updates",".","importance_sampling_exponent",":","power","to","which","importance","weights","are","raised","before","normalizing",".","learning_rate",":","learning","rate","for","the","q-network","update",".","target_update_period",":","number","of","learner","steps","to","perform","before","updating","the","target","networks",".","dataset",":","dataset","to","learn","from",",","whether","fixed","or","from","a","replay","buffer","(","see","`","acme.datasets.reverb.make_reverb_dataset","`","documentation",")",".","max_abs_reward",":","Optional","maximum","absolute","value","for","the","reward",".","huber_loss_parameter",":","Quadratic-linear","boundary","for","Huber","loss",".","replay_client",":","client","to","replay","to","allow","for","updating","priorities",".","counter",":","Counter","object","for","(","potentially","distributed",")","counting",".","logger",":","Logger","object","for","writing","logs","to",".","checkpoint",":","boolean","indicating","whether","to","checkpoint","the","learner",".","save_directory",":","string","indicating","where","the","learner","should","save","checkpoints","and","snapshots",".","max_gradient_norm",":","used","for","gradient","clipping","."]
code: def __init__(
      self,
      network: snt.Module,
      target_network: snt.Module,
      discount: float,
      importance_sampling_exponent: float,
      learning_rate: float,
      target_update_period: int,
      dataset: tf.data.Dataset,
      max_abs_reward: Optional[float] = 1.0,
      huber_loss_parameter: float = 1.0,
      replay_client: Optional[
          Union[reverb.Client, reverb.TFClient]
      ] = None,
      counter: Optional[counting.Counter] = None,
      logger: Optional[loggers.Logger] = None,
      checkpoint: bool = True,
      save_directory: str = "~/acme",
      max_gradient_norm: Optional[float] = None,
  ):
      """Initializes the learner.
  
      Args:
        network: the online Q network (the one being optimized)
        target_network: the target Q critic (which lags behind the online net).
        discount: discount to use for TD updates.
        importance_sampling_exponent: power to which importance weights are raised
          before normalizing.
        learning_rate: learning rate for the q-network update.
        target_update_period: number of learner steps to perform before updating
          the target networks.
        dataset: dataset to learn from, whether fixed or from a replay buffer (see
          `acme.datasets.reverb.make_reverb_dataset` documentation).
        max_abs_reward: Optional maximum absolute value for the reward.
        huber_loss_parameter: Quadratic-linear boundary for Huber loss.
        replay_client: client to replay to allow for updating priorities.
        counter: Counter object for (potentially distributed) counting.
        logger: Logger object for writing logs to.
        checkpoint: boolean indicating whether to checkpoint the learner.
        save_directory: string indicating where the learner should save
          checkpoints and snapshots.
        max_gradient_norm: used for gradient clipping.
      """
  
      # TODO(mwhoffman): stop allowing replay_client to be passed as a TFClient.
      # This is just here for backwards compatability for agents which reuse this
      # Learner and still pass a TFClient instance.
      if isinstance(replay_client, reverb.TFClient):
          # TODO(b/170419518): open source pytype does not understand this
          # isinstance() check because it does not have a way of getting precise
          # type information for pip-installed packages.
          replay_client = reverb.Client(
              replay_client._server_address
          )  # pytype: disable=attribute-error
  
      # Internalise agent components (replay buffer, networks, optimizer).
      # TODO(b/155086959): Fix type stubs and remove.
      self._iterator = iter(dataset)  # pytype: disable=wrong-arg-types
      self._network = network
      self._target_network = target_network
      self._optimizer = snt.optimizers.Adam(learning_rate)
      self._replay_client = replay_client
  
      # Make sure to initialize the optimizer so that its variables (e.g. the Adam
      # moments) are included in the state returned by the learner (which can then
      # be checkpointed and restored).
      self._optimizer._initialize(
          network.trainable_variables
      )  # pylint: disable= protected-access
  
      # Internalise the hyperparameters.
      self._discount = discount
      self._target_update_period = target_update_period
      self._importance_sampling_exponent = importance_sampling_exponent
      self._max_abs_reward = max_abs_reward
      self._huber_loss_parameter = huber_loss_parameter
      if max_gradient_norm is None:
          max_gradient_norm = (
              1e10  # A very large number. Infinity results in NaNs.
          )
      self._max_gradient_norm = tf.convert_to_tensor(max_gradient_norm)
  
      # Learner state.
      self._variables: List[List[tf.Tensor]] = [
          network.trainable_variables
      ]
      self._num_steps = tf.Variable(0, dtype=tf.int32)
  
      # Internalise logging/counting objects.
      self._counter = counter or counting.Counter()
      self._logger = logger or loggers.TerminalLogger(
          "learner", time_delta=1.0
      )
  
      # Create a snapshotter object.
      if checkpoint:
          self._snapshotter = tf2_savers.Snapshotter(
              objects_to_save={"network": network},
              directory=save_directory,
              time_delta_minutes=60.0,
          )
      else:
          self._snapshotter = None
  
      # Do not record timestamps until after the first learning step is done.
      # This is to avoid including the time it takes for actors to come online and
      # fill the replay buffer.
      self._timestamp = None
  
imports: import time
  from typing import Dict, List, Optional, Union
  import acme
  from acme import types
  from acme.adders import reverb
  from acme.tf import losses
  from acme.tf import savers
  from acme.tf import utils
  from acme.utils import counting
  from acme.utils import loggers
  import numpy
  import reverb
  import sonnet
  import tensorflow
  import trfl
label_desc: Use image node_id 1 to create a new DQNLearner object from inherited base classes: acme, tf2_savers with example: obj = DQNLearner(network, target_network, discount, importance_sampling_exponent, learning_rate, target_update_period, dataset, max_abs_reward, huber_loss_parameter, replay_client, counter, logger, checkpoint, save_directory, max_gradient_norm)