class_docstr: Class to build Q-Policies.
class_docstr_tok: ["Class","to","build","Q-Policies","."]
docstr: Builds a Q-Policy given a q_network.
  
  Args:
    time_step_spec: A `TimeStep` spec of the expected time_steps.
    action_spec: A nest of BoundedTensorSpec representing the actions.
    q_network: An instance of a `tf_agents.network.Network`, callable via
      `network(observation, step_type) -> (output, final_state)`.
    emit_log_probability: Whether to emit log-probs in info of `PolicyStep`.
    observation_and_action_constraint_splitter: A function used to process
      observations with action constraints. These constraints can indicate,
      for example, a mask of valid/invalid actions for a given state of the
      environment. The function takes in a full observation and returns a
      tuple consisting of 1) the part of the observation intended as input to
      the network and 2) the constraint. An example
      `observation_and_action_constraint_splitter` could be as simple as: ```
      def observation_and_action_constraint_splitter(observation): return
      observation['network_input'], observation['constraint'] ``` *Note*: when
      using `observation_and_action_constraint_splitter`, make sure the
      provided `q_network` is compatible with the network-specific half of the
      output of the `observation_and_action_constraint_splitter`. In
      particular, `observation_and_action_constraint_splitter` will be called
      on the observation before passing to the network. If
      `observation_and_action_constraint_splitter` is None, action constraints
      are not applied.
    validate_action_spec_and_network: If `True` (default), action_spec is
      checked to make sure it is a single scalar spec with a minimum of zero.
      Also validates that the network's output matches the spec.
    name: The name of this policy. All variables in this module will fall
      under that name. Defaults to the class name.
  
  Raises:
    ValueError: If `q_network.action_spec` exists and is not compatible with
      `action_spec`.
    NotImplementedError: If `action_spec` contains more than one
      `BoundedTensorSpec`.
docstr_tok: ["Builds","a","Q-Policy","given","a","q_network",".","Args",":","time_step_spec",":","A","`","TimeStep","`","spec","of","the","expected","time_steps",".","action_spec",":","A","nest","of","BoundedTensorSpec","representing","the","actions",".","q_network",":","An","instance","of","a","`","tf_agents.network.Network","`",",","callable","via","`","network","(","observation",",","step_type",")","-",">","(","output",",","final_state",")","`",".","emit_log_probability",":","Whether","to","emit","log-probs","in","info","of","`","PolicyStep","`",".","observation_and_action_constraint_splitter",":","A","function","used","to","process","observations","with","action","constraints",".","These","constraints","can","indicate",",","for","example",",","a","mask","of","valid\/invalid","actions","for","a","given","state","of","the","environment",".","The","function","takes","in","a","full","observation","and","returns","a","tuple","consisting","of","1",")","the","part","of","the","observation","intended","as","input","to","the","network","and","2",")","the","constraint",".","An","example","`","observation_and_action_constraint_splitter","`","could","be","as","simple","as",":","``","`","def","observation_and_action_constraint_splitter","(","observation",")",":","return","observation","[","'network_input","'","]",",","observation","[","'constraint","'","]","``","`","*","Note","*",":","when","using","`","observation_and_action_constraint_splitter","`",",","make","sure","the","provided","`","q_network","`","is","compatible","with","the","network-specific","half","of","the","output","of","the","`","observation_and_action_constraint_splitter","`",".","In","particular",",","`","observation_and_action_constraint_splitter","`","will","be","called","on","the","observation","before","passing","to","the","network",".","If","`","observation_and_action_constraint_splitter","`","is","None",",","action","constraints","are","not","applied",".","validate_action_spec_and_network",":","If","`","True","`","(","default",")",",","action_spec","is","checked","to","make","sure","it","is","a","single","scalar","spec","with","a","minimum","of","zero",".","Also","validates","that","the","network","'s","output","matches","the","spec",".","name",":","The","name","of","this","policy",".","All","variables","in","this","module","will","fall","under","that","name",".","Defaults","to","the","class","name",".","Raises",":","ValueError",":","If","`","q_network.action_spec","`","exists","and","is","not","compatible","with","`","action_spec","`",".","NotImplementedError",":","If","`","action_spec","`","contains","more","than","one","`","BoundedTensorSpec","`","."]
code: def __init__(
      self,
      time_step_spec: ts.TimeStep,
      action_spec: types.NestedTensorSpec,
      q_network: network.Network,
      emit_log_probability: bool = False,
      observation_and_action_constraint_splitter: Optional[
          types.Splitter
      ] = None,
      validate_action_spec_and_network: bool = True,
      name: Optional[Text] = None,
  ):
      """Builds a Q-Policy given a q_network.
  
      Args:
        time_step_spec: A `TimeStep` spec of the expected time_steps.
        action_spec: A nest of BoundedTensorSpec representing the actions.
        q_network: An instance of a `tf_agents.network.Network`, callable via
          `network(observation, step_type) -> (output, final_state)`.
        emit_log_probability: Whether to emit log-probs in info of `PolicyStep`.
        observation_and_action_constraint_splitter: A function used to process
          observations with action constraints. These constraints can indicate,
          for example, a mask of valid/invalid actions for a given state of the
          environment. The function takes in a full observation and returns a
          tuple consisting of 1) the part of the observation intended as input to
          the network and 2) the constraint. An example
          `observation_and_action_constraint_splitter` could be as simple as: ```
          def observation_and_action_constraint_splitter(observation): return
          observation['network_input'], observation['constraint'] ``` *Note*: when
          using `observation_and_action_constraint_splitter`, make sure the
          provided `q_network` is compatible with the network-specific half of the
          output of the `observation_and_action_constraint_splitter`. In
          particular, `observation_and_action_constraint_splitter` will be called
          on the observation before passing to the network. If
          `observation_and_action_constraint_splitter` is None, action constraints
          are not applied.
        validate_action_spec_and_network: If `True` (default), action_spec is
          checked to make sure it is a single scalar spec with a minimum of zero.
          Also validates that the network's output matches the spec.
        name: The name of this policy. All variables in this module will fall
          under that name. Defaults to the class name.
  
      Raises:
        ValueError: If `q_network.action_spec` exists and is not compatible with
          `action_spec`.
        NotImplementedError: If `action_spec` contains more than one
          `BoundedTensorSpec`.
      """
      action_spec = tensor_spec.from_spec(action_spec)
      time_step_spec = tensor_spec.from_spec(time_step_spec)
  
      network_action_spec = getattr(q_network, "action_spec", None)
  
      if network_action_spec is not None:
          action_spec = cast(tf.TypeSpec, action_spec)
          if not action_spec.is_compatible_with(network_action_spec):
              raise ValueError(
                  "action_spec must be compatible with q_network.action_spec; "
                  "instead got action_spec=%s, q_network.action_spec=%s"
                  % (action_spec, network_action_spec)
              )
  
      flat_action_spec = tf.nest.flatten(action_spec)
      if len(flat_action_spec) > 1:
          raise ValueError(
              "Only scalar actions are supported now, but action spec is: {}".format(
                  action_spec
              )
          )
      if validate_action_spec_and_network:
          spec = flat_action_spec[0]
          if spec.shape.rank > 0:
              raise ValueError(
                  "Only scalar actions are supported now, but action spec is: {}".format(
                      action_spec
                  )
              )
  
          if spec.minimum != 0:
              raise ValueError(
                  "Action specs should have minimum of 0, but saw: {0}".format(
                      spec
                  )
              )
  
          num_actions = spec.maximum - spec.minimum + 1
          # enable checking of dueling Q networks
          outputs = q_network.create_variables()
          iterable = (
              list(outputs) if isinstance(outputs, tuple) else [outputs]
          )
          for output in iterable:
              network_utils.check_single_floating_network_output(
                  output, (num_actions,), str(q_network)
              )
  
      # We need to maintain the flat action spec for dtype, shape and range.
      self._flat_action_spec = flat_action_spec[0]
  
      self._q_network = q_network
      super(QPolicy, self).__init__(
          time_step_spec,
          action_spec,
          policy_state_spec=q_network.state_spec,
          clip=False,
          emit_log_probability=emit_log_probability,
          observation_and_action_constraint_splitter=(
              observation_and_action_constraint_splitter
          ),
          name=name,
      )
  
file_path: agents/tf_agents/policies/q_policy.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  from typing import Optional, Text, cast
  import gin
  import tensorflow
  import tensorflow_probability
  from tf_agents.distributions import shifted_categorical
  from tf_agents.networks import network
  from tf_agents.networks import utils
  from tf_agents.policies import tf_policy
  from tf_agents.specs import tensor_spec
  from tf_agents.trajectories import policy_step
  from tf_agents.trajectories import time_step
  from tf_agents.typing import types
label_desc: Use image node_id 1 to create a new QPolicy object from inherited base classes: tf_policy with example: obj = QPolicy(time_step_spec, action_spec, q_network, emit_log_probability, observation_and_action_constraint_splitter, validate_action_spec_and_network, name)