class_docstr: Actor-Critic Policy Gradient.
  
  See https://arxiv.org/pdf/1602.01783.pdf for a description of algorithm.
code: def _set_up_policy_network_and_value_network(self):
      shape_of_inputs = (
          self._shape_of_inputs
          if self._preprocessor is None
          else self._preprocessor.output_shape()
      )
      self._input_variables = C.ops.input_variable(
          shape=shape_of_inputs, dtype=np.float32
      )
  
      # Set up policy network.
      if self._parameters.policy_representation == "nn":
          model = Models.feedforward_network(
              shape_of_inputs,
              self._num_actions,
              self._parameters.policy_network_hidden_layers,
              C.losses.cross_entropy_with_softmax,
              use_placeholder_for_input=True,
          )
      else:
          try:
              model_definition_function = self._import_method(
                  self._parameters.policy_representation
              )
              model = model_definition_function(
                  shape_of_inputs,
                  self._num_actions,
                  C.losses.cross_entropy_with_softmax,
                  use_placeholder_for_input=True,
              )
          except ValueError:
              raise ValueError(
                  'Unknown representation for policy: "{0}"'
                  "\n".format(self._parameters.policy_representation)
              )
  
      self._policy_network = model["f"]
      self._policy_network.replace_placeholder(self._input_variables)
      self._policy_network_output_variables = model["outputs"]
      # The weight is computed as part of the Actor-Critic algorithm.
      self._policy_network_weight_variables = C.ops.input_variable(
          shape=(1,), dtype=np.float32
      )
      self._policy_network_loss = (
          model["loss"] * self._policy_network_weight_variables
      )
  
      # Initialized from a saved model.
      if self._parameters.initial_policy_network:
          self._policy_network.restore(
              self._parameters.initial_policy_network
          )
  
      print(
          "Parameterized the agent's policy using neural networks "
          '"{0}" with {1} actions.\n'
          "".format(
              self._parameters.policy_representation, self._num_actions
          )
      )
  
      # Set up value network.
      if self._parameters.shared_representation:
          # For shared representation, policy pi and value function V share
          # all non-output layers. To use cross_entropy_with_softmax loss
          # from cntk, _policy_network defined here doesn't include softmax
          # output layer. Therefore _value_network becomes _policy_network
          # plus one additional linear output layer.
          self._value_network = C.layers.Dense(1, activation=None)(
              self._policy_network
          )
          self._value_network_output_variables = C.ops.input_variable(
              shape=(1,), dtype=np.float32
          )
          self._value_network_loss = C.losses.squared_error(
              self._value_network, self._value_network_output_variables
          )
      else:
          if self._parameters.value_function_representation == "nn":
              model = Models.feedforward_network(
                  shape_of_inputs,
                  1,  # value network outputs a scalar
                  self._parameters.value_network_hidden_layers,
                  use_placeholder_for_input=True,
              )
          else:
              try:
                  model_definition_function = self._import_method(
                      self._parameters.value_function_representation
                  )
                  model = model_definition_function(
                      shape_of_inputs,
                      1,  # value network outputs a scalar
                      use_placeholder_for_input=True,
                  )
              except ValueError:
                  raise ValueError(
                      'Unknown representation for value function: "{0}"'
                      "\n".format(
                          self._parameters.value_function_representation
                      )
                  )
  
          self._value_network = model["f"]
          self._value_network.replace_placeholder(self._input_variables)
          self._value_network_output_variables = model["outputs"]
          self._value_network_loss = model[
              "loss"
          ]  # squared_error by default
  
      combined_networks = C.ops.combine(
          [self._policy_network, self._value_network]
      )
      combined_loss = (
          self._policy_network_loss
          + self._parameters.regularization_weight
          * negative_of_entropy_with_softmax(self._policy_network)
          + self._parameters.relative_step_size
          * self._value_network_loss
      )
  
      # The learning rate will be updated later before each minibatch
      # training.
      # TODO: allow user to specify learner through config file.
      self._trainer = C.train.trainer.Trainer(
          combined_networks,
          (combined_loss, None),
          C.learners.adam(
              combined_networks.parameters,
              C.learners.learning_parameter_schedule_per_sample(
                  self._parameters.initial_eta
              ),
              momentum=C.learners.momentum_schedule(
                  self._parameters.momentum
              ),
              variance_momentum=C.learners.momentum_schedule(0.999),
              minibatch_size=C.learners.IGNORE,
          ),
      )
  
      print(
          "Parameterized the agent's value function using neural network "
          '"{0}".\n'.format(
              self._parameters.policy_representation
              if self._parameters.shared_representation
              else self._parameters.value_function_representation
          )
      )
  
imports: import cntk
  import numpy
  import ast
  from .agent import AgentBaseClass
  from .shared.cntk_utils import negative_of_entropy_with_softmax
  from .shared.models import Models
  from .shared.policy_gradient_parameters import PolicyGradientParameters
label_desc: Use image node_id 6 for calling the ActorCritic obj's underlying member method code with example usage: obj._set_up_policy_network_and_value_network() without return types