code: def __init__(
      self,
      *,
      states,
      actions,
      max_episode_timesteps,
      policy,
      memory,
      update,
      optimizer,
      objective,
      reward_estimation,
      baseline,
      baseline_optimizer,
      baseline_objective,
      l2_regularization,
      entropy_regularization,
      state_preprocessing,
      exploration,
      variable_noise,
      parallel_interactions,
      config,
      saver,
      summarizer,
      tracking,
  ):
      super().__init__(
          states=states,
          actions=actions,
          l2_regularization=l2_regularization,
          parallel_interactions=parallel_interactions,
          config=config,
          saver=saver,
          summarizer=summarizer,
          tracking=tracking,
      )
  
      if max_episode_timesteps is None:
          self.max_episode_timesteps = None
      else:
          self.max_episode_timesteps = int(max_episode_timesteps)
  
      # State preprocessing
      self.processed_states_spec = TensorsSpec()
      self.state_preprocessing = ModuleDict()
      if state_preprocessing == "linear_normalization":
          # Default handling, otherwise layer will be applied to all input types
          state_preprocessing = {
              name: ["linear_normalization"]
              for name, spec in self.states_spec.items()
              if spec.type == "float"
              and spec.min_value is not None
              and spec.max_value is not None
          }
      if not isinstance(state_preprocessing, dict) or any(
          name not in self.states_spec for name in state_preprocessing
      ):
          state_preprocessing = {
              name: state_preprocessing for name in self.states_spec
          }
  
      for name, spec in self.states_spec.items():
          if name in state_preprocessing:
              layers = state_preprocessing[name]
          elif spec.type in state_preprocessing:
              layers = state_preprocessing[spec.type]
          else:
              layers = None
  
          if layers is None:
              self.processed_states_spec[name] = self.states_spec[name]
          else:
              if name is None:
                  module_name = "state_preprocessing"
              else:
                  module_name = name + "_preprocessing"
              self.state_preprocessing[name] = self.submodule(
                  name=module_name,
                  module=Preprocessor,
                  is_trainable=False,
                  input_spec=spec,
                  layers=layers,
              )
              spec = self.state_preprocessing[name].output_spec()
              self.processed_states_spec[name] = spec
  
          if (
              spec.type == "float"
              and spec.min_value is not None
              and spec.max_value is not None
          ):
              if isinstance(spec.min_value, float):
                  if not (-10.0 <= spec.min_value < 0.0) or not (
                      0.0 < spec.max_value <= 10.0
                  ):
                      logging.warning(
                          "{}tate{} does not seem to be normalized, consider "
                          "adding linear_normalization preprocessing.".format(
                              "S"
                              if layers is None
                              else "Preprocessed s",
                              "" if name is None else " " + name,
                          )
                      )
              else:
                  # TODO: missing +/-10.0 check, but cases of values +/-inf are already covered by
                  # previous no-bound warning
                  if (spec.min_value >= 0.0).any() or (
                      spec.max_value <= 0.0
                  ).any():
                      logging.warning(
                          "{}tate{} does not seem to be normalized, consider "
                          "adding linear_normalization preprocessing.".format(
                              "S"
                              if layers is None
                              else "Preprocessed s",
                              "" if name is None else " " + name,
                          )
                      )
  
      # Action exploration
      if exploration is None:
          exploration = 0.0
      if isinstance(exploration, dict) and all(
          name in self.actions_spec for name in exploration
      ):
          # Different exploration per action
          self.exploration = ModuleDict()
          for name, spec in self.actions_spec.items():
              if name in exploration:
                  module = exploration[name]
              elif spec.type in exploration:
                  module = exploration[spec.type]
              else:
                  module = None
              if module is None:
                  pass
              elif spec.type in ("bool", "int"):
                  self.exploration[name] = self.submodule(
                      name=(name + "_exploration"),
                      module=module,
                      modules=parameter_modules,
                      is_trainable=False,
                      dtype="float",
                      min_value=0.0,
                      max_value=1.0,
                  )
              else:
                  self.exploration[name] = self.submodule(
                      name=(name + "_exploration"),
                      module=module,
                      modules=parameter_modules,
                      is_trainable=False,
                      dtype="float",
                      min_value=0.0,
                  )
      else:
          # Same exploration for all actions
          self.exploration = self.submodule(
              name="exploration",
              module=exploration,
              modules=parameter_modules,
              is_trainable=False,
              dtype="float",
              min_value=0.0,
          )
  
      # Variable noise
      if variable_noise is None:
          variable_noise = 0.0
      self.variable_noise = self.submodule(
          name="variable_noise",
          module=variable_noise,
          modules=parameter_modules,
          is_trainable=False,
          dtype="float",
          min_value=0.0,
      )
  
      # Reward estimation argument check
      if not all(
          key
          in (
              "advantage_processing",
              "discount",
              "estimate_advantage",
              "gae_decay",
              "horizon",
              "predict_action_values",
              "predict_horizon_values",
              "predict_terminal_values",
              "return_processing",
              "reward_processing",
              "trace_decay",
          )
          for key in reward_estimation
      ):
          raise TensorforceError.value(
              name="agent",
              argument="reward_estimation",
              value=reward_estimation,
              hint="not from {advantage_processing,discount,estimate_advantage,gae_decay,"
              "horizon,predict_action_values,predict_horizon_values,predict_terminal_values,"
              "return_processing,reward_processing,trace_decay}",
          )
  
      # Reward estimation
      self.estimate_advantage = reward_estimation.get(
          "estimate_advantage", False
      )
      self.predict_horizon_values = reward_estimation.get(
          "predict_horizon_values"
      )
      self.predict_action_values = reward_estimation.get(
          "predict_action_values", False
      )
      self.predict_terminal_values = reward_estimation.get(
          "predict_terminal_values", False
      )
  
      # Return horizon
      if reward_estimation["horizon"] == "episode":
          self.reward_horizon = "episode"
          if self.predict_horizon_values is None:
              self.predict_horizon_values = "early"
          elif self.predict_horizon_values == "late":
              raise TensorforceError.value(
                  name="agent",
                  argument="reward_estimation[predict_horizon_values]",
                  value=self.predict_horizon_values,
                  condition='reward_estimation[reward_horizon] is "episode"',
              )
      else:
          self.reward_horizon = self.submodule(
              name="reward_horizon",
              module=reward_estimation["horizon"],
              modules=parameter_modules,
              dtype="int",
              min_value=1,
              max_value=self.max_episode_timesteps,
          )
          if self.predict_horizon_values is None:
              self.predict_horizon_values = "late"
  
      # Reward discount
      reward_discount = reward_estimation.get("discount")
      if reward_discount is None:
          reward_discount = 1.0
      self.reward_discount = self.submodule(
          name="reward_discount",
          module=reward_discount,
          modules=parameter_modules,
          dtype="float",
          min_value=0.0,
          max_value=1.0,
      )
  
      # Entropy regularization
      if entropy_regularization is None:
          entropy_regularization = 0.0
      self.entropy_regularization = self.submodule(
          name="entropy_regularization",
          module=entropy_regularization,
          modules=parameter_modules,
          is_trainable=False,
          dtype="float",
          min_value=0.0,
      )
  
      # Update mode
      if not all(
          key in ("batch_size", "frequency", "start", "unit")
          for key in update
      ):
          raise TensorforceError.value(
              name="agent",
              argument="update",
              value=list(update),
              hint="not from {batch_size,frequency,start,unit}",
          )
      # update: unit
      elif "unit" not in update:
          raise TensorforceError.required(
              name="agent", argument="update[unit]"
          )
      elif update["unit"] not in ("timesteps", "episodes"):
          raise TensorforceError.value(
              name="agent",
              argument="update[unit]",
              value=update["unit"],
              hint="not in {timesteps,episodes}",
          )
      # update: batch_size
      elif "batch_size" not in update:
          raise TensorforceError.required(
              name="agent", argument="update[batch_size]"
          )
  
      self.update_unit = update["unit"]
      self.update_batch_size = self.submodule(
          name="update_batch_size",
          module=update["batch_size"],
          modules=parameter_modules,
          is_trainable=False,
          dtype="int",
          min_value=1,
      )
      if "frequency" in update and update["frequency"] == "never":
          self.update_frequency = None
      else:
          frequency = update.get("frequency")
          if frequency is None:
              frequency = update["batch_size"]
          elif isinstance(frequency, float):
              if frequency <= 0.0 or frequency > 1.0:
                  raise TensorforceError.value(
                      name="agent",
                      argument="update[frequency]",
                      value=update["frequency"],
                      hint="not in (0.0, 1.0]",
                  )
              else:
                  frequency = max(
                      1, int(frequency * update["batch_size"])
                  )
          self.update_frequency = self.submodule(
              name="update_frequency",
              module=frequency,
              modules=parameter_modules,
              is_trainable=False,
              dtype="int",
              min_value=1,
              max_value=max(2, self.update_batch_size.max_value()),
          )
          start = update.get("start")
          if start is None:
              start = 0
          self.update_start = self.submodule(
              name="update_start",
              module=start,
              modules=parameter_modules,
              is_trainable=False,
              dtype="int",
              min_value=0,
          )
  
      # Baseline optimization overview:
      # Policy    Objective   Optimizer   Config
      #   n         n           n           default predict_horizon_values=False
      #   n         n           f           default predict_horizon=False
      #   n         n           y           default predict_horizon=False
      #   n         y           n           main policy, shared loss/kldiv, weighted 1.0
      #   n         y           f           main policy, shared loss/kldiv, weighted
      #   n         y           y           main policy, separate
      #   y         n           n           estimate_advantage=True,advantage_in_loss=True
      #   y         n           f           shared objective/loss/kldiv, weighted
      #   y         n           y           shared objective
      #   y         y           n           shared loss/kldiv, weighted 1.0, equal horizon
      #   y         y           f           shared loss/kldiv, weighted, equal horizon
      #   y         y           y           separate
  
      self.separate_baseline = baseline is not None
  
      if (
          baseline is None
          and baseline_objective is None
          and "predict_horizon_values" not in reward_estimation
      ):
          self.predict_horizon_values = False
  
      if (
          baseline is not None
          and baseline_objective is None
          and baseline_optimizer is None
      ):
          if "estimate_advantage" not in reward_estimation:
              self.estimate_advantage = "late"
          self.advantage_in_loss = True
      else:
          self.advantage_in_loss = False
  
      if baseline_optimizer is None and baseline_objective is not None:
          baseline_optimizer = 1.0
  
      if baseline_optimizer is None or isinstance(
          baseline_optimizer, float
      ):
          baseline_is_trainable = True
      else:
          baseline_is_trainable = False
  
      # Reward processing
      reward_processing = reward_estimation.get("reward_processing")
      if reward_processing is None:
          self.reward_processing = None
      else:
          self.reward_processing = self.submodule(
              name="reward_processing",
              module=Preprocessor,
              is_trainable=False,
              input_spec=self.reward_spec,
              layers=reward_processing,
          )
          if self.reward_processing.output_spec() != self.reward_spec:
              raise TensorforceError.mismatch(
                  name="reward_estimation[reward_processing]",
                  argument="output spec",
                  value1=self.reward_processing.output_spec(),
                  value2=self.reward_spec,
              )
  
      # Return processing
      return_processing = reward_estimation.get("return_processing")
      if return_processing is None:
          self.return_processing = None
      else:
          self.return_processing = self.submodule(
              name="return_processing",
              module=Preprocessor,
              is_trainable=False,
              input_spec=self.reward_spec,
              layers=return_processing,
              is_preprocessing_layer_valid=False,
          )
          if self.return_processing.output_spec() != self.reward_spec:
              raise TensorforceError.mismatch(
                  name="reward_estimation[return_processing]",
                  argument="output spec",
                  value1=self.return_processing.output_spec(),
                  value2=self.reward_spec,
              )
  
      # Advantage processing
      advantage_processing = reward_estimation.get(
          "advantage_processing"
      )
      if advantage_processing is None:
          self.advantage_processing = None
      else:
          if self.estimate_advantage is False:
              raise TensorforceError.invalid(
                  name="agent",
                  argument="reward_estimation[advantage_processing]",
                  condition="reward_estimation[estimate_advantage] is false",
              )
          self.advantage_processing = self.submodule(
              name="advantage_processing",
              module=Preprocessor,
              is_trainable=False,
              input_spec=self.reward_spec,
              layers=advantage_processing,
              is_preprocessing_layer_valid=False,
          )
          if (
              self.advantage_processing.output_spec()
              != self.reward_spec
          ):
              raise TensorforceError.mismatch(
                  name="reward_estimation[advantage_processing]",
                  argument="output spec",
                  value1=self.advantage_processing.output_spec(),
                  value2=self.reward_spec,
              )
  
      # Objectives
      self.objective = self.submodule(
          name="policy_objective",
          module=objective,
          modules=objective_modules,
          states_spec=self.processed_states_spec,
          auxiliaries_spec=self.auxiliaries_spec,
          actions_spec=self.actions_spec,
          reward_spec=self.reward_spec,
      )
      if baseline_objective is None:
          self.baseline_objective = None
      else:
          self.baseline_objective = self.submodule(
              name="baseline_objective",
              module=baseline_objective,
              modules=objective_modules,
              is_trainable=baseline_is_trainable,
              states_spec=self.processed_states_spec,
              auxiliaries_spec=self.auxiliaries_spec,
              actions_spec=self.actions_spec,
              reward_spec=self.reward_spec,
          )
          assert (
              len(self.baseline_objective.required_baseline_fns()) == 0
          )
  
      # Policy
      required_fns = {"policy"}
      required_fns.update(self.objective.required_policy_fns())
      if not self.separate_baseline:
          if (
              self.predict_horizon_values is not False
              or self.estimate_advantage is not False
          ):
              if self.predict_action_values:
                  required_fns.add("action_value")
              else:
                  required_fns.add("state_value")
          required_fns.update(self.objective.required_baseline_fns())
          if self.baseline_objective is not None:
              required_fns.update(
                  self.baseline_objective.required_policy_fns()
              )
  
      if required_fns <= {"state_value"}:
          default_module = "parametrized_state_value"
      elif required_fns <= {"action_value"} and all(
          spec.type == "float" for spec in self.actions_spec.values()
      ):
          default_module = "parametrized_action_value"
      elif required_fns <= {
          "policy",
          "action_value",
          "state_value",
      } and all(
          spec.type in ("bool", "int")
          for spec in self.actions_spec.values()
      ):
          default_module = "parametrized_value_policy"
      elif required_fns <= {"policy", "stochastic"}:
          default_module = "parametrized_distributions"
      else:
          logging.warning(
              "Policy type should be explicitly specified for non-standard agent configuration."
          )
          default_module = "parametrized_distributions"
  
      self.policy = self.submodule(
          name="policy",
          module=policy,
          modules=policy_modules,
          default_module=default_module,
          states_spec=self.processed_states_spec,
          auxiliaries_spec=self.auxiliaries_spec,
          actions_spec=self.actions_spec,
      )
      self.internals_spec["policy"] = self.policy.internals_spec
      self.initial_internals["policy"] = self.policy.internals_init()
      self.objective.internals_spec = self.policy.internals_spec
  
      if not self.entropy_regularization.is_constant(
          value=0.0
      ) and not isinstance(self.policy, StochasticPolicy):
          raise TensorforceError.invalid(
              name="agent",
              argument="entropy_regularization",
              condition="policy is not stochastic",
          )
  
      # Baseline
      if self.separate_baseline:
          if (
              self.predict_horizon_values is not False
              or self.estimate_advantage is not False
          ):
              if self.predict_action_values:
                  required_fns = {"action_value"}
              else:
                  required_fns = {"state_value"}
          required_fns.update(self.objective.required_baseline_fns())
          if self.baseline_objective is not None:
              required_fns.update(
                  self.baseline_objective.required_policy_fns()
              )
  
          if required_fns <= {"state_value"}:
              default_module = "parametrized_state_value"
          elif required_fns <= {"action_value"} and all(
              spec.type == "float"
              for spec in self.actions_spec.values()
          ):
              default_module = "parametrized_action_value"
          elif required_fns <= {
              "policy",
              "action_value",
              "state_value",
          } and all(
              spec.type in ("bool", "int")
              for spec in self.actions_spec.values()
          ):
              default_module = "parametrized_value_policy"
          elif required_fns <= {"policy", "stochastic"}:
              default_module = "parametrized_distributions"
          else:
              logging.warning(
                  "Policy type should be explicitly specified for non-standard agent "
                  "configuration."
              )
              default_module = "parametrized_distributions"
  
          self.baseline = self.submodule(
              name="baseline",
              module=baseline,
              modules=policy_modules,
              default_module=default_module,
              is_trainable=baseline_is_trainable,
              states_spec=self.processed_states_spec,
              auxiliaries_spec=self.auxiliaries_spec,
              actions_spec=self.actions_spec,
          )
          self.internals_spec["baseline"] = self.baseline.internals_spec
          self.initial_internals[
              "baseline"
          ] = self.baseline.internals_init()
  
      else:
          self.baseline = self.policy
  
      if self.baseline_objective is not None:
          self.baseline_objective.internals_spec = (
              self.baseline.internals_spec
          )
  
      # Check for name collisions
      for name in self.internals_spec:
          if name in self.value_names:
              raise TensorforceError.exists(
                  name="value name", value=name
              )
          self.value_names.add(name)
  
      # Optimizers
      if baseline_optimizer is None:
          self.baseline_loss_weight = None
          internals_spec = self.internals_spec
          self.baseline_optimizer = None
      elif isinstance(baseline_optimizer, float):
          self.baseline_loss_weight = self.submodule(
              name="baseline_loss_weight",
              module=baseline_optimizer,
              modules=parameter_modules,
              is_trainable=False,
              dtype="float",
              min_value=0.0,
          )
          internals_spec = self.internals_spec
          self.baseline_optimizer = None
      else:
          self.baseline_loss_weight = None
          internals_spec = self.internals_spec["policy"]
          if self.separate_baseline:
              baseline_internals = self.internals_spec["baseline"]
          else:
              baseline_internals = self.internals_spec["policy"]
          arguments_spec = TensorsSpec(
              states=self.processed_states_spec,
              horizons=TensorSpec(type="int", shape=(2,)),
              internals=baseline_internals,
              auxiliaries=self.auxiliaries_spec,
              actions=self.actions_spec,
              reward=self.reward_spec,
          )
          if self.baseline_objective is not None:
              arguments_spec[
                  "reference"
              ] = self.baseline_objective.reference_spec()
          self.baseline_optimizer = self.submodule(
              name="baseline_optimizer",
              module=baseline_optimizer,
              modules=optimizer_modules,
              is_trainable=False,
              arguments_spec=arguments_spec,
          )
      arguments_spec = TensorsSpec(
          states=self.processed_states_spec,
          horizons=TensorSpec(type="int", shape=(2,)),
          internals=internals_spec,
          auxiliaries=self.auxiliaries_spec,
          actions=self.actions_spec,
          reward=self.reward_spec,
      )
      if (
          self.baseline_objective is not None
          and self.baseline_loss_weight is not None
          and not self.baseline_loss_weight.is_constant(value=0.0)
      ):
          arguments_spec["reference"] = TensorsSpec(
              policy=self.objective.reference_spec(),
              baseline=self.baseline_objective.reference_spec(),
          )
      else:
          arguments_spec["reference"] = self.objective.reference_spec()
      self.optimizer = self.submodule(
          name="policy_optimizer",
          module=optimizer,
          modules=optimizer_modules,
          arguments_spec=arguments_spec,
      )
  
      # Memory
      values_spec = TensorsSpec(
          states=self.processed_states_spec,
          internals=self.internals_spec,
          auxiliaries=self.auxiliaries_spec,
          actions=self.actions_spec,
          terminal=self.terminal_spec,
          reward=self.reward_spec,
      )
      if self.update_unit == "timesteps":
          max_past_horizon = max(
              self.policy.max_past_horizon(on_policy=False),
              self.baseline.max_past_horizon(on_policy=False),
          )
          min_capacity = (
              self.update_batch_size.max_value() + 1 + max_past_horizon
          )
          if self.reward_horizon == "episode":
              min_capacity += self.max_episode_timesteps
          else:
              min_capacity += self.reward_horizon.max_value()
          if self.max_episode_timesteps is not None:
              min_capacity = max(
                  min_capacity, self.max_episode_timesteps
              )
      elif self.update_unit == "episodes":
          if self.max_episode_timesteps is None:
              min_capacity = None
          else:
              min_capacity = (
                  self.update_batch_size.max_value() + 1
              ) * self.max_episode_timesteps
      else:
          assert False
      if self.config.buffer_observe == "episode":
          if self.max_episode_timesteps is not None:
              min_capacity = max(
                  min_capacity, 2 * self.max_episode_timesteps
              )
      elif isinstance(self.config.buffer_observe, int):
          if min_capacity is None:
              min_capacity = 2 * self.config.buffer_observe
          else:
              min_capacity = max(
                  min_capacity, 2 * self.config.buffer_observe
              )
  
      self.memory = self.submodule(
          name="memory",
          module=memory,
          modules=memory_modules,
          is_trainable=False,
          values_spec=values_spec,
          min_capacity=min_capacity,
      )
  
      # Trace decay
      trace_decay = reward_estimation.get("trace_decay", 1.0)
      if trace_decay != 1.0 and self.predict_horizon_values != "early":
          raise TensorforceError.invalid(
              name="agent",
              argument="reward_estimation[trace_decay]",
              condition='reward_estimation[predict_horizon_values] != "early"',
          )
      self.trace_decay = self.submodule(
          name="trace_decay",
          module=trace_decay,
          modules=parameter_modules,
          dtype="float",
          min_value=0.0,
          max_value=1.0,
      )
  
      # GAE decay
      gae_decay = reward_estimation.get("gae_decay", 0.0)
      if gae_decay != 0.0:
          from tensorforce.core.memories import Recent
  
          if not isinstance(self.memory, Recent):
              raise TensorforceError.invalid(
                  name="agent",
                  argument="reward_estimation[gae_decay]",
                  condition="memory type is not Recent",
              )
          elif self.estimate_advantage is False:
              raise TensorforceError.invalid(
                  name="agent",
                  argument="reward_estimation[gae_decay]",
                  condition="reward_estimation[estimate_advantage] is false",
              )
          elif self.advantage_in_loss:
              raise TensorforceError.invalid(
                  name="agent",
                  argument="reward_estimation[gae_decay]",
                  condition="advantage-in-loss mode",
              )
      self.gae_decay = self.submodule(
          name="gae_decay",
          module=gae_decay,
          modules=parameter_modules,
          dtype="float",
          min_value=0.0,
          max_value=1.0,
      )
  
imports: import logging
  import tensorflow
  from tensorforce import TensorforceError
  from tensorforce.core import ModuleDict, memory_modules, optimizer_modules, parameter_modules, SignatureDict, TensorDict, TensorSpec, TensorsSpec, tf_function, tf_util, VariableDict
  from tensorforce.core.models import Model
  from tensorforce.core.networks import Preprocessor
  from tensorforce.core.objectives import objective_modules
  from tensorforce.core.policies import policy_modules, StochasticPolicy
label_desc: Use image node_id 1 to create a new TensorforceModel object from inherited base classes: Model with example: obj = TensorforceModel()