text: #import wandb
  import tensorflow as tf
  from tensorflow.keras.layers import Input, Dense, Lambda
  import gym
  import argparse
  import numpy as np
  
  tf.keras.backend.set_floatx('float32')
  
  #wandb.init(name='PPO', project="deep-rl-tf2")
  
  parser = argparse.ArgumentParser()
  parser.add_argument('--gamma', type=float, default=0.99)
  parser.add_argument('--update_interval', type=int, default=5)
  parser.add_argument('--actor_lr', type=float, default=0.0005)
  parser.add_argument('--critic_lr', type=float, default=0.001)
  parser.add_argument('--clip_ratio', type=float, default=0.1)
  parser.add_argument('--lmbda', type=float, default=0.95)
  parser.add_argument('--epochs', type=int, default=3)
  
  args = parser.parse_args()
  
  
  class Actor:
      def __init__(self, state_dim, action_dim, action_bound, std_bound):
          self.state_dim = state_dim
          self.action_dim = action_dim
          self.action_bound = action_bound
          self.std_bound = std_bound
          self.model = self.create_model()
          self.opt = tf.keras.optimizers.Adam(args.actor_lr)
  
      def get_action(self, state):
          state = np.reshape(state, [1, self.state_dim])
          mu, std = self.model.predict(state)
          action = np.random.normal(mu[0], std[0], size=self.action_dim)
          action = np.clip(action, -self.action_bound, self.action_bound)
          log_policy = self.log_pdf(mu, std, action)
  
          return log_policy, action
  
      def log_pdf(self, mu, std, action):
          std = tf.clip_by_value(std, self.std_bound[0], self.std_bound[1])
          var = std ** 2
          log_policy_pdf = -0.5 * (action - mu) ** 2 / \
              var - 0.5 * tf.math.log(var * 2 * np.pi)
          return tf.reduce_sum(log_policy_pdf, 1, keepdims=True)
  
      def create_model(self):
          state_input = Input((self.state_dim,))
          dense_1 = Dense(32, activation='relu')(state_input)
          dense_2 = Dense(32, activation='relu')(dense_1)
          out_mu = Dense(self.action_dim, activation='tanh')(dense_2)
          mu_output = Lambda(lambda x: x * self.action_bound)(out_mu)
          std_output = Dense(self.action_dim, activation='softplus')(dense_2)
          return tf.keras.models.Model(state_input, [mu_output, std_output])
  
      def compute_loss(self, log_old_policy, log_new_policy, actions, gaes):
          ratio = tf.exp(log_new_policy - tf.stop_gradient(log_old_policy))
          gaes = tf.stop_gradient(gaes)
          clipped_ratio = tf.clip_by_value(ratio, 1.0-args.clip_ratio, 1.0+args.clip_ratio)
          
          ratio = tf.dtypes.cast(ratio, tf.float64)
          gaes = tf.dtypes.cast(gaes, tf.float64)
          clipped_ratio = tf.dtypes.cast(clipped_ratio, tf.float64)
          #print(gaes)
          #print(ratio * gaes)
          #print(clipped_ratio * gaes)
          surrogate = -tf.minimum(ratio * gaes, clipped_ratio * gaes)
          return tf.reduce_mean(surrogate)
  
      def train(self, log_old_policy, states, actions, gaes):
          with tf.GradientTape() as tape:
              mu, std = self.model(states, training=True)
              log_new_policy = self.log_pdf(mu, std, actions)
              loss = self.compute_loss(log_old_policy, log_new_policy, actions, gaes)
          grads = tape.gradient(loss, self.model.trainable_variables)
          self.opt.apply_gradients(zip(grads, self.model.trainable_variables))
          return loss
  
  
  class Critic:
      def __init__(self, state_dim):
          self.state_dim = state_dim
          self.model = self.create_model()
          self.opt = tf.keras.optimizers.Adam(args.critic_lr)
  
      def create_model(self):
          return tf.keras.Sequential([
              Input((self.state_dim,)),
              Dense(32, activation='relu'),
              Dense(32, activation='relu'),
              Dense(16, activation='relu'),
              Dense(1, activation='linear')
          ])
  
      def compute_loss(self, v_pred, td_targets):
          mse = tf.keras.losses.MeanSquaredError()
          return mse(td_targets, v_pred)
  
      def train(self, states, td_targets):
          with tf.GradientTape() as tape:
              v_pred = self.model(states, training=True)
              assert v_pred.shape == td_targets.shape
              loss = self.compute_loss(v_pred, tf.stop_gradient(td_targets))
          grads = tape.gradient(loss, self.model.trainable_variables)
          self.opt.apply_gradients(zip(grads, self.model.trainable_variables))
          return loss
  
  
  class Agent:
      def __init__(self, env, building_info, observation_spaces = None, action_spaces = None):
          #Hyper-parameters
          self.discount = 0.992 #Discount factor
          self.batch_size = 100 #Size of each MINI-BATCH
          self.iterations = 1 # Number of updates of the actor-critic networks every time-step
          self.policy_freq = 2 # Number of iterations after which the actor and target networks are updated
          self.tau = 5e-3 #Rate at which the target networks are updated
          self.lr_init = 1e-3 #5e-2
          self.lr_final = 1e-3 #3e-3
          self.lr_decay_rate = 1/(78*8760)
          self.expl_noise_init = 0.75 # Exploration noise at time-step 0
          self.expl_noise_final = 0.01 # Magnitude of the minimum exploration noise
          self.expl_noise_decay_rate = 1/(290*8760)  # Decay rate of the exploration noise in 1/h
          self.policy_noise = 0.025*0
          self.noise_clip = 0.04*0
          self.max_action = 0.25
          self.min_samples_training = 400 #Min number of tuples that are stored in the batch before the training process begins
                  
          self.env = env
          self.state_dim = []
          self.action_dim = []
          self.action_bound = []
          self.n_buildings = len(observation_spaces)
          self.std_bound = [1e-2, 1.0]
  
          self.actor, self.critic, self.target_actor, self.target_critic=  {}, {}, {}, {}
  
          self.actor_opt = tf.keras.optimizers.Adam(args.actor_lr)
          self.critic_opt = tf.keras.optimizers.Adam(args.critic_lr)
  
          for i, (o, a) in enumerate(zip(observation_spaces, action_spaces)):
              self.state_dim.append(o.shape[0])
              self.action_dim.append(a.shape[0])
              self.action_bound.append(a.high[0])
              
              self.actor[i] = Actor(self.state_dim[i], self.action_dim[i], self.action_bound[i], self.std_bound)
              self.critic[i] = Critic(self.state_dim[i])
              
  
      def gae_target(self, rewards, v_values, next_v_value, done):
          n_step_targets = np.zeros_like(rewards)
          gae = np.zeros_like(rewards)
          gae_cumulative = 0
          forward_val = 0
  
          if not done:
              forward_val = next_v_value
  
          for k in reversed(range(0, len(rewards))):
              delta = rewards[k] + args.gamma * forward_val - v_values[k]
              gae_cumulative = args.gamma * args.lmbda * gae_cumulative + delta
              gae[k] = gae_cumulative
              forward_val = v_values[k]
              n_step_targets[k] = gae[k] + v_values[k]
          return gae, n_step_targets
  
      def list_to_batch(self, list):
          batch = list[0]
          for elem in list[1:]:
              batch = np.append(batch, elem, axis=0)
          return batch
  
      def train(self, max_episodes=1000):
          for ep in range(max_episodes):
              state_batch = []
              action_batch = []
              reward_batch = []
              old_policy_batch = []
  
              episode_reward, done = 0, False
  
              state = self.env.reset()
  
              while not done:
                  # self.env.render()
                  log_old_policy, action = self.actor.get_action(state)
  
                  next_state, reward, done, _ = self.env.step(action)
  
                  state = np.reshape(state, [1, self.state_dim])
                  action = np.reshape(action, [1, 1])
                  next_state = np.reshape(next_state, [1, self.state_dim])
                  reward = np.reshape(reward, [1, 1])
                  log_old_policy = np.reshape(log_old_policy, [1, 1])
  
                  state_batch.append(state)
                  action_batch.append(action)
                  reward_batch.append((reward+8)/8)
                  old_policy_batch.append(log_old_policy)
  
                  if len(state_batch) >= args.update_interval or done:
                      states = self.list_to_batch(state_batch)
                      actions = self.list_to_batch(action_batch)
                      rewards = self.list_to_batch(reward_batch)
                      old_policys = self.list_to_batch(old_policy_batch)
  
                      v_values = self.critic.model.predict(states)
                      next_v_value = self.critic.model.predict(next_state)
  
                      gaes, td_targets = self.gae_target(
                          rewards, v_values, next_v_value, done)
  
                      for epoch in range(args.epochs):
                          actor_loss = self.actor.train(
                              old_policys, states, actions, gaes)
                          critic_loss = self.critic.train(states, td_targets)
  
                      state_batch = []
                      action_batch = []
                      reward_batch = []
                      old_policy_batch = []
  
                  episode_reward += reward[0][0]
                  state = next_state[0]
  
              print('EP{} EpisodeReward={}'.format(ep, episode_reward))
              #wandb.log({'Reward': episode_reward})
  
  
  def main():
      env_name = 'Pendulum-v0'
      env = gym.make(env_name)
      agent = Agent(env)
      agent.train()
  
  
  if __name__ == "__main__":
      main()
  