docstr: Ensemble Strategy that combines PPO, A2C and DDPG
docstr_tok: ["Ensemble","Strategy","that","combines","PPO",",","A2C","and","DDPG"]
code: def run_ensemble_strategy(
      self,
      A2C_model_kwargs,
      PPO_model_kwargs,
      DDPG_model_kwargs,
      timesteps_dict,
  ):
      """Ensemble Strategy that combines PPO, A2C and DDPG"""
      print("============Start Ensemble Strategy============")
      # for ensemble model, it's necessary to feed the last state
      # of the previous model to the current model as the initial state
      last_state_ensemble = []
  
      ppo_sharpe_list = []
      ddpg_sharpe_list = []
      a2c_sharpe_list = []
  
      model_use = []
      validation_start_date_list = []
      validation_end_date_list = []
      iteration_list = []
  
      insample_turbulence = self.df[
          (self.df.date < self.train_period[1])
          & (self.df.date >= self.train_period[0])
      ]
      insample_turbulence_threshold = np.quantile(
          insample_turbulence.turbulence.values, 0.90
      )
  
      start = time.time()
      for i in range(
          self.rebalance_window + self.validation_window,
          len(self.unique_trade_date),
          self.rebalance_window,
      ):
          validation_start_date = self.unique_trade_date[
              i - self.rebalance_window - self.validation_window
          ]
          validation_end_date = self.unique_trade_date[
              i - self.rebalance_window
          ]
  
          validation_start_date_list.append(validation_start_date)
          validation_end_date_list.append(validation_end_date)
          iteration_list.append(i)
  
          print("============================================")
          # initial state is empty
          if i - self.rebalance_window - self.validation_window == 0:
              # inital state
              initial = True
          else:
              # previous state
              initial = False
  
          # Tuning trubulence index based on historical data
          # Turbulence lookback window is one quarter (63 days)
          end_date_index = self.df.index[
              self.df["date"]
              == self.unique_trade_date[
                  i - self.rebalance_window - self.validation_window
              ]
          ].to_list()[-1]
          start_date_index = end_date_index - 63 + 1
  
          historical_turbulence = self.df.iloc[
              start_date_index : (end_date_index + 1), :
          ]
  
          historical_turbulence = historical_turbulence.drop_duplicates(
              subset=["date"]
          )
  
          historical_turbulence_mean = np.mean(
              historical_turbulence.turbulence.values
          )
  
          # print(historical_turbulence_mean)
  
          if historical_turbulence_mean > insample_turbulence_threshold:
              # if the mean of the historical data is greater than the 90% quantile of insample turbulence data
              # then we assume that the current market is volatile,
              # therefore we set the 90% quantile of insample turbulence data as the turbulence threshold
              # meaning the current turbulence can't exceed the 90% quantile of insample turbulence data
              turbulence_threshold = insample_turbulence_threshold
          else:
              # if the mean of the historical data is less than the 90% quantile of insample turbulence data
              # then we tune up the turbulence_threshold, meaning we lower the risk
              turbulence_threshold = np.quantile(
                  insample_turbulence.turbulence.values, 1
              )
  
          turbulence_threshold = np.quantile(
              insample_turbulence.turbulence.values, 0.99
          )
          print("turbulence_threshold: ", turbulence_threshold)
  
          # Environment Setup starts
          # training env
          train = data_split(
              self.df,
              start=self.train_period[0],
              end=self.unique_trade_date[
                  i - self.rebalance_window - self.validation_window
              ],
          )
          self.train_env = DummyVecEnv(
              [
                  lambda: StockTradingEnv(
                      df=train,
                      stock_dim=self.stock_dim,
                      hmax=self.hmax,
                      initial_amount=self.initial_amount,
                      num_stock_shares=[0] * self.stock_dim,
                      buy_cost_pct=[self.buy_cost_pct] * self.stock_dim,
                      sell_cost_pct=[self.sell_cost_pct]
                      * self.stock_dim,
                      reward_scaling=self.reward_scaling,
                      state_space=self.state_space,
                      action_space=self.action_space,
                      tech_indicator_list=self.tech_indicator_list,
                      print_verbosity=self.print_verbosity,
                  )
              ]
          )
  
          validation = data_split(
              self.df,
              start=self.unique_trade_date[
                  i - self.rebalance_window - self.validation_window
              ],
              end=self.unique_trade_date[i - self.rebalance_window],
          )
          # Environment Setup ends
  
          # Training and Validation starts
          print(
              "======Model training from: ",
              self.train_period[0],
              "to ",
              self.unique_trade_date[
                  i - self.rebalance_window - self.validation_window
              ],
          )
          # print("training: ",len(data_split(df, start=20090000, end=test.datadate.unique()[i-rebalance_window]) ))
          # print("==============Model Training===========")
          print("======A2C Training========")
          model_a2c = self.get_model(
              "a2c",
              self.train_env,
              policy="MlpPolicy",
              model_kwargs=A2C_model_kwargs,
          )
          model_a2c = self.train_model(
              model_a2c,
              "a2c",
              tb_log_name=f"a2c_{i}",
              iter_num=i,
              total_timesteps=timesteps_dict["a2c"],
          )  # 100_000
  
          print(
              "======A2C Validation from: ",
              validation_start_date,
              "to ",
              validation_end_date,
          )
          val_env_a2c = DummyVecEnv(
              [
                  lambda: StockTradingEnv(
                      df=validation,
                      stock_dim=self.stock_dim,
                      hmax=self.hmax,
                      initial_amount=self.initial_amount,
                      num_stock_shares=[0] * self.stock_dim,
                      buy_cost_pct=[self.buy_cost_pct] * self.stock_dim,
                      sell_cost_pct=[self.sell_cost_pct]
                      * self.stock_dim,
                      reward_scaling=self.reward_scaling,
                      state_space=self.state_space,
                      action_space=self.action_space,
                      tech_indicator_list=self.tech_indicator_list,
                      turbulence_threshold=turbulence_threshold,
                      iteration=i,
                      model_name="A2C",
                      mode="validation",
                      print_verbosity=self.print_verbosity,
                  )
              ]
          )
          val_obs_a2c = val_env_a2c.reset()
          self.DRL_validation(
              model=model_a2c,
              test_data=validation,
              test_env=val_env_a2c,
              test_obs=val_obs_a2c,
          )
          sharpe_a2c = self.get_validation_sharpe(i, model_name="A2C")
          print("A2C Sharpe Ratio: ", sharpe_a2c)
  
          print("======PPO Training========")
          model_ppo = self.get_model(
              "ppo",
              self.train_env,
              policy="MlpPolicy",
              model_kwargs=PPO_model_kwargs,
          )
          model_ppo = self.train_model(
              model_ppo,
              "ppo",
              tb_log_name=f"ppo_{i}",
              iter_num=i,
              total_timesteps=timesteps_dict["ppo"],
          )  # 100_000
          print(
              "======PPO Validation from: ",
              validation_start_date,
              "to ",
              validation_end_date,
          )
          val_env_ppo = DummyVecEnv(
              [
                  lambda: StockTradingEnv(
                      df=validation,
                      stock_dim=self.stock_dim,
                      hmax=self.hmax,
                      initial_amount=self.initial_amount,
                      num_stock_shares=[0] * self.stock_dim,
                      buy_cost_pct=[self.buy_cost_pct] * self.stock_dim,
                      sell_cost_pct=[self.sell_cost_pct]
                      * self.stock_dim,
                      reward_scaling=self.reward_scaling,
                      state_space=self.state_space,
                      action_space=self.action_space,
                      tech_indicator_list=self.tech_indicator_list,
                      turbulence_threshold=turbulence_threshold,
                      iteration=i,
                      model_name="PPO",
                      mode="validation",
                      print_verbosity=self.print_verbosity,
                  )
              ]
          )
          val_obs_ppo = val_env_ppo.reset()
          self.DRL_validation(
              model=model_ppo,
              test_data=validation,
              test_env=val_env_ppo,
              test_obs=val_obs_ppo,
          )
          sharpe_ppo = self.get_validation_sharpe(i, model_name="PPO")
          print("PPO Sharpe Ratio: ", sharpe_ppo)
  
          print("======DDPG Training========")
          model_ddpg = self.get_model(
              "ddpg",
              self.train_env,
              policy="MlpPolicy",
              model_kwargs=DDPG_model_kwargs,
          )
          model_ddpg = self.train_model(
              model_ddpg,
              "ddpg",
              tb_log_name=f"ddpg_{i}",
              iter_num=i,
              total_timesteps=timesteps_dict["ddpg"],
          )  # 50_000
          print(
              "======DDPG Validation from: ",
              validation_start_date,
              "to ",
              validation_end_date,
          )
          val_env_ddpg = DummyVecEnv(
              [
                  lambda: StockTradingEnv(
                      df=validation,
                      stock_dim=self.stock_dim,
                      hmax=self.hmax,
                      initial_amount=self.initial_amount,
                      num_stock_shares=[0] * self.stock_dim,
                      buy_cost_pct=[self.buy_cost_pct] * self.stock_dim,
                      sell_cost_pct=[self.sell_cost_pct]
                      * self.stock_dim,
                      reward_scaling=self.reward_scaling,
                      state_space=self.state_space,
                      action_space=self.action_space,
                      tech_indicator_list=self.tech_indicator_list,
                      turbulence_threshold=turbulence_threshold,
                      iteration=i,
                      model_name="DDPG",
                      mode="validation",
                      print_verbosity=self.print_verbosity,
                  )
              ]
          )
          val_obs_ddpg = val_env_ddpg.reset()
          self.DRL_validation(
              model=model_ddpg,
              test_data=validation,
              test_env=val_env_ddpg,
              test_obs=val_obs_ddpg,
          )
          sharpe_ddpg = self.get_validation_sharpe(i, model_name="DDPG")
  
          ppo_sharpe_list.append(sharpe_ppo)
          a2c_sharpe_list.append(sharpe_a2c)
          ddpg_sharpe_list.append(sharpe_ddpg)
  
          print(
              "======Best Model Retraining from: ",
              self.train_period[0],
              "to ",
              self.unique_trade_date[i - self.rebalance_window],
          )
          # Environment setup for model retraining up to first trade date
          # train_full = data_split(self.df, start=self.train_period[0],
          # end=self.unique_trade_date[i - self.rebalance_window])
          # self.train_full_env = DummyVecEnv([lambda: StockTradingEnv(train_full,
          #                                               self.stock_dim,
          #                                               self.hmax,
          #                                               self.initial_amount,
          #                                               self.buy_cost_pct,
          #                                               self.sell_cost_pct,
          #                                               self.reward_scaling,
          #                                               self.state_space,
          #                                               self.action_space,
          #                                               self.tech_indicator_list,
          #                                              print_verbosity=self.print_verbosity
          # )])
          # Model Selection based on sharpe ratio
          if (sharpe_ppo >= sharpe_a2c) & (sharpe_ppo >= sharpe_ddpg):
              model_use.append("PPO")
              model_ensemble = model_ppo
  
              # model_ensemble = self.get_model("ppo",
              # self.train_full_env,
              # policy="MlpPolicy",
              # model_kwargs=PPO_model_kwargs)
              # model_ensemble = self.train_model(model_ensemble,
              # "ensemble",
              # tb_log_name="ensemble_{}".format(i),
              # iter_num = i,
              # total_timesteps=timesteps_dict['ppo']) #100_000
          elif (sharpe_a2c > sharpe_ppo) & (sharpe_a2c > sharpe_ddpg):
              model_use.append("A2C")
              model_ensemble = model_a2c
  
              # model_ensemble = self.get_model("a2c",
              # self.train_full_env,
              # policy="MlpPolicy",
              # model_kwargs=A2C_model_kwargs)
              # model_ensemble = self.train_model(model_ensemble,
              # "ensemble",
              # tb_log_name="ensemble_{}".format(i),
              # iter_num = i,
              # total_timesteps=timesteps_dict['a2c']) #100_000
          else:
              model_use.append("DDPG")
              model_ensemble = model_ddpg
  
              # model_ensemble = self.get_model("ddpg",
              # self.train_full_env,
              # policy="MlpPolicy",
              # model_kwargs=DDPG_model_kwargs)
              # model_ensemble = self.train_model(model_ensemble,
              # "ensemble",
              # tb_log_name="ensemble_{}".format(i),
              #  iter_num = i,
              # total_timesteps=timesteps_dict['ddpg']) #50_000
  
          # Training and Validation ends
  
          # Trading starts
          print(
              "======Trading from: ",
              self.unique_trade_date[i - self.rebalance_window],
              "to ",
              self.unique_trade_date[i],
          )
          # print("Used Model: ", model_ensemble)
          last_state_ensemble = self.DRL_prediction(
              model=model_ensemble,
              name="ensemble",
              last_state=last_state_ensemble,
              iter_num=i,
              turbulence_threshold=turbulence_threshold,
              initial=initial,
          )
          # Trading ends
  
      end = time.time()
      print("Ensemble Strategy took: ", (end - start) / 60, " minutes")
  
      df_summary = pd.DataFrame(
          [
              iteration_list,
              validation_start_date_list,
              validation_end_date_list,
              model_use,
              a2c_sharpe_list,
              ppo_sharpe_list,
              ddpg_sharpe_list,
          ]
      ).T
      df_summary.columns = [
          "Iter",
          "Val Start",
          "Val End",
          "Model Used",
          "A2C Sharpe",
          "PPO Sharpe",
          "DDPG Sharpe",
      ]
  
      return df_summary
  
imports: from __future__ import annotations
  import time
  import numpy
  import pandas
  from stable_baselines3 import A2C
  from stable_baselines3 import DDPG
  from stable_baselines3 import PPO
  from stable_baselines3 import SAC
  from stable_baselines3 import TD3
  from stable_baselines3.common.callbacks import BaseCallback
  from stable_baselines3.common.noise import NormalActionNoise
  from stable_baselines3.common.noise import OrnsteinUhlenbeckActionNoise
  from stable_baselines3.common.vec_env import DummyVecEnv
  from finrl import config
  from finrl.meta.env_stock_trading.env_stocktrading import StockTradingEnv
  from finrl.meta.preprocessor.preprocessors import data_split
label_desc: Use image node_id 7 for calling the DRLEnsembleAgent obj's underlying member method code with example usage: obj.run_ensemble_strategy(A2C_model_kwargs, PPO_model_kwargs, DDPG_model_kwargs, timesteps_dict) and returns: df_summary