import tensorflow as tf

                    callback.on_rollout_start()
                if step % self.update_buffer_interval ==0 and step>self.learning_starts:
                    mean_agent = sum(all_r)/sum(all_r_step) 
                    mean_exp = sum(all_exp_r)/sum(all_exp_r_step) 
                    add_r = mean_agent>mean_exp-0.5
                    all_r = []
                    all_exp_r = []
                    all_r_step = []
                    all_exp_r_step = []
                    
                    if add_r:
                        self.ratio = min(self.ratio+2/self.batch_size,self.max_ratio)
                    else:
                        self.ratio = max(self.ratio-1/self.batch_size,self.init_ratio)
                    print('|new-ratio:',self.ratio,'|mean-agent:',mean_agent,'|mean-exp:',mean_exp-0.5,'|')
                    smry = tf.Summary(value=[tf.Summary.Value(tag="ratio", simple_value=self.ratio)])
                    writer.add_summary(smry,step)
                episode_rewards[-1] += reward_
                if done:
                    if self.action_noise is not None:
                        self.action_noise.reset()
                    if not isinstance(self.env, VecEnv):
                        obs = self.env.reset()
                    #if episode_rewards[-1] >= mean_expert_reward:
                    #    self.ratio  = np.clip((self.ratio+1/self.batch_size),0,60/self.batch_s
                    

                    episode_rewards.append(0.0)

                    maybe_is_success = info.get('is_success')
