spinup/algos/pytorch/ppo_lstm/core_lstm.py
def pad_and_compute_returns(trajectory_episodes, len_episodes, input_data):

    """
    Pad the trajectories up to hp.rollout_steps so they can be combined in a
    single tensor.
    Add advantages and discounted_returns to trajectories.
    trajector_episodes: dict{key: list[2d/1dtensor]}
    """
    parallel_rollouts = input_data["parallel_rollouts"]
    gamma = input_data["discount"]
    lam = input_data["gae_lambda"]
    epochs = input_data["epochs"]
    episode_count = len(len_episodes)
    advantages_episodes, discounted_returns_episodes = [], []
    padded_trajectories = {key: [] for key in trajectory_episodes.keys()}
    padded_trajectories["advantages"] = []
    padded_trajectories["discounted_returns"] = []

    for i in range(episode_count):
        single_padding = torch.zeros(epochs - len_episodes[i])
        for key, value in trajectory_episodes.items():
            if value[i].ndim > 1:
                padding = torch.zeros(epochs - len_episodes[i], value[0].shape[1], dtype=value[i].dtype)
            else:
                padding = torch.zeros(epochs - len_episodes[i], dtype=value[i].dtype)
            padded_trajectories[key].append(torch.cat((value[i], padding)))
        padded_trajectories["advantages"].append(torch.cat((compute_advantages(rewards=trajectory_episodes["rewards"][i],
                                                                               values=trajectory_episodes["values"][i],
                                                                               discount=gamma,
                                                                               gae_lambda=lam), single_padding)))
        padded_trajectories["discounted_returns"].append(torch.cat((calc_discounted_return(rewards=trajectory_episodes["rewards"][i],
                                                                                           discount=gamma,
                                                                                           final_value=trajectory_episodes["values"][i][-1]), single_padding)))
    return_val = {k: torch.stack(v) for k, v in padded_trajectories.items()}
    return_val["seq_len"] = torch.tensor(len_episodes)

    return return_val

    Pad the trajectories up to hp.rollout_steps so they can be combined in a
    single tensor.
    Add advantages and discounted_returns to trajectories.
    trajector_episodes: dict{key: list[2d/1dtensor]}
    
Pad the trajectories up to hp.rollout_steps so they can be combined in a
single tensor.
Add advantages and discounted_returns to trajectories.
Pad the trajectories up to hp . rollout_steps so they can be combined in a single tensor . Add advantages and discounted_returns to trajectories .
def pad_and_compute_returns(trajectory_episodes, len_episodes, input_data):
    parallel_rollouts = input_data["parallel_rollouts"]
    gamma = input_data["discount"]
    lam = input_data["gae_lambda"]
    epochs = input_data["epochs"]
    episode_count = len(len_episodes)
    advantages_episodes, discounted_returns_episodes = [], []
    padded_trajectories = {key: [] for key in trajectory_episodes.keys()}
    padded_trajectories["advantages"] = []
    padded_trajectories["discounted_returns"] = []
    for i in range(episode_count):
        single_padding = torch.zeros(epochs - len_episodes[i])
        for key, value in trajectory_episodes.items():
            if value[i].ndim > 1:
                padding = torch.zeros(epochs - len_episodes[i], value[0].shape[1], dtype=value[i].dtype)
            else:
                padding = torch.zeros(epochs - len_episodes[i], dtype=value[i].dtype)
            padded_trajectories[key].append(torch.cat((value[i], padding)))
        padded_trajectories["advantages"].append(torch.cat((compute_advantages(rewards=trajectory_episodes["rewards"][i],
                                                                               values=trajectory_episodes["values"][i],
                                                                               discount=gamma,
                                                                               gae_lambda=lam), single_padding)))
        padded_trajectories["discounted_returns"].append(torch.cat((calc_discounted_return(rewards=trajectory_episodes["rewards"][i],
                                                                                           discount=gamma,
                                                                                           final_value=trajectory_episodes["values"][i][-1]), single_padding)))
    return_val = {k: torch.stack(v) for k, v in padded_trajectories.items()}
    return_val["seq_len"] = torch.tensor(len_episodes)
    return return_val
def pad_and_compute_returns ( trajectory_episodes , len_episodes , input_data ) : parallel_rollouts = input_data [ "parallel_rollouts" ] gamma = input_data [ "discount" ] lam = input_data [ "gae_lambda" ] epochs = input_data [ "epochs" ] episode_count = len ( len_episodes ) advantages_episodes , discounted_returns_episodes = [ ] , [ ] padded_trajectories = { key : [ ] for key in trajectory_episodes . keys ( ) } padded_trajectories [ "advantages" ] = [ ] padded_trajectories [ "discounted_returns" ] = [ ] for i in range ( episode_count ) : single_padding = torch . zeros ( epochs - len_episodes [ i ] ) for key , value in trajectory_episodes . items ( ) : if value [ i ] . ndim > 1 : padding = torch . zeros ( epochs - len_episodes [ i ] , value [ 0 ] . shape [ 1 ] , dtype = value [ i ] . dtype ) else : padding = torch . zeros ( epochs - len_episodes [ i ] , dtype = value [ i ] . dtype ) padded_trajectories [ key ] . append ( torch . cat ( ( value [ i ] , padding ) ) ) padded_trajectories [ "advantages" ] . append ( torch . cat ( ( compute_advantages ( rewards = trajectory_episodes [ "rewards" ] [ i ] , values = trajectory_episodes [ "values" ] [ i ] , discount = gamma , gae_lambda = lam ) , single_padding ) ) ) padded_trajectories [ "discounted_returns" ] . append ( torch . cat ( ( calc_discounted_return ( rewards = trajectory_episodes [ "rewards" ] [ i ] , discount = gamma , final_value = trajectory_episodes [ "values" ] [ i ] [ - 1 ] ) , single_padding ) ) ) return_val = { k : torch . stack ( v ) for k , v in padded_trajectories . items ( ) } return_val [ "seq_len" ] = torch . tensor ( len_episodes ) return return_val
Pad the trajectories up to hp.rollout_steps so they can be combined in a
single tensor.
Pad the trajectories up to hp . rollout_steps so they can be combined in a single tensor .
"""
    Pad the trajectories up to hp.rollout_steps so they can be combined in a
    single tensor.
    Add advantages and discounted_returns to trajectories.
    trajector_episodes: dict{key: list[2d/1dtensor]}
    """
trajectory_episodes
len_episodes
input_data