spinup/algos/pytorch/ppo_lstm/core_lstm.py
def gather_trajectories(input_data):
    """
    Gather policy trajectories from gym environment.
    """

    # Unpack inputs.
    env = input_data["env"]
    ac = input_data["actor_critic"]
    parallel_rollouts = input_data["parallel_rollouts"]
    steps = input_data["steps"]

    # Initialise variables.
    obsv = env.reset()
    trajectory_data = {"states": [],
                       "actions": [],
                       "action_probabilities": [],
                       "rewards": [],
                       # "true_rewards": [],
                       "values": [],
                       "terminals": [],
                       "actor_hidden_states": [],
                       "actor_cell_states": [],
                       "critic_hidden_states": [],
                       "critic_cell_states": []}
    terminal = torch.ones(parallel_rollouts)

    with torch.no_grad():
        # Reset actor and critic state.
        ac.pi.get_init_state(parallel_rollouts, GATHER_DEVICE)
        ac.v.get_init_state(parallel_rollouts, GATHER_DEVICE)
        # Take 1 additional step in order to collect the state and value for the final state.
        for i in range(steps):

            trajectory_data["actor_hidden_states"].append(ac.pi.hidden_cell[0].squeeze(0).cpu())
            trajectory_data["actor_cell_states"].append(ac.pi.hidden_cell[1].squeeze(0).cpu())
            trajectory_data["critic_hidden_states"].append(ac.v.hidden_cell[0].squeeze(0).cpu())
            trajectory_data["critic_cell_states"].append(ac.v.hidden_cell[1].squeeze(0).cpu())

            # Choose next action
            state = torch.tensor(obsv, dtype=torch.float32)
            trajectory_data["states"].append(state)
            action, value, logp_a = ac.step(obs=state.unsqueeze(0).to(GATHER_DEVICE), terminal=terminal.to(GATHER_DEVICE))

            trajectory_data["values"].append(value.squeeze(1).cpu())
            trajectory_data["actions"].append(action.cpu())
            trajectory_data["action_probabilities"].append(logp_a)

            # Step environment
            action_np = action.cpu().numpy()
            obsv, reward, done, _ = env.step(action_np)
            terminal = torch.tensor(done).float()
            # transformed_reward = hp.scale_reward * torch.max(_MIN_REWARD_VALUES, torch.tensor(reward).float())
            trajectory_data["rewards"].append(torch.tensor(reward))
            # trajectory_data["rewards"].append(transformed_reward)
            # trajectory_data["true_rewards"].append(torch.tensor(reward).float())
            trajectory_data["terminals"].append(terminal)

        # Compute final value to allow for incomplete episodes.
        state = torch.tensor(obsv, dtype=torch.float32)
        _, value, _ = ac.step(obs=state.unsqueeze(0).to(GATHER_DEVICE), terminal=terminal.to(GATHER_DEVICE))
        # Future value for terminal episodes is 0.
        trajectory_data["values"].append(value.squeeze(1).cpu() * (1 - terminal))

    # Combine step lists into tensors.
    trajectory_tensors = {key: torch.stack(value) for key, value in trajectory_data.items()}
    return trajectory_tensors

    Gather policy trajectories from gym environment.
    
Gather policy trajectories from gym environment.
Gather policy trajectories from gym environment .
def gather_trajectories(input_data):
    env = input_data["env"]
    ac = input_data["actor_critic"]
    parallel_rollouts = input_data["parallel_rollouts"]
    steps = input_data["steps"]
    obsv = env.reset()
    trajectory_data = {"states": [],
                       "actions": [],
                       "action_probabilities": [],
                       "rewards": [],
                       "values": [],
                       "terminals": [],
                       "actor_hidden_states": [],
                       "actor_cell_states": [],
                       "critic_hidden_states": [],
                       "critic_cell_states": []}
    terminal = torch.ones(parallel_rollouts)
    with torch.no_grad():
        ac.pi.get_init_state(parallel_rollouts, GATHER_DEVICE)
        ac.v.get_init_state(parallel_rollouts, GATHER_DEVICE)
        for i in range(steps):
            trajectory_data["actor_hidden_states"].append(ac.pi.hidden_cell[0].squeeze(0).cpu())
            trajectory_data["actor_cell_states"].append(ac.pi.hidden_cell[1].squeeze(0).cpu())
            trajectory_data["critic_hidden_states"].append(ac.v.hidden_cell[0].squeeze(0).cpu())
            trajectory_data["critic_cell_states"].append(ac.v.hidden_cell[1].squeeze(0).cpu())
            state = torch.tensor(obsv, dtype=torch.float32)
            trajectory_data["states"].append(state)
            action, value, logp_a = ac.step(obs=state.unsqueeze(0).to(GATHER_DEVICE), terminal=terminal.to(GATHER_DEVICE))
            trajectory_data["values"].append(value.squeeze(1).cpu())
            trajectory_data["actions"].append(action.cpu())
            trajectory_data["action_probabilities"].append(logp_a)
            action_np = action.cpu().numpy()
            obsv, reward, done, _ = env.step(action_np)
            terminal = torch.tensor(done).float()
            trajectory_data["rewards"].append(torch.tensor(reward))
            trajectory_data["terminals"].append(terminal)
        state = torch.tensor(obsv, dtype=torch.float32)
        _, value, _ = ac.step(obs=state.unsqueeze(0).to(GATHER_DEVICE), terminal=terminal.to(GATHER_DEVICE))
        trajectory_data["values"].append(value.squeeze(1).cpu() * (1 - terminal))
    trajectory_tensors = {key: torch.stack(value) for key, value in trajectory_data.items()}
    return trajectory_tensors
def gather_trajectories ( input_data ) : env = input_data [ "env" ] ac = input_data [ "actor_critic" ] parallel_rollouts = input_data [ "parallel_rollouts" ] steps = input_data [ "steps" ] obsv = env . reset ( ) trajectory_data = { "states" : [ ] , "actions" : [ ] , "action_probabilities" : [ ] , "rewards" : [ ] , "values" : [ ] , "terminals" : [ ] , "actor_hidden_states" : [ ] , "actor_cell_states" : [ ] , "critic_hidden_states" : [ ] , "critic_cell_states" : [ ] } terminal = torch . ones ( parallel_rollouts ) with torch . no_grad ( ) : ac . pi . get_init_state ( parallel_rollouts , GATHER_DEVICE ) ac . v . get_init_state ( parallel_rollouts , GATHER_DEVICE ) for i in range ( steps ) : trajectory_data [ "actor_hidden_states" ] . append ( ac . pi . hidden_cell [ 0 ] . squeeze ( 0 ) . cpu ( ) ) trajectory_data [ "actor_cell_states" ] . append ( ac . pi . hidden_cell [ 1 ] . squeeze ( 0 ) . cpu ( ) ) trajectory_data [ "critic_hidden_states" ] . append ( ac . v . hidden_cell [ 0 ] . squeeze ( 0 ) . cpu ( ) ) trajectory_data [ "critic_cell_states" ] . append ( ac . v . hidden_cell [ 1 ] . squeeze ( 0 ) . cpu ( ) ) state = torch . tensor ( obsv , dtype = torch . float32 ) trajectory_data [ "states" ] . append ( state ) action , value , logp_a = ac . step ( obs = state . unsqueeze ( 0 ) . to ( GATHER_DEVICE ) , terminal = terminal . to ( GATHER_DEVICE ) ) trajectory_data [ "values" ] . append ( value . squeeze ( 1 ) . cpu ( ) ) trajectory_data [ "actions" ] . append ( action . cpu ( ) ) trajectory_data [ "action_probabilities" ] . append ( logp_a ) action_np = action . cpu ( ) . numpy ( ) obsv , reward , done , _ = env . step ( action_np ) terminal = torch . tensor ( done ) . float ( ) trajectory_data [ "rewards" ] . append ( torch . tensor ( reward ) ) trajectory_data [ "terminals" ] . append ( terminal ) state = torch . tensor ( obsv , dtype = torch . float32 ) _ , value , _ = ac . step ( obs = state . unsqueeze ( 0 ) . to ( GATHER_DEVICE ) , terminal = terminal . to ( GATHER_DEVICE ) ) trajectory_data [ "values" ] . append ( value . squeeze ( 1 ) . cpu ( ) * ( 1 - terminal ) ) trajectory_tensors = { key : torch . stack ( value ) for key , value in trajectory_data . items ( ) } return trajectory_tensors
Gather policy trajectories from gym environment.
Gather policy trajectories from gym environment .
"""
    Gather policy trajectories from gym environment.
    """ # Unpack inputs. # Initialise variables. # "true_rewards": [], # Reset actor and critic state. # Take 1 additional step in order to collect the state and value for the final state. # Choose next action # Step environment # transformed_reward = hp.scale_reward * torch.max(_MIN_REWARD_VALUES, torch.tensor(reward).float()) # trajectory_data["rewards"].append(transformed_reward) # trajectory_data["true_rewards"].append(torch.tensor(reward).float()) # Compute final value to allow for incomplete episodes. # Future value for terminal episodes is 0. # Combine step lists into tensors.
input_data