\begin{algorithmic}[1]
\Require trajectory $\tau$ 
    \For{layer $i$ in HKSL (begin with the lowest layer)}
        \For{layer $j$ in HKSL (begin with the highest layer)}
            \If{$i == j$}
                \State break loop
            \EndIf
            \State Embed first observation $o$ in $\tau$ using layer $j$'s encoder
            \If{layer $j$ is the top layer}
                \For{step layer $j$ can take in $\tau$}
                    \State Compute forward-step using layer $j$'s forward model and actions from $\tau$
                    \State Store the prediction
                \EndFor
            \Else{ layer $j$ is not the top layer}
                \For{step layer $j$ can take in $\tau$}
                    \State Compute forward-step using layer $j$'s forward model, actions from $\tau$, and output from communication manager using the stored rollout from above level
                    \State Store the prediction
                \EndFor
            \EndIf
        \EndFor
        \State Embed first observation $o$ in $\tau$ using layer $i$'s encoder
        \For{step layer $i$ can take in $\tau$}
            \State Compute forward-step using layer $i$'s forward model, actions from $\tau$, and output from communication manager using the stored rollout from above level
            \State Project the forward model's output with layer $i$'s nonlinear projection
            \State Compute loss per Equation~\ref{eqn:hksl-loss}
        \EndFor
        \State Update layer $i$'s weights
    \EndFor
\end{algorithmic}

\begin{algorithmic}
[1]
\Require trajectory $\tau$
\For{layer $i$ in HKSL (begin with the lowest layer)}
\For{layer $j$ in HKSL (begin with the highest layer)}
\If{$i == j$}
\State break loop
\EndIf
\State Embed first observation $o$ in $\tau$ using layer $j$'s encoder
\If{layer $j$ is the top layer}
\For{step layer $j$ can take in $\tau$}
\State Compute forward-step using layer $j$'s forward model and actions from $\tau$
\State Store the prediction
\EndFor
\Else{ layer $j$ is not the top layer}
\For{step layer $j$ can take in $\tau$}
\State Compute forward-step using layer $j$'s forward model, actions from $\tau$, and output from communication manager using the stored rollout from above level
\State Store the prediction
\EndFor
\EndIf
\EndFor
\State Embed first observation $o$ in $\tau$ using layer $i$'s encoder
\For{step layer $i$ can take in $\tau$}
\State Compute forward-step using layer $i$'s forward model, actions from $\tau$, and output from communication manager using the stored rollout from above level
\State Project the forward model's output with layer $i$'s nonlinear projection
\State Compute loss per Equation~\ref{eqn:hksl-loss}
\EndFor
\State Update layer $i$'s weights
\EndFor
\end{algorithmic}
{
    "title": "multi-horizon representations with hierarchical forward models for   reinforcement learning",
    "id": "2206.11396",
    "abstract": "learning control from pixels is difficult for reinforcement learning (rl) agents because representation learning and policy learning are intertwined. previous approaches remedy this issue with auxiliary representation learning tasks, but they either do not consider the temporal aspect of the problem or only consider single-step transitions, which may cause learning inefficiencies if important environmental changes take many steps to manifest. we propose hierarchical $k$-step latent (hksl), an auxiliary task that learns multiple representations via a hierarchy of forward models that learn to communicate and an ensemble of $n$-step critics that all operate at varying magnitudes of step skipping. we evaluate hksl in a suite of 30 robotic control tasks with and without distractors and a task of our creation. we find that hksl either converges to higher or optimal episodic returns more quickly than several alternative representation learning approaches. furthermore, we find that hksl's representations capture task-relevant details accurately across timescales (even in the presence of distractors) and that communication channels between hierarchy levels organize information based on both sides of the communication process, both of which improve sample efficiency.",
    "categories": "cs.lg",
    "doi": "",
    "created": "2022-06-22",
    "updated": "2024-01-29",
    "authors": [
        "trevor mcinroe",
        "lukas sch\u00e4fer",
        "stefano v. albrecht"
    ],
    "affiliation": [],
    "url": "https://arxiv.org/abs/2206.11396"
}
{
    "ToxicityFilter": {
        "text_to_toxicity_attributes": {
            "main.bbl": {
                "toxicity_score": 0.011560776,
                "severe_toxicity_score": 0.0011253357,
                "identity_attack_score": 0.0038292783,
                "insult_score": 0.0074178753,
                "profanity_score": 0.014532468,
                "threat_score": 0.0074306927,
                "sexually_explicit_score": 0.011264639,
                "flirtation_score": 0.19708732
            },
            "math_commands.tex": {
                "toxicity_score": 0.01005285,
                "severe_toxicity_score": 0.00084400177,
                "identity_attack_score": 0.0023678627,
                "insult_score": 0.0074368757,
                "profanity_score": 0.011526455,
                "threat_score": 0.006602164,
                "sexually_explicit_score": 0.0059272056,
                "flirtation_score": 0.37736848
            },
            "LICENSE": {
                "toxicity_score": 0.00823077,
                "severe_toxicity_score": 0.0007581711,
                "identity_attack_score": 0.001951637,
                "insult_score": 0.0069713728,
                "profanity_score": 0.012175481,
                "threat_score": 0.0068028234,
                "sexually_explicit_score": 0.0065759546,
                "flirtation_score": 0.17881343
            },
            "tmlr.bst": {
                "toxicity_score": 0.02048268,
                "severe_toxicity_score": 0.0017642975,
                "identity_attack_score": 0.003921773,
                "insult_score": 0.00983089,
                "profanity_score": 0.023379711,
                "threat_score": 0.008129764,
                "sexually_explicit_score": 0.021585643,
                "flirtation_score": 0.53546125
            },
            "fancyhdr.sty": {
                "toxicity_score": 0.06065326,
                "severe_toxicity_score": 0.0034332275,
                "identity_attack_score": 0.0056236745,
                "insult_score": 0.022255141,
                "profanity_score": 0.05268429,
                "threat_score": 0.007922632,
                "sexually_explicit_score": 0.036801755,
                "flirtation_score": 0.3852209
            },
            "main.bib": {
                "toxicity_score": 0.013885498,
                "severe_toxicity_score": 0.0012016296,
                "identity_attack_score": 0.0046987277,
                "insult_score": 0.008443882,
                "profanity_score": 0.015010698,
                "threat_score": 0.0077802283,
                "sexually_explicit_score": 0.0115595255,
                "flirtation_score": 0.4994272
            },
            "tmlr.sty": {
                "toxicity_score": 0.006754258,
                "severe_toxicity_score": 0.00047922134,
                "identity_attack_score": 0.0013781702,
                "insult_score": 0.006999873,
                "profanity_score": 0.011458136,
                "threat_score": 0.0057606893,
                "sexually_explicit_score": 0.005543854,
                "flirtation_score": 0.48204565
            },
            "main.tex": {
                "toxicity_score": 0.017969469,
                "severe_toxicity_score": 0.0016021729,
                "identity_attack_score": 0.0061416444,
                "insult_score": 0.009127886,
                "profanity_score": 0.018392462,
                "threat_score": 0.008764107,
                "sexually_explicit_score": 0.01875474,
                "flirtation_score": 0.36146027
            },
            "README.md": {
                "toxicity_score": 0.013822667,
                "severe_toxicity_score": 0.0014019012,
                "identity_attack_score": 0.0036997858,
                "insult_score": 0.00810188,
                "profanity_score": 0.016342908,
                "threat_score": 0.00780612,
                "sexually_explicit_score": 0.015334064,
                "flirtation_score": 0.31661955
            }
        },
        "reason": null
    }
}
{
    "num_done": {
        "figure": 0,
        "algorithm": 2
    }
}
{
    "NonTrivialRenderingFilter": {
        "white_pixels_ratio": 91.66482670393972,
        "hash": "039f8f8f8f9f801f",
        "most_frequent_color_ratio": 91.66482670393972
    }
}