  \begin{algorithmic}[1] 
    \State Initialize the weights of an RL agent and a Self-Explainer (SE-Net)
    \State Initialize buffers $\mathcal{D}_{success}$ and $\mathcal{D}_{failure}$ for training the Self-Explainer and $D_{RL}$ for RL from Demonstrations as in the RLfD works \cite{hester2017deep,vecerik2017leveraging} 
    \State Store expert experiences into $\mathcal{D}_{success}$ and $\mathcal{D}_{RL}$. Pretrain the RL agent with experiences sampled from $\mathcal{D}_{RL}$ 
    \State Sample $K$ trajectories with a random policy and add them to $D_{RL}$. Also add successful and unsuccessful trajectories to $\mathcal{D}_{success}$ and $\mathcal{D}_{failure}$ respectively.
    
    \For{\texttt{$episode=1; episode \leq N; episode++$}} 
        \State Sample experiences (including grounded predicate values) from $\mathcal{D}_{success}$ and $\mathcal{D}_{failure}$ \Comment{\textbf{Train Self-Explainer}} 
        \State Compute utility weights $u$ and shaped reward prediction $\hat{r}_\theta(s_t,u_t,a_t,s_{t+1},u_{t+1})$ by using SE-Net, Eqs. \ref{eq:dot-prod} and \ref{eq:serl_rwd} 
        \State Update the SE-Net via binary cross entropy loss $L_{SE}$ (Eq. \ref{eq:l_se}) to distinguish successful experiences from unsuccessful experiences 
        \State Sample experiences with grounded predicate values from $\mathcal{D}_{RL}$ \Comment{\textbf{Train RL agent}}
        \State Run SE-Net on sampled states to obtain utility weights $u$ 
        \State Augment input states with grounded predicate values and utility weight values 
        \State Augment rewards with predicted shaped reward by using Eq. \ref{eq:serl_rwd} 
        \State Update RL agent with the augmented experiences 
        \State Use RL agent to sample a new trajectory and add it to $D_{RL}$. If the trajectory is successful, it would also be added to $\mathcal{D}_{success}$. Otherwise, it would also be added to $\mathcal{D}_{failure}$ \Comment{\textbf{Sample a new trajectory}}
    \EndFor \\
    \Return a trained RL agent and SE-Net 
  \end{algorithmic}

\begin{algorithmic}[1]
\State Initialize the weights of an RL agent and a Self-Explainer (SE-Net)
\State Initialize buffers $\mathcal{D}_{success}$ and $\mathcal{D}_{failure}$ for training the Self-Explainer and $D_{RL}$ for RL from Demonstrations as in the RLfD works \cite{hester2017deep,vecerik2017leveraging}
\State Store expert experiences into $\mathcal{D}_{success}$ and $\mathcal{D}_{RL}$. Pretrain the RL agent with experiences sampled from $\mathcal{D}_{RL}$
\State Sample $K$ trajectories with a random policy and add them to $D_{RL}$. Also add successful and unsuccessful trajectories to $\mathcal{D}_{success}$ and $\mathcal{D}_{failure}$ respectively.
\For{\texttt{$episode=1; episode \leq N; episode++$}}
\State Sample experiences (including grounded predicate values) from $\mathcal{D}_{success}$ and $\mathcal{D}_{failure}$ \Comment{\textbf{Train Self-Explainer}}
\State Compute utility weights $u$ and shaped reward prediction $\hat{r}_\theta(s_t,u_t,a_t,s_{t+1},u_{t+1})$ by using SE-Net, Eqs. \ref{eq:dot-prod} and \ref{eq:serl_rwd}
\State Update the SE-Net via binary cross entropy loss $L_{SE}$ (Eq. \ref{eq:l_se}) to distinguish successful experiences from unsuccessful experiences
\State Sample experiences with grounded predicate values from $\mathcal{D}_{RL}$ \Comment{\textbf{Train RL agent}}
\State Run SE-Net on sampled states to obtain utility weights $u$
\State Augment input states with grounded predicate values and utility weight values
\State Augment rewards with predicted shaped reward by using Eq. \ref{eq:serl_rwd}
\State Update RL agent with the augmented experiences
\State Use RL agent to sample a new trajectory and add it to $D_{RL}$. If the trajectory is successful, it would also be added to $\mathcal{D}_{success}$. Otherwise, it would also be added to $\mathcal{D}_{failure}$ \Comment{\textbf{Sample a new trajectory}}
\EndFor \\
\Return a trained RL agent and SE-Net
\end{algorithmic}
{
    "title": "learning from ambiguous demonstrations with self-explanation guided   reinforcement learning",
    "id": "2110.05286",
    "abstract": "our work aims at efficiently leveraging ambiguous demonstrations for the training of a reinforcement learning (rl) agent. an ambiguous demonstration can usually be interpreted in multiple ways, which severely hinders the rl-agent from learning stably and efficiently. since an optimal demonstration may also suffer from being ambiguous, previous works that combine rl and learning from demonstration (rlfd works) may not work well. inspired by how humans handle such situations, we propose to use self-explanation (an agent generates explanations for itself) to recognize valuable high-level relational features as an interpretation of why a successful trajectory is successful. this way, the agent can provide some guidance for its rl learning. our main contribution is to propose the self-explanation for rl from demonstrations (serlfd) framework, which can overcome the limitations of traditional rlfd works. our experimental results show that an rlfd model can be improved by using our serlfd framework in terms of training stability and performance.",
    "categories": "cs.lg",
    "doi": "",
    "created": "2021-10-11",
    "updated": "2024-02-07",
    "authors": [
        "yantian zha",
        "lin guan",
        "subbarao kambhampati"
    ],
    "affiliation": [],
    "url": "https://arxiv.org/abs/2110.05286"
}
{
    "ToxicityFilter": {
        "text_to_toxicity_attributes": {
            "output.tex": {
                "toxicity_score": 0.007099825,
                "severe_toxicity_score": 0.0007724762,
                "identity_attack_score": 0.0020256327,
                "insult_score": 0.0065913703,
                "profanity_score": 0.012858666,
                "threat_score": 0.0066345283,
                "sexually_explicit_score": 0.010321004,
                "flirtation_score": 0.527787
            },
            "aaai24.sty": {
                "toxicity_score": 0.004837934,
                "severe_toxicity_score": 0.0011110306,
                "identity_attack_score": 0.00069833454,
                "insult_score": 0.0057743657,
                "profanity_score": 0.015079016,
                "threat_score": 0.0055859215,
                "sexually_explicit_score": 0.018164968,
                "flirtation_score": 0.38160232
            },
            "aaai24.bst": {
                "toxicity_score": 0.016712861,
                "severe_toxicity_score": 0.0015354156,
                "identity_attack_score": 0.0039402717,
                "insult_score": 0.009165886,
                "profanity_score": 0.018324144,
                "threat_score": 0.0075083673,
                "sexually_explicit_score": 0.015805881,
                "flirtation_score": 0.40847883
            },
            "output.bbl": {
                "toxicity_score": 0.012817383,
                "severe_toxicity_score": 0.0011587143,
                "identity_attack_score": 0.004069764,
                "insult_score": 0.0077028773,
                "profanity_score": 0.015044857,
                "threat_score": 0.0076766624,
                "sexually_explicit_score": 0.011854411,
                "flirtation_score": 0.20439981
            },
            "sesl.bib": {
                "toxicity_score": 0.008670582,
                "severe_toxicity_score": 0.001168251,
                "identity_attack_score": 0.0016371552,
                "insult_score": 0.0069238725,
                "profanity_score": 0.014088398,
                "threat_score": 0.0067057298,
                "sexually_explicit_score": 0.0143904295,
                "flirtation_score": 0.24170095
            },
            "aaai24.bib": {
                "toxicity_score": 0.009927189,
                "severe_toxicity_score": 0.0015163422,
                "identity_attack_score": 0.0018591423,
                "insult_score": 0.007322875,
                "profanity_score": 0.016411226,
                "threat_score": 0.0065697995,
                "sexually_explicit_score": 0.015334064,
                "flirtation_score": 0.3011931
            },
            "new.tex": {
                "toxicity_score": 0.007099825,
                "severe_toxicity_score": 0.0007724762,
                "identity_attack_score": 0.0020256327,
                "insult_score": 0.0065913703,
                "profanity_score": 0.012858666,
                "threat_score": 0.0066345283,
                "sexually_explicit_score": 0.010321004,
                "flirtation_score": 0.527787
            },
            "placeonpage.sty": {
                "toxicity_score": 0.016838523,
                "severe_toxicity_score": 0.0020599365,
                "identity_attack_score": 0.0023493639,
                "insult_score": 0.008633883,
                "profanity_score": 0.021330157,
                "threat_score": 0.0074565844,
                "sexually_explicit_score": 0.023590868,
                "flirtation_score": 0.35887584
            }
        },
        "reason": null
    }
}
{
    "num_done": {
        "figure": 0,
        "algorithm": 2,
        "plot": 0
    }
}
{
    "NonTrivialRenderingFilter": {
        "white_pixels_ratio": 87.26165212070758,
        "hash": "010f098903bf831f",
        "most_frequent_color_ratio": 87.26165212070758
    }
}