\begin{algorithmic}[1]

\State Set the step-size $(\alpha_k)_{k=0}^\infty$, and initialize $(\theta _0,\lambda_0 )$.


\For{$k \in \{0,\ldots\}$}


\State Observe $s_k \sim d^{\beta}$, $a_k \sim \beta(\cdot|s_k)$, and $s_k'\sim P(\cdot | s_k,a_k)$, $r_k :=r(s_k,a_k,s_k')$.

\State Update parameters according to
\begin{align*}
&\theta _{k + 1}  = \theta _k  + \alpha _k [(\phi_k  - \gamma \rho _k \phi_{k}')(\phi_k ^T \lambda _k ) - \phi_k (\phi_k ^T \theta_k )],\\
&\lambda _{k + 1}  = \lambda _k  + \alpha _k \delta _k \phi_k,
\end{align*}
where $\phi_k:=\phi(s_k),\phi_{k}':=\phi(s_{k}')$, $\rho_k : = \frac{{\pi (a_k |s_k )}}{{\beta (a_k |s_k )}}$, and $\delta_k =\rho _k r_k +\gamma \rho _k (\phi_{k}')^T \theta_k -\phi_k^T \theta_k$.



\EndFor
\end{algorithmic}

\begin{algorithmic}
[1]
\State Set the step-size $(\alpha_k)_{k=0}^\infty$, and initialize $(\theta _0,\lambda_0 )$.
\For{$k \in \{0,\ldots\}$}
\State Observe $s_k \sim d^{\beta}$, $a_k \sim \beta(\cdot|s_k)$, and $s_k'\sim P(\cdot | s_k,a_k)$, $r_k :=r(s_k,a_k,s_k')$.
\State Update parameters according to
\begin{align*}
&\theta _{k + 1} = \theta _k + \alpha _k [(\phi_k - \gamma \rho _k \phi_{k}')(\phi_k ^T \lambda _k ) - \phi_k (\phi_k ^T \theta_k )],\\
&\lambda _{k + 1} = \lambda _k + \alpha _k \delta _k \phi_k,
\end{align*}
where $\phi_k:=\phi(s_k),\phi_{k}':=\phi(s_{k}')$, $\rho_k : = \frac{{\pi (a_k |s_k )}}{{\beta (a_k |s_k )}}$, and $\delta_k =\rho _k r_k +\gamma \rho _k (\phi_{k}')^T \theta_k -\phi_k^T \theta_k$.
\EndFor
\end{algorithmic}
{
    "title": "new versions of gradient temporal difference learning",
    "id": "2109.04033",
    "abstract": "sutton, szepesv\\'{a}ri and maei introduced the first gradient temporal-difference (gtd) learning algorithms compatible with both linear function approximation and off-policy training. the goal of this paper is (a) to propose some variants of gtds with extensive comparative analysis and (b) to establish new theoretical analysis frameworks for the gtds. these variants are based on convex-concave saddle-point interpretations of gtds, which effectively unify all the gtds into a single framework, and provide simple stability analysis based on recent results on primal-dual gradient dynamics. finally, numerical comparative analysis is given to evaluate these approaches.",
    "categories": "cs.lg",
    "doi": "",
    "created": "2021-09-09",
    "updated": "2024-01-22",
    "authors": [
        "donghwan lee",
        "han-dong lim",
        "jihoon park",
        "okyong choi"
    ],
    "affiliation": [],
    "url": "https://arxiv.org/abs/2109.04033"
}
{
    "ToxicityFilter": {
        "text_to_toxicity_attributes": {
            "fig3.eps": {
                "toxicity_score": 0.0058118035,
                "severe_toxicity_score": 0.00071525574,
                "identity_attack_score": 0.0011515582,
                "insult_score": 0.006344369,
                "profanity_score": 0.012175481,
                "threat_score": 0.0060196044,
                "sexually_explicit_score": 0.0072836806,
                "flirtation_score": 0.49011314
            },
            "paper.bbl": {
                "toxicity_score": 0.00816794,
                "severe_toxicity_score": 0.0008869171,
                "identity_attack_score": 0.0030338243,
                "insult_score": 0.006999873,
                "profanity_score": 0.013336895,
                "threat_score": 0.0070164283,
                "sexually_explicit_score": 0.01020305,
                "flirtation_score": 0.5114233
            },
            "fig5.eps": {
                "toxicity_score": 0.0058118035,
                "severe_toxicity_score": 0.00071525574,
                "identity_attack_score": 0.0011515582,
                "insult_score": 0.006344369,
                "profanity_score": 0.012175481,
                "threat_score": 0.0060196044,
                "sexually_explicit_score": 0.0072836806,
                "flirtation_score": 0.49011314
            },
            "fig4.eps": {
                "toxicity_score": 0.0058118035,
                "severe_toxicity_score": 0.00071525574,
                "identity_attack_score": 0.0011515582,
                "insult_score": 0.006344369,
                "profanity_score": 0.012175481,
                "threat_score": 0.0060196044,
                "sexually_explicit_score": 0.0072836806,
                "flirtation_score": 0.49011314
            },
            "fig2.eps": {
                "toxicity_score": 0.0058118035,
                "severe_toxicity_score": 0.00071525574,
                "identity_attack_score": 0.0011515582,
                "insult_score": 0.006344369,
                "profanity_score": 0.012175481,
                "threat_score": 0.0060196044,
                "sexually_explicit_score": 0.0072836806,
                "flirtation_score": 0.49011314
            },
            "paper.tex": {
                "toxicity_score": 0.008105109,
                "severe_toxicity_score": 0.0011301041,
                "identity_attack_score": 0.0023123662,
                "insult_score": 0.0067908717,
                "profanity_score": 0.014600786,
                "threat_score": 0.006550381,
                "sexually_explicit_score": 0.012562137,
                "flirtation_score": 0.3556029
            },
            "fig1.eps": {
                "toxicity_score": 0.0058118035,
                "severe_toxicity_score": 0.00071525574,
                "identity_attack_score": 0.0011515582,
                "insult_score": 0.006344369,
                "profanity_score": 0.012175481,
                "threat_score": 0.0060196044,
                "sexually_explicit_score": 0.0072836806,
                "flirtation_score": 0.49011314
            }
        },
        "reason": null
    }
}
{
    "num_done": {
        "figure": 0,
        "algorithm": 3
    }
}
{
    "NonTrivialRenderingFilter": {
        "white_pixels_ratio": 93.15865577254752,
        "hash": "07000ff1cfff007f",
        "most_frequent_color_ratio": 93.15865577254752
    }
}