\begin{algorithmic} [1]
\State Initialize actor network $\mu_\theta$ and critic networks $Q_{\omega_1}$ and $Q_{\omega_2}$.
\State Initialize corresponding target networks: $\theta' \leftarrow \theta$, ${\omega'}_1 \leftarrow \omega_1$, ${\omega'}_2 \leftarrow \omega_2$ and choose $\rho_\text{p} \in (0, 1)$.
\State Initialize replay buffer $\mathcal{R}$.
\State Choose a batch size $K$, a gradient based optimization algorithm and a corresponding learning rate $\lambda_\text{actor}, \lambda_\text{critic} > 0$ for both optimization procedures, a time step size $\Delta t$ and a stopping criterion.
\State Choose standard deviation exploration noise $\sigma_\text{expl}$ and lower and upper action bounds $a_\text{low}, a_\text{high}$.
\Repeat
\State Select clipped action and step the environment dynamics forward.
\begin{equation*}
a= \text{clip}(\mu_\theta(s) + \epsilon, a_\text{low}, a_\text{high}), \quad \epsilon \sim \mathcal{N}(0, \sigma_\text{expl}).
\end{equation*}
\State Observe next state $s'$, reward $r$, and done signal $d$ and store the tuple $(s, a, r, s', d)$ in the replay buffer.
\If{$s'$ is terminal} 
\State Reset trajectory. 
\EndIf
\For{$j$ in range(\textit{update frequency})}
\State Sample batch $\mathcal{B} = \{(s^{(k)}, a^{(k)}, r^{(k)}, {s'}^{(k)}, d^{(k)})\}_{k=1}^K$ from replay buffer.
\State Compute targets (Clipped Double Q-learning and policy smoothing).
\begin{equation*}
y(r, s', d) = r + (1-d) \min\limits_{i=1, 2} \left\{ Q_{{\omega'}_i}(s', \tilde{a}) \right\}, \quad \tilde{a}= \text{clip}(\mu_{\theta'}(s) + \epsilon, a_\text{low}, a_\text{high}), \quad \epsilon \sim \mathcal{N}(0, \sigma_\text{target}). 
\end{equation*}

\State Estimate critic gradient $\nabla_\omega L(Q_\omega^{\mu_\theta})$ by
\begin{equation*}
\nabla_{\omega_i} \Biggl(\frac{1}{K} \sum\limits_{k=1}^K \left(Q_{\omega_i}(s^{(k)}, a^{(k)}) - y(r^{(k)}, {s'}^{(k)}, d^{(k)}) \right)^2 \Biggr), \quad \text{for} \,\, i=1,2.
\end{equation*}
\State Update the critic parameters $\omega_i$ based on the optimization algorithm.
\If{$j \text{ mod } \textit{policy delay frequency} = 0$}
\State Estimate actor gradient $\nabla_\theta J(\mu_\theta)$ by
\begin{equation*}
\nabla_{\theta} \Biggl(\frac{1}{K} \sum\limits_{k=1}^K Q_{\omega_1}(s^{(k)}, \mu_\theta(s^{(k)})) \Biggr).
\end{equation*}
\State Update the actor parameters $\theta$ based on the optimization algorithm.
\State Update target networks softly: 
\begin{equation*}
\theta' \leftarrow \rho_\text{p} \theta' + (1 - \rho_\text{p})\theta, \quad 
{\omega'}_i \leftarrow \rho_\text{p} {\omega'}_i + (1 - \rho_\text{p})\omega_i, \quad \text{for} \, \, i=1, 2.
\end{equation*}
\EndIf
\EndFor
\Until{stopping criterion is fulfilled.}
\end{algorithmic}

\begin{algorithmic}
[1]
\State Initialize actor network $\mu_\theta$ and critic networks $Q_{\omega_1}$ and $Q_{\omega_2}$.
\State Initialize corresponding target networks: $\theta' \leftarrow \theta$, ${\omega'}_1 \leftarrow \omega_1$, ${\omega'}_2 \leftarrow \omega_2$ and choose $\rho_\text{p} \in (0, 1)$.
\State Initialize replay buffer $\mathcal{R}$.
\State Choose a batch size $K$, a gradient based optimization algorithm and a corresponding learning rate $\lambda_\text{actor}, \lambda_\text{critic} > 0$ for both optimization procedures, a time step size $\Delta t$ and a stopping criterion.
\State Choose standard deviation exploration noise $\sigma_\text{expl}$ and lower and upper action bounds $a_\text{low}, a_\text{high}$.
\Repeat
\State Select clipped action and step the environment dynamics forward.
\begin{equation*}
a= \text{clip}(\mu_\theta(s) + \epsilon, a_\text{low}, a_\text{high}), \quad \epsilon \sim \mathcal{N}(0, \sigma_\text{expl}).
\end{equation*}
\State Observe next state $s'$, reward $r$, and done signal $d$ and store the tuple $(s, a, r, s', d)$ in the replay buffer.
\If{$s'$ is terminal}
\State Reset trajectory.
\EndIf
\For{$j$ in range(\textit{update frequency})}
\State Sample batch $\mathcal{B} = \{(s^{(k)}, a^{(k)}, r^{(k)}, {s'}^{(k)}, d^{(k)})\}_{k=1}^K$ from replay buffer.
\State Compute targets (Clipped Double Q-learning and policy smoothing).
\begin{equation*}
y(r, s', d) = r + (1-d) \min\limits_{i=1, 2} \left\{ Q_{{\omega'}_i}(s', \tilde{a}) \right\}, \quad \tilde{a}= \text{clip}(\mu_{\theta'}(s) + \epsilon, a_\text{low}, a_\text{high}), \quad \epsilon \sim \mathcal{N}(0, \sigma_\text{target}).
\end{equation*}
\State Estimate critic gradient $\nabla_\omega L(Q_\omega^{\mu_\theta})$ by
\begin{equation*}
\nabla_{\omega_i} \Biggl(\frac{1}{K} \sum\limits_{k=1}^K \left(Q_{\omega_i}(s^{(k)}, a^{(k)}) - y(r^{(k)}, {s'}^{(k)}, d^{(k)}) \right)^2 \Biggr), \quad \text{for} \,\, i=1,2.
\end{equation*}
\State Update the critic parameters $\omega_i$ based on the optimization algorithm.
\If{$j \text{ mod } \textit{policy delay frequency} = 0$}
\State Estimate actor gradient $\nabla_\theta J(\mu_\theta)$ by
\begin{equation*}
\nabla_{\theta} \Biggl(\frac{1}{K} \sum\limits_{k=1}^K Q_{\omega_1}(s^{(k)}, \mu_\theta(s^{(k)})) \Biggr).
\end{equation*}
\State Update the actor parameters $\theta$ based on the optimization algorithm.
\State Update target networks softly:
\begin{equation*}
\theta' \leftarrow \rho_\text{p} \theta' + (1 - \rho_\text{p})\theta, \quad
{\omega'}_i \leftarrow \rho_\text{p} {\omega'}_i + (1 - \rho_\text{p})\omega_i, \quad \text{for} \, \, i=1, 2.
\end{equation*}
\EndIf
\EndFor
\Until{stopping criterion is fulfilled.}
\end{algorithmic}
{
    "title": "connecting stochastic optimal control and reinforcement learning",
    "id": "2211.02474",
    "abstract": "in this paper the connection between stochastic optimal control and reinforcement learning is investigated. our main motivation is to apply importance sampling to sampling rare events which can be reformulated as an optimal control problem. by using a parameterised approach the optimal control problem becomes a stochastic optimization problem which still raises some open questions regarding how to tackle the scalability to high-dimensional problems and how to deal with the intrinsic metastability of the system. to explore new methods we link the optimal control problem to reinforcement learning since both share the same underlying framework, namely a markov decision process (mdp). for the optimal control problem we show how the mdp can be formulated. in addition we discuss how the stochastic optimal control problem can be interpreted in the framework of reinforcement learning. at the end of the article we present the application of two different reinforcement learning algorithms to the optimal control problem and a comparison of the advantages and disadvantages of the two algorithms.",
    "categories": "math.oc",
    "doi": "",
    "created": "2022-11-04",
    "updated": "2024-02-15",
    "authors": [
        "jannes quer",
        "enric ribera borrell"
    ],
    "affiliation": [],
    "url": "https://arxiv.org/abs/2211.02474"
}
{
    "ToxicityFilter": {
        "text_to_toxicity_attributes": {
            "references.bib": {
                "toxicity_score": 0.006377276,
                "severe_toxicity_score": 0.0011634827,
                "identity_attack_score": 0.0011284347,
                "insult_score": 0.006714871,
                "profanity_score": 0.013610168,
                "threat_score": 0.0058513097,
                "sexually_explicit_score": 0.011972365,
                "flirtation_score": 0.18740572
            },
            "main.tex": {
                "toxicity_score": 0.010304171,
                "severe_toxicity_score": 0.0010061264,
                "identity_attack_score": 0.0032558115,
                "insult_score": 0.007189874,
                "profanity_score": 0.014259194,
                "threat_score": 0.0072882893,
                "sexually_explicit_score": 0.0108517995,
                "flirtation_score": 0.18085068
            },
            "main.bbl": {
                "toxicity_score": 0.011874928,
                "severe_toxicity_score": 0.0011348724,
                "identity_attack_score": 0.003866276,
                "insult_score": 0.0075508766,
                "profanity_score": 0.014532468,
                "threat_score": 0.007521313,
                "sexually_explicit_score": 0.010969753,
                "flirtation_score": 0.27629572
            }
        },
        "reason": null
    }
}
{
    "num_done": {
        "table": 0,
        "figure": 0,
        "algorithm": 3
    }
}
{
    "NonTrivialRenderingFilter": {
        "white_pixels_ratio": 93.46838996998741,
        "hash": "1f0f80003f3f1f7f",
        "most_frequent_color_ratio": 93.46838996998741
    }
}