\begin{algorithm}
\caption{Discrete Soft Actor-Critic for \textit{de novo} drug design}\label{alg:sac}
\begin{algorithmic}[1]
\Statex \textbf{Input:} $\phi$, $\theta$, initial episodes $K_{\mathrm{init}}$, total budget of episodes $K_{\mathrm{E}}$,
\Statex \textbf{Init:} $\phi' \gets \phi$, $\theta' \gets \theta$, $\mathcal{D} \gets \emptyset$
\For{each initial episode $1,\dots,K_{\mathrm{init}}$}
\State Sample a batch $\mathcal{T}$ of $M$ sequences using pre-trained policy $\pi_\theta$
\State Score each sequence in $\mathcal{T}$
\State Add unique, valid sequences to replay memory $\mathcal{D}$
\EndFor
\For{each episode $K_{\mathrm{init}}+1,\dots,K_{\mathrm{E}}$}
\State Sample a batch $\mathcal{T}$ of $M$ sequences using current policy $\pi_\theta$
\State Score each sequence in $\mathcal{T}$
\State Add unique, valid sequences to replay memory $\mathcal{D}$
\State $\phi \gets \phi - \lambda_Q \hat{\nabla}_\phi J_Q (\phi \vert \mathcal{T})$ \Comment{On-policy update of Q-function parameters}
\State $\theta \gets \theta - \lambda_\pi \hat{\nabla}_\theta J_\pi (\theta\vert \mathcal{T})$ \Comment{On-policy update of policy parameters}
\State $\alpha \gets \alpha - \lambda_\alpha \hat{\nabla}_\alpha J_\alpha (\alpha \vert \mathcal{T})$ \Comment{On-policy update of temperature}
\State $\phi' \gets \tau \phi' + (1-\tau) \phi$  \Comment{Update target parameters}
\State $\theta' \gets \tau \theta' + (1-\tau) \theta$  \Comment{Update average policy parameters}
\For{each off-policy update}
\State $\phi \gets \phi - \lambda_Q \hat{\nabla}_\phi J_Q (\phi \vert \mathcal{D})$
\State $\theta \gets \theta - \lambda_\pi \hat{\nabla}_\theta J_\pi (\theta\vert \mathcal{D})$
\State $\alpha \gets \alpha - \lambda_\alpha \hat{\nabla}_\alpha J_a (\alpha \vert \mathcal{D})$
\State $\phi' \gets \tau \phi' + (1-\tau) \phi$
\State $\theta' \gets \tau \theta' + (1-\tau) \theta$
\EndFor
\EndFor
\end{algorithmic}
\end{algorithm}

\begin{algorithm}
\caption{Discrete Soft Actor-Critic for \textit{de novo} drug design}\begin{algorithmic}
[1]
\Statex \textbf{Input:} $\phi$, $\theta$, initial episodes $K_{\mathrm{init}}$, total budget of episodes $K_{\mathrm{E}}$,
\Statex \textbf{Init:} $\phi' \gets \phi$, $\theta' \gets \theta$, $\mathcal{D} \gets \emptyset$
\For{each initial episode $1,\dots,K_{\mathrm{init}}$}
\State Sample a batch $\mathcal{T}$ of $M$ sequences using pre-trained policy $\pi_\theta$
\State Score each sequence in $\mathcal{T}$
\State Add unique, valid sequences to replay memory $\mathcal{D}$
\EndFor
\For{each episode $K_{\mathrm{init}}+1,\dots,K_{\mathrm{E}}$}
\State Sample a batch $\mathcal{T}$ of $M$ sequences using current policy $\pi_\theta$
\State Score each sequence in $\mathcal{T}$
\State Add unique, valid sequences to replay memory $\mathcal{D}$
\State $\phi \gets \phi - \lambda_Q \hat{\nabla}_\phi J_Q (\phi \vert \mathcal{T})$ \Comment{On-policy update of Q-function parameters}
\State $\theta \gets \theta - \lambda_\pi \hat{\nabla}_\theta J_\pi (\theta\vert \mathcal{T})$ \Comment{On-policy update of policy parameters}
\State $\alpha \gets \alpha - \lambda_\alpha \hat{\nabla}_\alpha J_\alpha (\alpha \vert \mathcal{T})$ \Comment{On-policy update of temperature}
\State $\phi' \gets \tau \phi' + (1-\tau) \phi$ \Comment{Update target parameters}
\State $\theta' \gets \tau \theta' + (1-\tau) \theta$ \Comment{Update average policy parameters}
\For{each off-policy update}
\State $\phi \gets \phi - \lambda_Q \hat{\nabla}_\phi J_Q (\phi \vert \mathcal{D})$
\State $\theta \gets \theta - \lambda_\pi \hat{\nabla}_\theta J_\pi (\theta\vert \mathcal{D})$
\State $\alpha \gets \alpha - \lambda_\alpha \hat{\nabla}_\alpha J_a (\alpha \vert \mathcal{D})$
\State $\phi' \gets \tau \phi' + (1-\tau) \phi$
\State $\theta' \gets \tau \theta' + (1-\tau) \theta$
\EndFor
\EndFor
\end{algorithmic}
\end{algorithm}
{
    "title": "utilizing reinforcement learning for de novo drug design",
    "id": "2303.17615",
    "abstract": "deep learning-based approaches for generating novel drug molecules with specific properties have gained a lot of interest in the last few years. recent studies have demonstrated promising performance for string-based generation of novel molecules utilizing reinforcement learning. in this paper, we develop a unified framework for using reinforcement learning for de novo drug design, wherein we systematically study various on- and off-policy reinforcement learning algorithms and replay buffers to learn an rnn-based policy to generate novel molecules predicted to be active against the dopamine receptor drd2. our findings suggest that it is advantageous to use at least both top-scoring and low-scoring molecules for updating the policy when structural diversity is essential. using all generated molecules at an iteration seems to enhance performance stability for on-policy algorithms. in addition, when replaying high, intermediate, and low-scoring molecules, off-policy algorithms display the potential of improving the structural diversity and number of active molecules generated, but possibly at the cost of a longer exploration phase. our work provides an open-source framework enabling researchers to investigate various reinforcement learning methods for de novo drug design.",
    "categories": "q-bio.bm cs.lg",
    "doi": "",
    "created": "2023-03-30",
    "updated": "2024-01-30",
    "authors": [
        "hampus gummesson svensson",
        "christian tyrchan",
        "ola engkvist",
        "morteza haghir chehreghani"
    ],
    "affiliation": [],
    "url": "https://arxiv.org/abs/2303.17615"
}
{
    "ToxicityFilter": {
        "text_to_toxicity_attributes": {
            "method.tex": {
                "toxicity_score": 0.03828422,
                "severe_toxicity_score": 0.0020217896,
                "identity_attack_score": 0.00621564,
                "insult_score": 0.012870909,
                "profanity_score": 0.024951037,
                "threat_score": 0.009281937,
                "sexually_explicit_score": 0.023826776,
                "flirtation_score": 0.31568006
            },
            "results.tex": {
                "toxicity_score": 0.006722843,
                "severe_toxicity_score": 0.00074386597,
                "identity_attack_score": 0.0017204003,
                "insult_score": 0.0062208683,
                "profanity_score": 0.01220964,
                "threat_score": 0.0064985976,
                "sexually_explicit_score": 0.010026119,
                "flirtation_score": 0.22919944
            },
            "sn-jnl.cls": {
                "toxicity_score": 0.011812098,
                "severe_toxicity_score": 0.0011396408,
                "identity_attack_score": 0.003366805,
                "insult_score": 0.007721877,
                "profanity_score": 0.01402008,
                "threat_score": 0.007145886,
                "sexually_explicit_score": 0.010379982,
                "flirtation_score": 0.512107
            },
            "intro.tex": {
                "toxicity_score": 0.006754258,
                "severe_toxicity_score": 0.00067710876,
                "identity_attack_score": 0.0019146391,
                "insult_score": 0.0064393696,
                "profanity_score": 0.012243799,
                "threat_score": 0.0065180166,
                "sexually_explicit_score": 0.009318393,
                "flirtation_score": 0.27036667
            },
            "conclusions.tex": {
                "toxicity_score": 0.0063144458,
                "severe_toxicity_score": 0.00064849854,
                "identity_attack_score": 0.0016464046,
                "insult_score": 0.0062588686,
                "profanity_score": 0.011475217,
                "threat_score": 0.006356194,
                "sexually_explicit_score": 0.007843964,
                "flirtation_score": 0.24166472
            },
            "sn-mathphys.bst": {
                "toxicity_score": 0.015142105,
                "severe_toxicity_score": 0.0013542175,
                "identity_attack_score": 0.004236255,
                "insult_score": 0.008690883,
                "profanity_score": 0.017572641,
                "threat_score": 0.007586042,
                "sexually_explicit_score": 0.013977589,
                "flirtation_score": 0.32881752
            },
            "abstract.tex": {
                "toxicity_score": 0.022964042,
                "severe_toxicity_score": 0.0010967255,
                "identity_attack_score": 0.00310782,
                "insult_score": 0.008386881,
                "profanity_score": 0.014805742,
                "threat_score": 0.008634649,
                "sexually_explicit_score": 0.009849187,
                "flirtation_score": 0.13218522
            },
            "appendix.tex": {
                "toxicity_score": 0.0047436883,
                "severe_toxicity_score": 0.00064849854,
                "identity_attack_score": 0.0011793067,
                "insult_score": 0.0059548668,
                "profanity_score": 0.011697251,
                "threat_score": 0.0059904763,
                "sexually_explicit_score": 0.008138849,
                "flirtation_score": 0.23938377
            },
            "sn-bibliography.bib": {
                "toxicity_score": 0.0068799187,
                "severe_toxicity_score": 0.0010442734,
                "identity_attack_score": 0.0016001574,
                "insult_score": 0.006676871,
                "profanity_score": 0.013029462,
                "threat_score": 0.006162008,
                "sexually_explicit_score": 0.009554301,
                "flirtation_score": 0.21113603
            },
            "discussion.tex": {
                "toxicity_score": 0.007916619,
                "severe_toxicity_score": 0.00084877014,
                "identity_attack_score": 0.002173624,
                "insult_score": 0.0067908717,
                "profanity_score": 0.013576009,
                "threat_score": 0.006641001,
                "sexually_explicit_score": 0.010733845,
                "flirtation_score": 0.26285875
            },
            "main.tex": {
                "toxicity_score": 0.0056547276,
                "severe_toxicity_score": 0.0012588501,
                "identity_attack_score": 0.0008648249,
                "insult_score": 0.0061068675,
                "profanity_score": 0.017436003,
                "threat_score": 0.0057930537,
                "sexually_explicit_score": 0.014331453,
                "flirtation_score": 0.22525011
            },
            "main.bbl": {
                "toxicity_score": 0.007351146,
                "severe_toxicity_score": 0.000834465,
                "identity_attack_score": 0.0024603575,
                "insult_score": 0.00654387,
                "profanity_score": 0.012585391,
                "threat_score": 0.006647474,
                "sexually_explicit_score": 0.009200439,
                "flirtation_score": 0.5493274
            },
            "scaffold.eps": {
                "toxicity_score": 0.0058118035,
                "severe_toxicity_score": 0.00071525574,
                "identity_attack_score": 0.0011515582,
                "insult_score": 0.006344369,
                "profanity_score": 0.012175481,
                "threat_score": 0.0060196044,
                "sexually_explicit_score": 0.0072836806,
                "flirtation_score": 0.49011314
            }
        },
        "reason": null
    }
}
{
    "num_done": {
        "figure": 0,
        "algorithm": 2,
        "plot": 0
    }
}
{
    "NonTrivialRenderingFilter": {
        "white_pixels_ratio": 91.15829695482692,
        "hash": "07070f83909fdf3f",
        "most_frequent_color_ratio": 91.15829695482692
    }
}