\begin{algorithmic}
\For{$\ell =0,\cdots,T_\text{out}-1$}
\For{$k=0,\cdots,T_\text{in}-1$} (concurrently for all $s\in\mathcal{S}$)
\State $\theta_{k+1}^{(i)} \gets (1-\eta\lambda)\theta_k^{(i)}-2\eta \nabla_{\theta,k}^{(i)}+\eta\widetilde{\nabla}_\theta^{(i)} +\sqrt{2\lambda\eta} \cdot\omega_k^{(i)}$, $\omega_k^{(i)}\sim\mathcal{N}(0,\textbf{I})$

\State $\phi_{k+1}^{(i)} \gets (1-\eta\lambda)\phi_k^{(i)}+2\eta \nabla_{\phi,k}^{(i)}-\eta\widetilde{\nabla}_\phi^{(i)} +\sqrt{2\lambda\eta} \cdot\beta_k^{(i)}$, $\beta_k^{(i)}\sim\mathcal{N}(0,\textbf{I})$
\State (computed with respect to the current action-value network $Q_\ell$)

\If{$k$ mod $K \equiv 0$}
\State $\widetilde{\nabla}_\theta^{(i)}, \widetilde{\nabla}_\phi^{(i)} \gets \nabla_{\theta,k}^{(i)}, \nabla_{\phi,k}^{(i)}$
\EndIf
\EndFor
\State perform some TD learning update $(Q_{\ell+1}(x,y|s))_{s\in\mathcal{S}}\gets (Q_\ell(x,y|s))_{s\in\mathcal{S}}$ using $\theta_{T_\text{in}}, \phi_{T_\text{in}}$
\EndFor

\Return $Q_{T_\text{out}}, \theta_{T_\text{in}}, \phi_{T_\text{in}}$
\end{algorithmic}

\begin{algorithmic}
\For{$\ell =0,\cdots,T_\text{out}-1$}
\For{$k=0,\cdots,T_\text{in}-1$} (concurrently for all $s\in\mathcal{S}$)
\State $\theta_{k+1}^{(i)} \gets (1-\eta\lambda)\theta_k^{(i)}-2\eta \nabla_{\theta,k}^{(i)}+\eta\widetilde{\nabla}_\theta^{(i)} +\sqrt{2\lambda\eta} \cdot\omega_k^{(i)}$, $\omega_k^{(i)}\sim\mathcal{N}(0,\textbf{I})$
\State $\phi_{k+1}^{(i)} \gets (1-\eta\lambda)\phi_k^{(i)}+2\eta \nabla_{\phi,k}^{(i)}-\eta\widetilde{\nabla}_\phi^{(i)} +\sqrt{2\lambda\eta} \cdot\beta_k^{(i)}$, $\beta_k^{(i)}\sim\mathcal{N}(0,\textbf{I})$
\State (computed with respect to the current action-value network $Q_\ell$)
\If{$k$ mod $K \equiv 0$}
\State $\widetilde{\nabla}_\theta^{(i)}, \widetilde{\nabla}_\phi^{(i)} \gets \nabla_{\theta,k}^{(i)}, \nabla_{\phi,k}^{(i)}$
\EndIf
\EndFor
\State perform some TD learning update $(Q_{\ell+1}(x,y|s))_{s\in\mathcal{S}}\gets (Q_\ell(x,y|s))_{s\in\mathcal{S}}$ using $\theta_{T_\text{in}}, \phi_{T_\text{in}}$
\EndFor
\Return $Q_{T_\text{out}}, \theta_{T_\text{in}}, \phi_{T_\text{in}}$
\end{algorithmic}
{
    "title": "symmetric mean-field langevin dynamics for distributional minimax   problems",
    "id": "2312.01127",
    "abstract": "in this paper, we extend mean-field langevin dynamics to minimax optimization over probability distributions for the first time with symmetric and provably convergent updates. we propose mean-field langevin averaged gradient (mfl-ag), a single-loop algorithm that implements gradient descent ascent in the distribution spaces with a novel weighted averaging, and establish average-iterate convergence to the mixed nash equilibrium. we also study both time and particle discretization regimes and prove a new uniform-in-time propagation of chaos result which accounts for the dependency of the particle interactions on all previous distributions. furthermore, we propose mean-field langevin anchored best response (mfl-abr), a symmetric double-loop algorithm based on best response dynamics with linear last-iterate convergence. finally, we study applications to zero-sum markov games and conduct simulations demonstrating long-term optimality.",
    "categories": "math.oc stat.ml",
    "doi": "",
    "created": "2023-12-02",
    "updated": "2024-02-16",
    "authors": [
        "juno kim",
        "kakei yamamoto",
        "kazusato oko",
        "zhuoran yang",
        "taiji suzuki"
    ],
    "affiliation": [],
    "url": "https://arxiv.org/abs/2312.01127"
}
{
    "ToxicityFilter": {
        "text_to_toxicity_attributes": {
            "Misc/rebuttal.tex": {
                "toxicity_score": 0.010304171,
                "severe_toxicity_score": 0.0010061264,
                "identity_attack_score": 0.0032558115,
                "insult_score": 0.007189874,
                "profanity_score": 0.014259194,
                "threat_score": 0.0072882893,
                "sexually_explicit_score": 0.0108517995,
                "flirtation_score": 0.18085068
            },
            "fancyhdr.sty": {
                "toxicity_score": 0.031449065,
                "severe_toxicity_score": 0.0022506714,
                "identity_attack_score": 0.005956655,
                "insult_score": 0.013440913,
                "profanity_score": 0.02795705,
                "threat_score": 0.009023022,
                "sexually_explicit_score": 0.022883141,
                "flirtation_score": 0.34986767
            },
            "Misc/ideas.tex": {
                "toxicity_score": 0.010304171,
                "severe_toxicity_score": 0.0010061264,
                "identity_attack_score": 0.0032558115,
                "insult_score": 0.007189874,
                "profanity_score": 0.014259194,
                "threat_score": 0.0072882893,
                "sexually_explicit_score": 0.0108517995,
                "flirtation_score": 0.18085068
            },
            "iclr2024_conference.bib": {
                "toxicity_score": 0.009675868,
                "severe_toxicity_score": 0.0009918213,
                "identity_attack_score": 0.0024788564,
                "insult_score": 0.0072088744,
                "profanity_score": 0.014771583,
                "threat_score": 0.006874025,
                "sexually_explicit_score": 0.010733845,
                "flirtation_score": 0.20372084
            },
            "natbib.sty": {
                "toxicity_score": 0.05750068,
                "severe_toxicity_score": 0.0035858154,
                "identity_attack_score": 0.008139528,
                "insult_score": 0.02173949,
                "profanity_score": 0.043415543,
                "threat_score": 0.008168601,
                "sexually_explicit_score": 0.028662905,
                "flirtation_score": 0.41074932
            },
            "iclr2024_conference.bst": {
                "toxicity_score": 0.014765122,
                "severe_toxicity_score": 0.0014591217,
                "identity_attack_score": 0.0030153254,
                "insult_score": 0.008709883,
                "profanity_score": 0.019553876,
                "threat_score": 0.00746953,
                "sexually_explicit_score": 0.01686747,
                "flirtation_score": 0.5479165
            },
            "iclr2024_conference.bbl": {
                "toxicity_score": 0.013697007,
                "severe_toxicity_score": 0.0012207031,
                "identity_attack_score": 0.0043102503,
                "insult_score": 0.007835878,
                "profanity_score": 0.015488927,
                "threat_score": 0.0077672824,
                "sexually_explicit_score": 0.012444182,
                "flirtation_score": 0.20307733
            },
            "iclr2024_conference.sty": {
                "toxicity_score": 0.011937759,
                "severe_toxicity_score": 0.00068187714,
                "identity_attack_score": 0.0021643746,
                "insult_score": 0.0094128875,
                "profanity_score": 0.01220964,
                "threat_score": 0.0058739646,
                "sexually_explicit_score": 0.0057207854,
                "flirtation_score": 0.47478196
            },
            "iclr2024_conference.tex": {
                "toxicity_score": 0.067380086,
                "severe_toxicity_score": 0.0039482117,
                "identity_attack_score": 0.015254694,
                "insult_score": 0.016708933,
                "profanity_score": 0.04516909,
                "threat_score": 0.024062606,
                "sexually_explicit_score": 0.045746468,
                "flirtation_score": 0.19775106
            }
        },
        "reason": null
    }
}
{
    "num_done": {
        "figure": 0,
        "algorithm": 2,
        "plot": 0
    }
}
{
    "NonTrivialRenderingFilter": {
        "white_pixels_ratio": 92.06504280033691,
        "hash": "0780808fbf3f7f3f",
        "most_frequent_color_ratio": 92.06504280033691
    }
}