\begin{algorithm}[t]
\caption{Adv-NTK (Solving Eq.~(\ref{eq:adv-ntk-objective}) with SGD and GradNorm)}
\label{algo:advntk}
\begin{algorithmic}[1]
\Require
Training set $\mathcal{D}$, validation set size $M_{\mathrm{val}}$, learning rate $\zeta$, training iteration $T$, PGD function for finding adversarial validation data.
\Ensure An infinite-width adversarially robust DNN.
    \State Randomly separate $\mathcal D$ into subsets $\mathcal D_{\mathrm{opt}}$ and $\mathcal D_{\mathrm{val}}$ such that $| \mathcal D_{\mathrm{val}} | = M_{\mathrm{val}}$.
    \State Initialize trainable parameter $\varpi_0 \in \mathbb{R}^{|\mathcal D_{\mathrm{val}}| \cdot c}$ with zeros.
    \For{$t$ \textbf{in} $1, \cdots, T$}
        \State Sample a minibatch $(x, y) \sim \mathcal D_{\mathrm{val}}$.
        \State $x' \leftarrow \mathrm{PGD}(x,y,f_{\varpi_{t-1}})$ 
        \Comment{Finding adversarial validation examples.}
        \State $g_t \leftarrow \partial_{\varpi} \frac{1}{2} \| f_{\varpi_{t-1}}(x') - y \|_2^2$
        \State $\varpi_t \leftarrow \varpi_{t-1} - \zeta \cdot \frac{g_t}{\|g_t\|_2}$
        \Comment{Update model parameter via SGD and $\ell_2$-GardNorm.}
    \EndFor
    \State \Return $f_{\varpi_T}$
\end{algorithmic}
\end{algorithm}

\begin{algorithm}
[t]
\caption{Adv-NTK (Solving Eq.~(\ref{eq:adv-ntk-objective}) with SGD and GradNorm)}
\begin{algorithmic}
[1]
\Require
Training set $\mathcal{D}$, validation set size $M_{\mathrm{val}}$, learning rate $\zeta$, training iteration $T$, PGD function for finding adversarial validation data.
\Ensure An infinite-width adversarially robust DNN.
\State Randomly separate $\mathcal D$ into subsets $\mathcal D_{\mathrm{opt}}$ and $\mathcal D_{\mathrm{val}}$ such that $| \mathcal D_{\mathrm{val}} | = M_{\mathrm{val}}$.
\State Initialize trainable parameter $\varpi_0 \in \mathbb{R}^{|\mathcal D_{\mathrm{val}}| \cdot c}$ with zeros.
\For{$t$ \textbf{in} $1, \cdots, T$}
\State Sample a minibatch $(x, y) \sim \mathcal D_{\mathrm{val}}$.
\State $x' \leftarrow \mathrm{PGD}(x,y,f_{\varpi_{t-1}})$
\Comment{Finding adversarial validation examples.}
\State $g_t \leftarrow \partial_{\varpi} \frac{1}{2} \| f_{\varpi_{t-1}}(x') - y \|_2^2$
\State $\varpi_t \leftarrow \varpi_{t-1} - \zeta \cdot \frac{g_t}{\|g_t\|_2}$
\Comment{Update model parameter via SGD and $\ell_2$-GardNorm.}
\EndFor
\State \Return $f_{\varpi_T}$
\end{algorithmic}
\end{algorithm}
{
    "title": "theoretical analysis of robust overfitting for wide dnns: an ntk   approach",
    "id": "2310.06112",
    "abstract": "adversarial training (at) is a canonical method for enhancing the robustness of deep neural networks (dnns). however, recent studies empirically demonstrated that it suffers from robust overfitting, i.e., a long time at can be detrimental to the robustness of dnns. this paper presents a theoretical explanation of robust overfitting for dnns. specifically, we non-trivially extend the neural tangent kernel (ntk) theory to at and prove that an adversarially trained wide dnn can be well approximated by a linearized dnn. moreover, for squared loss, closed-form at dynamics for the linearized dnn can be derived, which reveals a new at degeneration phenomenon: a long-term at will result in a wide dnn degenerates to that obtained without at and thus cause robust overfitting. based on our theoretical results, we further design a method namely adv-ntk, the first at algorithm for infinite-width dnns. experiments on real-world datasets show that adv-ntk can help infinite-width dnns enhance comparable robustness to that of their finite-width counterparts, which in turn justifies our theoretical findings. the code is available at https://github.com/fshp971/adv-ntk.",
    "categories": "cs.lg stat.ml",
    "doi": "",
    "created": "2023-10-09",
    "updated": "2024-02-04",
    "authors": [
        "shaopeng fu",
        "di wang"
    ],
    "affiliation": [],
    "url": "https://arxiv.org/abs/2310.06112"
}
{
    "ToxicityFilter": {
        "text_to_toxicity_attributes": {
            "fancyhdr.sty": {
                "toxicity_score": 0.031449065,
                "severe_toxicity_score": 0.0022506714,
                "identity_attack_score": 0.005956655,
                "insult_score": 0.013440913,
                "profanity_score": 0.02795705,
                "threat_score": 0.009023022,
                "sexually_explicit_score": 0.022883141,
                "flirtation_score": 0.34986767
            },
            "atntk.tex": {
                "toxicity_score": 0.015016444,
                "severe_toxicity_score": 0.0013828278,
                "identity_attack_score": 0.0052166977,
                "insult_score": 0.008386881,
                "profanity_score": 0.016821137,
                "threat_score": 0.008259222,
                "sexually_explicit_score": 0.016395653,
                "flirtation_score": 0.35838574
            },
            "atntk.bbl": {
                "toxicity_score": 0.011812098,
                "severe_toxicity_score": 0.0011110306,
                "identity_attack_score": 0.0037737815,
                "insult_score": 0.007493876,
                "profanity_score": 0.014498309,
                "threat_score": 0.0074824756,
                "sexually_explicit_score": 0.011323617,
                "flirtation_score": 0.21378003
            },
            "natbib.sty": {
                "toxicity_score": 0.05750068,
                "severe_toxicity_score": 0.0035858154,
                "identity_attack_score": 0.008139528,
                "insult_score": 0.02173949,
                "profanity_score": 0.043415543,
                "threat_score": 0.008168601,
                "sexually_explicit_score": 0.028662905,
                "flirtation_score": 0.41074932
            },
            "math_commands.tex": {
                "toxicity_score": 0.01005285,
                "severe_toxicity_score": 0.00084400177,
                "identity_attack_score": 0.0023678627,
                "insult_score": 0.0074368757,
                "profanity_score": 0.011526455,
                "threat_score": 0.006602164,
                "sexually_explicit_score": 0.0059272056,
                "flirtation_score": 0.37736848
            },
            "iclr2024_conference.bib": {
                "toxicity_score": 0.0063144458,
                "severe_toxicity_score": 0.0011444092,
                "identity_attack_score": 0.001276426,
                "insult_score": 0.00649637,
                "profanity_score": 0.013849284,
                "threat_score": 0.0059678215,
                "sexually_explicit_score": 0.011500548,
                "flirtation_score": 0.3139184
            },
            "iclr2024_conference.sty": {
                "toxicity_score": 0.011937759,
                "severe_toxicity_score": 0.00068187714,
                "identity_attack_score": 0.0021643746,
                "insult_score": 0.0094128875,
                "profanity_score": 0.01220964,
                "threat_score": 0.0058739646,
                "sexually_explicit_score": 0.0057207854,
                "flirtation_score": 0.47478196
            },
            "iclr2024_conference.bst": {
                "toxicity_score": 0.014765122,
                "severe_toxicity_score": 0.0014591217,
                "identity_attack_score": 0.0030153254,
                "insult_score": 0.008709883,
                "profanity_score": 0.019553876,
                "threat_score": 0.00746953,
                "sexually_explicit_score": 0.01686747,
                "flirtation_score": 0.5479165
            },
            "atntk.bib": {
                "toxicity_score": 0.0063144458,
                "severe_toxicity_score": 0.001206398,
                "identity_attack_score": 0.0011793067,
                "insult_score": 0.00647737,
                "profanity_score": 0.014054239,
                "threat_score": 0.0059548756,
                "sexually_explicit_score": 0.012562137,
                "flirtation_score": 0.40836242
            }
        },
        "reason": null
    }
}
{
    "num_done": {
        "figure": 0,
        "algorithm": 2,
        "plot": 0
    }
}
{
    "NonTrivialRenderingFilter": {
        "white_pixels_ratio": 89.60280373831776,
        "hash": "0103031f959a3f7f",
        "most_frequent_color_ratio": 89.60280373831776
    }
}