\begin{algorithm}[t]
\small
\caption{On-the-fly DA Denoising (ODDA)}\label{alg:alg}
\begin{tabular}{p{2em}p{20em}}
\textbf{Input:} & Teacher model $f_T(\cdot)$, student model $f(\cdot)$, original dataset $\mathcal{D}=\{(x_i, y_i)\}, i=1,\cdots,n$, augmented dataset $\mathcal{D}'=\{(x_i', y_i')\}, i=1,\cdots,kn$, OD temperature $\tau$, SR coefficient $\alpha$. Max training steps for the organic teacher $s_T$ and the student $s_S$. \\
\textbf{Output:} & \hspace{0.5em} The trained student model $f(\cdot)$  \\
\end{tabular}


\begin{algorithmic}[1]
\State Initialize the teacher model $f_T(\cdot)$
\State $s \gets 0$ \Comment{Training steps for OD}
\While{$s < s_T$}
\State Sample a batch $\mathcal{B}$ from $\{(x_i, y_i)\}$
\State Train $f_T(\cdot)$ with cross-entropy loss on $\mathcal{B}$
\EndWhile
\State $s \gets 0$ \Comment{Training steps for Denoising}
\State $\mathcal{D}^+ \gets \{(x_i, y_i)\} \cup \{(x_i', y_i')\}$ \Comment{Mix $\mathcal{D}$ \& $\mathcal{D}'$}
\While{$s < s_S$}
\State Sample a batch $\mathcal{B}'$ from $\mathcal{D}^+$
\State Train $f(\cdot)$ with loss in Eq.~(\ref{eq:overall_loss}) on $\mathcal{B}'$ with Organic Distillation and Self-Regularization to do deonising
\EndWhile
\end{algorithmic}
\end{algorithm}

\begin{algorithm}
[t]
\small
\caption{On-the-fly DA Denoising (ODDA)}\begin{tabular}
{p{2em}p{20em}}
\textbf{Input:} & Teacher model $f_T(\cdot)$, student model $f(\cdot)$, original dataset $\mathcal{D}=\{(x_i, y_i)\}, i=1,\cdots,n$, augmented dataset $\mathcal{D}'=\{(x_i', y_i')\}, i=1,\cdots,kn$, OD temperature $\tau$, SR coefficient $\alpha$. Max training steps for the organic teacher $s_T$ and the student $s_S$. \\
\textbf{Output:} & \hspace{0.5em} The trained student model $f(\cdot)$ \\
\end{tabular}
\begin{algorithmic}
[1]
\State Initialize the teacher model $f_T(\cdot)$
\State $s \gets 0$ \Comment{Training steps for OD}
\While{$s < s_T$}
\State Sample a batch $\mathcal{B}$ from $\{(x_i, y_i)\}$
\State Train $f_T(\cdot)$ with cross-entropy loss on $\mathcal{B}$
\EndWhile
\State $s \gets 0$ \Comment{Training steps for Denoising}
\State $\mathcal{D}^+ \gets \{(x_i, y_i)\} \cup \{(x_i', y_i')\}$ \Comment{Mix $\mathcal{D}$ \& $\mathcal{D}'$}
\While{$s < s_S$}
\State Sample a batch $\mathcal{B}'$ from $\mathcal{D}^+$
\State Train $f(\cdot)$ with loss in Eq.~(\ref{eq:overall_loss}) on $\mathcal{B}'$ with Organic Distillation and Self-Regularization to do deonising
\EndWhile
\end{algorithmic}
\end{algorithm}
{
    "title": "on-the-fly denoising for data augmentation in natural language   understanding",
    "id": "2212.10558",
    "abstract": "data augmentation (da) is frequently used to provide additional training data without extra human annotation automatically. however, data augmentation may introduce noisy data that impairs training. to guarantee the quality of augmented data, existing methods either assume no noise exists in the augmented data and adopt consistency training or use simple heuristics such as training loss and diversity constraints to filter out \"noisy\" data. however, those filtered examples may still contain useful information, and dropping them completely causes a loss of supervision signals. in this paper, based on the assumption that the original dataset is cleaner than the augmented data, we propose an on-the-fly denoising technique for data augmentation that learns from soft augmented labels provided by an organic teacher model trained on the cleaner original data. to further prevent overfitting on noisy labels, a simple self-regularization module is applied to force the model prediction to be consistent across two distinct dropouts. our method can be applied to general augmentation techniques and consistently improve the performance on both text classification and question-answering tasks.",
    "categories": "cs.cl cs.ai",
    "doi": "",
    "created": "2022-12-20",
    "updated": "2024-01-31",
    "authors": [
        "tianqing fang",
        "wenxuan zhou",
        "fangyu liu",
        "hongming zhang",
        "yangqiu song",
        "muhao chen"
    ],
    "affiliation": [],
    "url": "https://arxiv.org/abs/2212.10558"
}
{
    "ToxicityFilter": {
        "text_to_toxicity_attributes": {
            "06_conclusion.tex": {
                "toxicity_score": 0.007853788,
                "severe_toxicity_score": 0.00081539154,
                "identity_attack_score": 0.0022846176,
                "insult_score": 0.00646787,
                "profanity_score": 0.012722028,
                "threat_score": 0.006912862,
                "sexually_explicit_score": 0.00896453,
                "flirtation_score": 0.18686433
            },
            "00_main.bbl": {
                "toxicity_score": 0.01225191,
                "severe_toxicity_score": 0.0011777878,
                "identity_attack_score": 0.0039957687,
                "insult_score": 0.007607877,
                "profanity_score": 0.014839902,
                "threat_score": 0.007598988,
                "sexually_explicit_score": 0.0122672515,
                "flirtation_score": 0.26163515
            },
            "00_main.tex": {
                "toxicity_score": 0.021549871,
                "severe_toxicity_score": 0.0016880035,
                "identity_attack_score": 0.0057346676,
                "insult_score": 0.0096028885,
                "profanity_score": 0.017914234,
                "threat_score": 0.0091524795,
                "sexually_explicit_score": 0.015923835,
                "flirtation_score": 0.28087527
            },
            "xx_appendix.tex": {
                "toxicity_score": 0.01646154,
                "severe_toxicity_score": 0.0016212463,
                "identity_attack_score": 0.0034408006,
                "insult_score": 0.008424882,
                "profanity_score": 0.017777596,
                "threat_score": 0.00866054,
                "sexually_explicit_score": 0.016395653,
                "flirtation_score": 0.19581212
            },
            "custom.bib": {
                "toxicity_score": 0.0054034065,
                "severe_toxicity_score": 0.0010871887,
                "identity_attack_score": 0.000966569,
                "insult_score": 0.0062873685,
                "profanity_score": 0.013917602,
                "threat_score": 0.0057347976,
                "sexually_explicit_score": 0.011795434,
                "flirtation_score": 0.33817694
            },
            "05_results_and_analysis.tex": {
                "toxicity_score": 0.009613037,
                "severe_toxicity_score": 0.00091552734,
                "identity_attack_score": 0.0032188136,
                "insult_score": 0.006838372,
                "profanity_score": 0.01309778,
                "threat_score": 0.007197669,
                "sexually_explicit_score": 0.009613278,
                "flirtation_score": 0.20907715
            },
            "03_method.tex": {
                "toxicity_score": 0.006377276,
                "severe_toxicity_score": 0.00074863434,
                "identity_attack_score": 0.0017758971,
                "insult_score": 0.006372869,
                "profanity_score": 0.012517073,
                "threat_score": 0.0065180166,
                "sexually_explicit_score": 0.009908165,
                "flirtation_score": 0.25127685
            },
            "arr.sty": {
                "toxicity_score": 0.017843807,
                "severe_toxicity_score": 0.0015068054,
                "identity_attack_score": 0.005105704,
                "insult_score": 0.0094128875,
                "profanity_score": 0.018734055,
                "threat_score": 0.007365964,
                "sexually_explicit_score": 0.018164968,
                "flirtation_score": 0.3796501
            },
            "04_experiments.tex": {
                "toxicity_score": 0.013257195,
                "severe_toxicity_score": 0.0015926361,
                "identity_attack_score": 0.0034038029,
                "insult_score": 0.007816878,
                "profanity_score": 0.01893901,
                "threat_score": 0.007637825,
                "sexually_explicit_score": 0.015452018,
                "flirtation_score": 0.40788114
            },
            "acl_natbib.bst": {
                "toxicity_score": 0.013697007,
                "severe_toxicity_score": 0.0011348724,
                "identity_attack_score": 0.005253696,
                "insult_score": 0.00810188,
                "profanity_score": 0.015010698,
                "threat_score": 0.007547205,
                "sexually_explicit_score": 0.012385205,
                "flirtation_score": 0.42531034
            }
        },
        "reason": null
    }
}
{
    "num_done": {
        "figure": 0,
        "algorithm": 2
    }
}
{
    "NonTrivialRenderingFilter": {
        "white_pixels_ratio": 92.06315789473685,
        "hash": "1c3f0f3f7c1f000f",
        "most_frequent_color_ratio": 92.06315789473685
    }
}