\begin{algorithmic}[1]
\State $\eta_1 = \eta$
\State $\theta_1^{in} = \theta_0$
\For{$b = 1,..., B$}
\State Run SGD with constant step size $\eta_b$ for $t_b$ steps, starting from $\theta_{b}^{in}$
\State Let the last update be $\theta_{b}^{last}$
\State $D_b = \textbf{Diagnostic}(\eta_b, w, l, q, \theta_{b}^{last})$
\State $\theta_{b+1}^{in} = \theta_{D_b}$
\If{$T_{D_b} = S$}
\State $\eta_{b+1} = \gamma\cdot\eta_b$ and $t_{b+1} = \lfloor t_b/\gamma \rfloor$
\Else
\State $\eta_{b+1} = \eta_b$ and $t_{b+1} = t_b$
\EndIf
\EndFor
\end{algorithmic}

\begin{algorithmic}
[1]
\State $\eta_1 = \eta$
\State $\theta_1^{in} = \theta_0$
\For{$b = 1,..., B$}
\State Run SGD with constant step size $\eta_b$ for $t_b$ steps, starting from $\theta_{b}^{in}$
\State Let the last update be $\theta_{b}^{last}$
\State $D_b = \textbf{Diagnostic}(\eta_b, w, l, q, \theta_{b}^{last})$
\State $\theta_{b+1}^{in} = \theta_{D_b}$
\If{$T_{D_b} = S$}
\State $\eta_{b+1} = \gamma\cdot\eta_b$ and $t_{b+1} = \lfloor t_b/\gamma \rfloor$
\Else
\State $\eta_{b+1} = \eta_b$ and $t_{b+1} = t_b$
\EndIf
\EndFor
\end{algorithmic}
{
    "title": "robust learning rate selection for stochastic optimization via splitting   diagnostic",
    "id": "1910.08597",
    "abstract": "this paper proposes splitsgd, a new dynamic learning rate schedule for stochastic optimization. this method decreases the learning rate for better adaptation to the local geometry of the objective function whenever a stationary phase is detected, that is, the iterates are likely to bounce at around a vicinity of a local minimum. the detection is performed by splitting the single thread into two and using the inner product of the gradients from the two threads as a measure of stationarity. owing to this simple yet provably valid stationarity detection, splitsgd is easy-to-implement and essentially does not incur additional computational cost than standard sgd. through a series of extensive experiments, we show that this method is appropriate for both convex problems and training (non-convex) neural networks, with performance compared favorably to other stochastic optimization methods. importantly, this method is observed to be very robust with a set of default parameters for a wide range of problems and, moreover, can yield better generalization performance than other adaptive gradient methods such as adam.",
    "categories": "stat.ml cs.lg math.oc stat.me",
    "doi": "",
    "created": "2019-10-18",
    "updated": "2024-02-16",
    "authors": [
        "matteo sordello",
        "niccol\u00f2 dalmasso",
        "hangfeng he",
        "weijie su"
    ],
    "affiliation": [],
    "url": "https://arxiv.org/abs/1910.08597"
}
{
    "ToxicityFilter": {
        "text_to_toxicity_attributes": {
            "main.tex": {
                "toxicity_score": 0.017969469,
                "severe_toxicity_score": 0.0016021729,
                "identity_attack_score": 0.0061416444,
                "insult_score": 0.009127886,
                "profanity_score": 0.018392462,
                "threat_score": 0.008764107,
                "sexually_explicit_score": 0.01875474,
                "flirtation_score": 0.36146027
            },
            "tmlr.sty": {
                "toxicity_score": 0.006754258,
                "severe_toxicity_score": 0.00047922134,
                "identity_attack_score": 0.0013781702,
                "insult_score": 0.006999873,
                "profanity_score": 0.011458136,
                "threat_score": 0.0057606893,
                "sexually_explicit_score": 0.005543854,
                "flirtation_score": 0.48204565
            },
            "tmlr.bst": {
                "toxicity_score": 0.02048268,
                "severe_toxicity_score": 0.0017642975,
                "identity_attack_score": 0.003921773,
                "insult_score": 0.00983089,
                "profanity_score": 0.023379711,
                "threat_score": 0.008129764,
                "sexually_explicit_score": 0.021585643,
                "flirtation_score": 0.53546125
            }
        },
        "reason": null
    }
}
{
    "num_done": {
        "figure": 0,
        "algorithm": 2,
        "plot": 1
    }
}
{
    "NonTrivialRenderingFilter": {
        "white_pixels_ratio": 95.63997794856827,
        "hash": "a45b0b02bbf1b4bf",
        "most_frequent_color_ratio": 95.63997794856827
    }
}