\label{alg:UCB-TL} \begin{algorithmic}[1]
\State{\textbf{Input:} arm set $\mathcal{I}$, horizon length
$n_{Q}$, smoothness parameters $\beta, C_{\beta}$, transfer parameters $\gamma$, exploration coefficient $\kappa$, $P$-data $\mathcal{D}^{P}$.}
\State{Initialize $\mathcal{L}_{1}\gets\{\mathcal{X}\}$, $\mathcal{I}(\mathcal{X})\gets\mathcal{I}$.} 
    \Comment{initialize partition and arm set}
\State{Initialize the policy $\widetilde{\pi}(\mathcal{X})$ by Procedure~\ref{alg:EA-TL}$\big(\mathcal{X},\mathcal{I}(\mathcal{X}),U,\mathcal{D}^{P} \big)$.}
\State{Initialize $N(\mathcal{X})\gets0$.}
    \Comment{initialize time for policy $\widetilde{\pi}(\mathcal{X})$}
\State{Initialize $\tau_{k}(\mathcal{X}) \gets 0$, and $\tau_{k}^{\star}(\mathcal{X};\mathcal{D}^{P}) $ as in \eqref{eq:tau-star}, $\forall k\in \mathcal{I}(\mathcal{X})$.} 
    \Comment{initialize rounds and set round upper bounds}
\For{$t=1,\dots,n_{Q}$}
\State{Draw a sample $X_{t}^{Q} \sim Q_{X}$.}
\State{Find the bin $B\in\mathcal{L}_{t}$ such that $X_{t}^{Q}\in B$.} 
\While{$|\mathcal{I}(B)|>1$
and $\tau_{k}(B)\geq\tau_{k}^{\star}(B;\mathcal{D}^{P}), \forall k\in\mathcal{I}(B)$}
    \Comment{keep partitioning $B$ until reaching suitable scale}
\If{$\tau_{k}^{\star}(B;\mathcal{D}^{P}) = 0, \forall k\in\mathcal{I}(B)$}
    \Comment{no exploration needed in $B$: discard suboptimal arms}
\State{Set $\underline{Y}^{\star}(B;\mathcal{D}^{P})\gets\max\limits_{k\in\mathcal{I}(B)}\big\{\overline{Y}_{k}^P(B;\mathcal{D}^{P})-  U_{k}(0,B;\mathcal{D}^{P}) \big\}$.}
    \Comment{set largest reward lower bound}
\State{Set $\mathcal{I}(B) \gets \big\{ k \in \mathcal{I}(B) : \overline{Y}_k^P(B;\mathcal{D}^{P})+U_{k}(0,B;\mathcal{D}^{P}) \geq \underline{Y}^{\star}(B;\mathcal{D}^{P}) \big\}$.}
    \Comment{update arm set}
\EndIf
\For{$B'\in\mathsf{child}(B)$}
\State{Set $\mathcal{I}(B')\gets\mathcal{I}(B)$.}
    \Comment{assign remaining arms in $B$ as initial arms in its children}
\State{Initialize the policy $\widetilde{\pi}(B')$ by Procedure~\ref{alg:EA-TL} $ \big(B',\mathcal{I}(B'),U,\mathcal{D}^{P} \big)$.}
\State{Set $N(B')\gets0$.}
    \Comment{initialize time for policy $\widetilde{\pi}(B')$}
\State{Set $\tau_{k}(B')\gets0$ and $\tau_{k}^{\star}(B';\mathcal{D}^{P})$ as in \eqref{eq:tau-star}, $\forall k\in\mathcal{I}(B')$.}
    \Comment{initialize rounds and set round upper bounds}
\EndFor
\State{Set $\mathcal{L}_{t}\gets (\mathcal{L}_{t}\setminus B)\cup\mathsf{child}(B)$.}
    \Comment{replace $B$ with its children in partition}
\State{Find the bin $B\in\mathcal{L}_{t}$ such that $ X_{t}^{Q}\in B$.}
\EndWhile
\State{Set $N(B)\gets N(B)+1$.}
    \Comment{update times $X_{t}^{Q} \in B$}
\State{Set $\pi_{t}\gets\widetilde{\pi}_{N(B)}(B)$.}
    \Comment{choose arm by policy $\widetilde{\pi}(B)$}
\State{Set $\mathcal{I}(B)\gets\widetilde{\mathcal{I}}_{N(B)} (B)$.}
    \Comment{update arm set by policy $\widetilde{\pi}(B)$}
\State{Set $\tau_{k}(B) \gets \widetilde{\tau}_{N(B),k}(B), \forall k\in\mathcal{I}(B)$.}
    \Comment{update numbers of rounds by policy $\widetilde{\pi}(B)$}
\EndFor
\State{\textbf{Output:} policy $\{\pi_{t}\}_{t\geq1}$.}
\end{algorithmic}

\begin{algorithmic}
[1]
\State{\textbf{Input:} arm set $\mathcal{I}$, horizon length
$n_{Q}$, smoothness parameters $\beta, C_{\beta}$, transfer parameters $\gamma$, exploration coefficient $\kappa$, $P$-data $\mathcal{D}^{P}$.}
\State{Initialize $\mathcal{L}_{1}\gets\{\mathcal{X}\}$, $\mathcal{I}(\mathcal{X})\gets\mathcal{I}$.}
\Comment{initialize partition and arm set}
\State{Initialize the policy $\widetilde{\pi}(\mathcal{X})$ by Procedure~\ref{alg:EA-TL}$\big(\mathcal{X},\mathcal{I}(\mathcal{X}),U,\mathcal{D}^{P} \big)$.}
\State{Initialize $N(\mathcal{X})\gets0$.}
\Comment{initialize time for policy $\widetilde{\pi}(\mathcal{X})$}
\State{Initialize $\tau_{k}(\mathcal{X}) \gets 0$, and $\tau_{k}^{\star}(\mathcal{X};\mathcal{D}^{P}) $ as in \eqref{eq:tau-star}, $\forall k\in \mathcal{I}(\mathcal{X})$.}
\Comment{initialize rounds and set round upper bounds}
\For{$t=1,\dots,n_{Q}$}
\State{Draw a sample $X_{t}^{Q} \sim Q_{X}$.}
\State{Find the bin $B\in\mathcal{L}_{t}$ such that $X_{t}^{Q}\in B$.}
\While{$|\mathcal{I}(B)|>1$
and $\tau_{k}(B)\geq\tau_{k}^{\star}(B;\mathcal{D}^{P}), \forall k\in\mathcal{I}(B)$}
\Comment{keep partitioning $B$ until reaching suitable scale}
\If{$\tau_{k}^{\star}(B;\mathcal{D}^{P}) = 0, \forall k\in\mathcal{I}(B)$}
\Comment{no exploration needed in $B$: discard suboptimal arms}
\State{Set $\underline{Y}^{\star}(B;\mathcal{D}^{P})\gets\max\limits_{k\in\mathcal{I}(B)}\big\{\overline{Y}_{k}^P(B;\mathcal{D}^{P})- U_{k}(0,B;\mathcal{D}^{P}) \big\}$.}
\Comment{set largest reward lower bound}
\State{Set $\mathcal{I}(B) \gets \big\{ k \in \mathcal{I}(B) : \overline{Y}_k^P(B;\mathcal{D}^{P})+U_{k}(0,B;\mathcal{D}^{P}) \geq \underline{Y}^{\star}(B;\mathcal{D}^{P}) \big\}$.}
\Comment{update arm set}
\EndIf
\For{$B'\in\mathsf{child}(B)$}
\State{Set $\mathcal{I}(B')\gets\mathcal{I}(B)$.}
\Comment{assign remaining arms in $B$ as initial arms in its children}
\State{Initialize the policy $\widetilde{\pi}(B')$ by Procedure~\ref{alg:EA-TL} $ \big(B',\mathcal{I}(B'),U,\mathcal{D}^{P} \big)$.}
\State{Set $N(B')\gets0$.}
\Comment{initialize time for policy $\widetilde{\pi}(B')$}
\State{Set $\tau_{k}(B')\gets0$ and $\tau_{k}^{\star}(B';\mathcal{D}^{P})$ as in \eqref{eq:tau-star}, $\forall k\in\mathcal{I}(B')$.}
\Comment{initialize rounds and set round upper bounds}
\EndFor
\State{Set $\mathcal{L}_{t}\gets (\mathcal{L}_{t}\setminus B)\cup\mathsf{child}(B)$.}
\Comment{replace $B$ with its children in partition}
\State{Find the bin $B\in\mathcal{L}_{t}$ such that $ X_{t}^{Q}\in B$.}
\EndWhile
\State{Set $N(B)\gets N(B)+1$.}
\Comment{update times $X_{t}^{Q} \in B$}
\State{Set $\pi_{t}\gets\widetilde{\pi}_{N(B)}(B)$.}
\Comment{choose arm by policy $\widetilde{\pi}(B)$}
\State{Set $\mathcal{I}(B)\gets\widetilde{\mathcal{I}}_{N(B)} (B)$.}
\Comment{update arm set by policy $\widetilde{\pi}(B)$}
\State{Set $\tau_{k}(B) \gets \widetilde{\tau}_{N(B),k}(B), \forall k\in\mathcal{I}(B)$.}
\Comment{update numbers of rounds by policy $\widetilde{\pi}(B)$}
\EndFor
\State{\textbf{Output:} policy $\{\pi_{t}\}_{t\geq1}$.}
\end{algorithmic}
{
    "title": "transfer learning for contextual multi-armed bandits",
    "id": "2211.12612",
    "abstract": "motivated by a range of applications, we study in this paper the problem of transfer learning for nonparametric contextual multi-armed bandits under the covariate shift model, where we have data collected on source bandits before the start of the target bandit learning. the minimax rate of convergence for the cumulative regret is established and a novel transfer learning algorithm that attains the minimax regret is proposed. the results quantify the contribution of the data from the source domains for learning in the target domain in the context of nonparametric contextual multi-armed bandits.   in view of the general impossibility of adaptation to unknown smoothness, we develop a data-driven algorithm that achieves near-optimal statistical guarantees (up to a logarithmic factor) while automatically adapting to the unknown parameters over a large collection of parameter spaces under an additional self-similarity assumption. a simulation study is carried out to illustrate the benefits of utilizing the data from the auxiliary source domains for learning in the target domain.",
    "categories": "stat.ml cs.lg math.st stat.th",
    "doi": "",
    "created": "2022-11-22",
    "updated": "2024-01-24",
    "authors": [
        "changxiao cai",
        "t. tony cai",
        "hongzhe li"
    ],
    "affiliation": [],
    "url": "https://arxiv.org/abs/2211.12612"
}
{
    "ToxicityFilter": {
        "text_to_toxicity_attributes": {
            "notation.tex": {
                "toxicity_score": 0.011623607,
                "severe_toxicity_score": 0.0011873245,
                "identity_attack_score": 0.0037182847,
                "insult_score": 0.007607877,
                "profanity_score": 0.015113175,
                "threat_score": 0.0074306927,
                "sexually_explicit_score": 0.012562137,
                "flirtation_score": 0.21015726
            },
            "numerical.tex": {
                "toxicity_score": 0.013068704,
                "severe_toxicity_score": 0.0010585785,
                "identity_attack_score": 0.0039957687,
                "insult_score": 0.007607877,
                "profanity_score": 0.014566627,
                "threat_score": 0.007832011,
                "sexually_explicit_score": 0.012208274,
                "flirtation_score": 0.2357703
            },
            "proof_lower_bound.tex": {
                "toxicity_score": 0.013320025,
                "severe_toxicity_score": 0.001039505,
                "identity_attack_score": 0.004384246,
                "insult_score": 0.0077598775,
                "profanity_score": 0.014327513,
                "threat_score": 0.00746953,
                "sexually_explicit_score": 0.010969753,
                "flirtation_score": 0.3022332
            },
            "Bandit_TL.bbl": {
                "toxicity_score": 0.012314741,
                "severe_toxicity_score": 0.0012016296,
                "identity_attack_score": 0.0041067624,
                "insult_score": 0.007797878,
                "profanity_score": 0.015181494,
                "threat_score": 0.007586042,
                "sexually_explicit_score": 0.011264639,
                "flirtation_score": 0.20508216
            },
            "nQ-K2-beta8-nQ1e5-nP1000000-kappa10-gamma1-ucb100.eps": {
                "toxicity_score": 0.0058118035,
                "severe_toxicity_score": 0.00071525574,
                "identity_attack_score": 0.0011515582,
                "insult_score": 0.006344369,
                "profanity_score": 0.012175481,
                "threat_score": 0.0060196044,
                "sexually_explicit_score": 0.0072836806,
                "flirtation_score": 0.49011314
            },
            "adp-nP-beta8-kappa10-nQ1e5-gamma1bmin5bmax10gmax2under0.5-ucb100.eps": {
                "toxicity_score": 0.0058118035,
                "severe_toxicity_score": 0.00071525574,
                "identity_attack_score": 0.0011515582,
                "insult_score": 0.006344369,
                "profanity_score": 0.012175481,
                "threat_score": 0.0060196044,
                "sexually_explicit_score": 0.0072836806,
                "flirtation_score": 0.49011314
            },
            "nP-K2-beta8-nQ1e5-gamma1-kappa1-ucb100.eps": {
                "toxicity_score": 0.0058118035,
                "severe_toxicity_score": 0.00071525574,
                "identity_attack_score": 0.0011515582,
                "insult_score": 0.006344369,
                "profanity_score": 0.012175481,
                "threat_score": 0.0060196044,
                "sexually_explicit_score": 0.0072836806,
                "flirtation_score": 0.49011314
            },
            "nQ-K4-beta8-nQ1e5-nP1000000-kappa10-gamma1-ucb100.eps": {
                "toxicity_score": 0.0058118035,
                "severe_toxicity_score": 0.00071525574,
                "identity_attack_score": 0.0011515582,
                "insult_score": 0.006344369,
                "profanity_score": 0.012175481,
                "threat_score": 0.0060196044,
                "sexually_explicit_score": 0.0072836806,
                "flirtation_score": 0.49011314
            },
            "discussion.tex": {
                "toxicity_score": 0.013382856,
                "severe_toxicity_score": 0.0012397766,
                "identity_attack_score": 0.0039402717,
                "insult_score": 0.007854878,
                "profanity_score": 0.016479544,
                "threat_score": 0.007935578,
                "sexually_explicit_score": 0.014862247,
                "flirtation_score": 0.22211537
            },
            "lemma.tex": {
                "toxicity_score": 0.011435116,
                "severe_toxicity_score": 0.0013828278,
                "identity_attack_score": 0.0034962974,
                "insult_score": 0.0075508766,
                "profanity_score": 0.016752819,
                "threat_score": 0.007534259,
                "sexually_explicit_score": 0.018047014,
                "flirtation_score": 0.2945637
            },
            "pic_alg.eps": {
                "toxicity_score": 0.006942749,
                "severe_toxicity_score": 0.0008058548,
                "identity_attack_score": 0.0014521659,
                "insult_score": 0.0066863713,
                "profanity_score": 0.012858666,
                "threat_score": 0.0062137907,
                "sexually_explicit_score": 0.0080798725,
                "flirtation_score": 0.43023613
            },
            "gamma-K2-beta8-kappa10-nQ1e5-nP10-ucb100.eps": {
                "toxicity_score": 0.0058118035,
                "severe_toxicity_score": 0.00071525574,
                "identity_attack_score": 0.0011515582,
                "insult_score": 0.006344369,
                "profanity_score": 0.012175481,
                "threat_score": 0.0060196044,
                "sexually_explicit_score": 0.0072836806,
                "flirtation_score": 0.49011314
            },
            "nQ-K2-beta8-nQ1e5-nP3-kappa10-gamma1-ucb100.eps": {
                "toxicity_score": 0.0058118035,
                "severe_toxicity_score": 0.00071525574,
                "identity_attack_score": 0.0011515582,
                "insult_score": 0.006344369,
                "profanity_score": 0.012175481,
                "threat_score": 0.0060196044,
                "sexually_explicit_score": 0.0072836806,
                "flirtation_score": 0.49011314
            },
            "abstract.tex": {
                "toxicity_score": 0.062236045,
                "severe_toxicity_score": 0.002412796,
                "identity_attack_score": 0.013948337,
                "insult_score": 0.024124376,
                "profanity_score": 0.029050145,
                "threat_score": 0.011560392,
                "sexually_explicit_score": 0.018990649,
                "flirtation_score": 0.29612872
            },
            "problem.tex": {
                "toxicity_score": 0.013005874,
                "severe_toxicity_score": 0.0009584427,
                "identity_attack_score": 0.003644289,
                "insult_score": 0.007797878,
                "profanity_score": 0.013917602,
                "threat_score": 0.007547205,
                "sexually_explicit_score": 0.008846575,
                "flirtation_score": 0.23518674
            },
            "tabular.tex": {
                "toxicity_score": 0.00804228,
                "severe_toxicity_score": 0.000705719,
                "identity_attack_score": 0.0023863618,
                "insult_score": 0.0067338715,
                "profanity_score": 0.012380436,
                "threat_score": 0.006738094,
                "sexually_explicit_score": 0.00855169,
                "flirtation_score": 0.23074746
            },
            "Bandit_TL.tex": {
                "toxicity_score": 0.01225191,
                "severe_toxicity_score": 0.0012588501,
                "identity_attack_score": 0.0045507364,
                "insult_score": 0.007607877,
                "profanity_score": 0.015898837,
                "threat_score": 0.0077154995,
                "sexually_explicit_score": 0.013269863,
                "flirtation_score": 0.20662601
            },
            "adaptivity.tex": {
                "toxicity_score": 0.010743983,
                "severe_toxicity_score": 0.0011396408,
                "identity_attack_score": 0.003144818,
                "insult_score": 0.0070948736,
                "profanity_score": 0.014361672,
                "threat_score": 0.007365964,
                "sexually_explicit_score": 0.0122672515,
                "flirtation_score": 0.19645838
            },
            "nQ-K2-beta8-nQ1e5-nP0.5-kappa10-gamma1-ucb100.eps": {
                "toxicity_score": 0.0058118035,
                "severe_toxicity_score": 0.00071525574,
                "identity_attack_score": 0.0011515582,
                "insult_score": 0.006344369,
                "profanity_score": 0.012175481,
                "threat_score": 0.0060196044,
                "sexually_explicit_score": 0.0072836806,
                "flirtation_score": 0.49011314
            },
            "para-nP-beta8-kappa10-nQ1e5-gamma1-ucb100.eps": {
                "toxicity_score": 0.0058118035,
                "severe_toxicity_score": 0.00071525574,
                "identity_attack_score": 0.0011515582,
                "insult_score": 0.006344369,
                "profanity_score": 0.012175481,
                "threat_score": 0.0060196044,
                "sexually_explicit_score": 0.0072836806,
                "flirtation_score": 0.49011314
            },
            "proof_smooth.tex": {
                "toxicity_score": 0.00816794,
                "severe_toxicity_score": 0.00074386597,
                "identity_attack_score": 0.0017204003,
                "insult_score": 0.008044879,
                "profanity_score": 0.013507691,
                "threat_score": 0.0059678215,
                "sexually_explicit_score": 0.009613278,
                "flirtation_score": 0.32680815
            },
            "adp-nP-beta6-kappa10-nQ1e5-gamma1bmin5bmax10gmax2under0.5-ucb100.eps": {
                "toxicity_score": 0.0058118035,
                "severe_toxicity_score": 0.00071525574,
                "identity_attack_score": 0.0011515582,
                "insult_score": 0.006344369,
                "profanity_score": 0.012175481,
                "threat_score": 0.0060196044,
                "sexually_explicit_score": 0.0072836806,
                "flirtation_score": 0.49011314
            },
            "proof_upper_bound.tex": {
                "toxicity_score": 0.014011159,
                "severe_toxicity_score": 0.0012493134,
                "identity_attack_score": 0.0045137387,
                "insult_score": 0.00806388,
                "profanity_score": 0.014976538,
                "threat_score": 0.00780612,
                "sexually_explicit_score": 0.011146685,
                "flirtation_score": 0.2661005
            },
            "bibfileRL.bib": {
                "toxicity_score": 0.010492661,
                "severe_toxicity_score": 0.0011825562,
                "identity_attack_score": 0.0022383702,
                "insult_score": 0.0074178753,
                "profanity_score": 0.014190876,
                "threat_score": 0.0066151097,
                "sexually_explicit_score": 0.010910776,
                "flirtation_score": 0.30652624
            },
            "intro.tex": {
                "toxicity_score": 0.012126249,
                "severe_toxicity_score": 0.0011634827,
                "identity_attack_score": 0.0039402717,
                "insult_score": 0.0075698765,
                "profanity_score": 0.015318131,
                "threat_score": 0.0077025536,
                "sexually_explicit_score": 0.013564749,
                "flirtation_score": 0.2300378
            },
            "proof_adaptivity.tex": {
                "toxicity_score": 0.014073989,
                "severe_toxicity_score": 0.0011968613,
                "identity_attack_score": 0.0047727237,
                "insult_score": 0.007911879,
                "profanity_score": 0.015283971,
                "threat_score": 0.007819066,
                "sexually_explicit_score": 0.012562137,
                "flirtation_score": 0.31647885
            },
            "results.tex": {
                "toxicity_score": 0.013697007,
                "severe_toxicity_score": 0.0010919571,
                "identity_attack_score": 0.004328749,
                "insult_score": 0.007721877,
                "profanity_score": 0.015147334,
                "threat_score": 0.0076637166,
                "sexually_explicit_score": 0.01332884,
                "flirtation_score": 0.20463914
            },
            "nQ-K4-beta8-nQ1e5-nP0.5-kappa10-gamma1-ucb100.eps": {
                "toxicity_score": 0.0058118035,
                "severe_toxicity_score": 0.00071525574,
                "identity_attack_score": 0.0011515582,
                "insult_score": 0.006344369,
                "profanity_score": 0.012175481,
                "threat_score": 0.0060196044,
                "sexually_explicit_score": 0.0072836806,
                "flirtation_score": 0.49011314
            },
            "related-work.tex": {
                "toxicity_score": 0.010555492,
                "severe_toxicity_score": 0.0009775162,
                "identity_attack_score": 0.003921773,
                "insult_score": 0.0072468747,
                "profanity_score": 0.0147374235,
                "threat_score": 0.0075083673,
                "sexually_explicit_score": 0.011382594,
                "flirtation_score": 0.23073672
            },
            "nQ-K4-beta8-nQ1e5-nP3-kappa10-gamma1-ucb100.eps": {
                "toxicity_score": 0.0058118035,
                "severe_toxicity_score": 0.00071525574,
                "identity_attack_score": 0.0011515582,
                "insult_score": 0.006344369,
                "profanity_score": 0.012175481,
                "threat_score": 0.0060196044,
                "sexually_explicit_score": 0.0072836806,
                "flirtation_score": 0.49011314
            },
            "kappa-K2-beta8-gamma1-nQ1e5-nP20-ucb100.eps": {
                "toxicity_score": 0.0058118035,
                "severe_toxicity_score": 0.00071525574,
                "identity_attack_score": 0.0011515582,
                "insult_score": 0.006344369,
                "profanity_score": 0.012175481,
                "threat_score": 0.0060196044,
                "sexually_explicit_score": 0.0072836806,
                "flirtation_score": 0.49011314
            }
        },
        "reason": null
    }
}
{
    "num_done": {
        "figure": 0,
        "algorithm": 3
    }
}
{
    "NonTrivialRenderingFilter": {
        "white_pixels_ratio": 92.62309024556194,
        "hash": "071c3f83843cbf3f",
        "most_frequent_color_ratio": 92.62309024556194
    }
}